Files
jobboard/app/cli.py
T
WolfandClaude Opus 5.5 06ff9f549b List unlinked places and unmapped words under Sources, fix them in code with the unmapped skill
Settings > Sources > Unmapped shows the place texts no place resolves, with
their roles' countries and companies, and the host words no table maps, and
copies both as text. Prep fingerprints places.seed(), so a town or spelling
added to the code is seeded and every open role is mapped again at the next
prep. app prep unlinked lists the places on the command line.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 13:06:22 +02:00

724 lines
31 KiB
Python

"""The command line.
app serve [--host H] [--port P] [--reload]
app job pull|extract|embed|score [--payload JSON]
app worker [--workers http=8,cpu=1] [--until-idle]
app companies seed|add|edit|checked|remove|by-hand|set-board|detach|discover|check
app searches add|list|remove
app agents list|prompt
app stages [extract|summarise on|off]
app dupes
app prep [unmapped|unlinked] [--remap]
app db init
`app job` queues the work on a queue of its own and drains it here, so a
cascade such as `pull` runs every stage it starts before the command returns,
and prints one line per stage. `app worker` is the boilerplate's: the same
queue with no web app, and the schedules fired from it.
`python -m app` is the same thing, for a checkout with no console script.
"""
from __future__ import annotations
import argparse
import json
import logging
import os
import sys
from pathlib import Path
from typing import TYPE_CHECKING, Any
from app import config as config_mod
if TYPE_CHECKING:
from app.storage import Store
#: The one plugin this app ships lives outside the `app` package, in
#: `plugins/`, so it is discovered by `desk.sources.registry` the same way a
#: user's own drop-in plugin would be. `companies` and `job pull` both want
#: its plain functions (`is_known_target`, `careers_url`, `fetch_board`) as
#: well as the `Source` class, so the repo root goes on `sys.path` once, here,
#: rather than each caller guessing where `plugins/` is relative to `cwd`.
_ROOT = Path(__file__).resolve().parents[1]
if str(_ROOT) not in sys.path:
sys.path.insert(0, str(_ROOT))
def build_parser() -> argparse.ArgumentParser:
from app.models import HAND_REASONS
parser = argparse.ArgumentParser(
prog="app", description="roledesk: a job-search engine on desk and the pipeline queue.")
parser.add_argument("-c", "--config", default=None,
help="the YAML file to read (default: $APP_CONFIG, then config.yaml)")
parser.add_argument("-v", "--verbose", action="store_true")
sub = parser.add_subparsers(dest="command", required=True)
serve = sub.add_parser("serve", help="run the web app")
serve.add_argument("--host", default="", help="override config server.host")
serve.add_argument("--port", type=int, default=0, help="override config server.port")
serve.add_argument("--reload", action="store_true", help="restart on a source change")
job = sub.add_parser("job", help="queue a job, task or cascade and drain the queue")
job.add_argument("kind", help="pull, extract, embed, score, or any registered job")
job.add_argument("--payload", default="{}",
help="JSON spread over a job handler's keyword arguments")
worker = sub.add_parser("worker", help="drain the job table with no web app")
worker.add_argument("--workers", default="", help="resource=count,... e.g. cpu=2")
worker.add_argument("--until-idle", action="store_true",
help="run what is queued, then exit")
companies = sub.add_parser("companies", help="the target company list")
companies_sub = companies.add_subparsers(dest="action", required=True)
seed = companies_sub.add_parser(
"seed", help="import seed/companies.json, and the boards it names for rows without one")
seed.add_argument("--path", default="seed/companies.json")
# One word after `add` is a company for the list checked by hand, two are
# a board and its company, which is what `add` meant before that list.
add = companies_sub.add_parser(
"add", help="add NAME to the list checked by hand, or with BOARD NAME follow a "
"company by its board, attaching or replacing an existing one's")
add.add_argument("first", metavar="NAME|BOARD",
help="the company, or host:slug, e.g. greenhouse:acme, before its name")
add.add_argument("second", metavar="NAME", nargs="?")
add.add_argument("--careers-url", default="", help="its careers page, http or https")
add.add_argument("--note", default="")
add.add_argument("--reason", default="added", choices=HAND_REASONS, metavar="REASON",
help=f"one of {', '.join(repr(r) for r in HAND_REASONS)}")
edit = companies_sub.add_parser("edit", help="change a company's name, link, note or reason")
edit.add_argument("key")
edit.add_argument("--name")
edit.add_argument("--careers-url")
edit.add_argument("--note")
edit.add_argument("--reason", choices=HAND_REASONS, metavar="REASON",
help=f"one of {', '.join(repr(r) for r in HAND_REASONS)}")
checked = companies_sub.add_parser(
"checked", help="record that its careers page was looked at just now")
checked.add_argument("key")
remove = companies_sub.add_parser(
"remove", help="delete a company with no board on and no open roles")
remove.add_argument("key")
companies_sub.add_parser(
"by-hand", help="the companies checked by hand, never checked first, then the "
"longest ago")
set_board = companies_sub.add_parser(
"set-board", help="attach a board to a company, or replace the one it has")
set_board.add_argument("key", help="the company key, e.g. trade-republic")
set_board.add_argument("board", help="host:slug, e.g. greenhouse:acme")
detach = companies_sub.add_parser(
"detach", help="stop reading a company's board and close its roles; the row stays")
detach.add_argument("key")
discover = companies_sub.add_parser(
"discover", help="find a company's board from its website; --add follows it")
discover.add_argument("name")
discover.add_argument("domain", help="its website, e.g. pleo.io")
discover.add_argument("--add", action="store_true",
help="add or update the company: its board when one is readable, "
"else the host it runs on")
companies_sub.add_parser("check", help="report boards that do not answer, now or in the pull")
searches = sub.add_parser("searches", help="saved searches on cross-company job boards")
searches_sub = searches.add_subparsers(dest="action", required=True)
search_add = searches_sub.add_parser(
"add", help="run a search once, then save it for every pull (or update it)")
search_add.add_argument("provider", choices=["jobindex", "adzuna", "platsbanken"])
search_add.add_argument("name", help='what to call it, e.g. "Copenhagen backend"')
search_add.add_argument("--q", required=True,
help="the query, in the provider's own syntax")
search_add.add_argument("--area", type=int, default=0,
help="jobindex geoareaid: 1 Region Hovedstaden, 8 Aarhus")
search_add.add_argument("--country", default="",
help="adzuna country: gb, de, nl, fr, at, be, es, it, pl, ch")
search_add.add_argument("--where", default="", help="adzuna place, e.g. Berlin")
search_add.add_argument("--region", default="",
help="platsbanken län id: CaRE_1nn_cSU is Skåne, empty all of Sweden")
search_add.add_argument("--pages", type=int, default=0,
help="result pages per pull (default 5, at most 10)")
searches_sub.add_parser("list", help="every saved search and its open roles")
search_remove = searches_sub.add_parser(
"remove", help="stop running a search and close its open roles")
search_remove.add_argument("name")
agents = sub.add_parser("agents", help="search agents and their prompts")
agents_sub = agents.add_subparsers(dest="action", required=True)
agents_sub.add_parser("list", help="every agent and its prompts")
prompt = agents_sub.add_parser(
"prompt", help="add a prompt to an agent, replace its text, or --remove it")
prompt.add_argument("agent", help="the agent's name")
prompt.add_argument("name", help='the prompt\'s name, e.g. "AI leadership"')
prompt.add_argument("text", nargs="?", default="",
help="what the agent looks for, in a sentence or a paragraph")
prompt.add_argument("--remove", action="store_true", help="remove the prompt instead")
stages = sub.add_parser(
"stages", help="show whether extract, summarise and enrich run, or switch one")
stages.add_argument("name", nargs="?", choices=["extract", "summarise", "enrich"])
stages.add_argument("state", nargs="?", choices=["on", "off"])
sub.add_parser("seniority", help="per host, how often the stated level equals the model's")
sub.add_parser("dupes", help="recompute which open roles are copies of one posting")
prep = sub.add_parser(
"prep", help="map the roles' host words and fold the copies, or list what is unmapped")
prep.add_argument("action", nargs="?", choices=["unmapped", "unlinked"],
help="list the host words no table maps, or the place texts no "
"place resolves, instead of running")
prep.add_argument("--remap", action="store_true",
help="map every role again, closed ones included")
db = sub.add_parser("db", help="database housekeeping")
db.add_argument("action", choices=["init"])
return parser
def main(argv: list[str] | None = None) -> int:
args = build_parser().parse_args(argv)
logging.basicConfig(level=logging.DEBUG if args.verbose else logging.INFO,
format="%(asctime)s %(levelname)s %(name)s: %(message)s")
try:
if args.command == "serve":
return serve(args)
if args.command == "job":
return run_job(args)
if args.command == "worker":
return run_worker(args)
if args.command == "companies":
return run_companies(args)
if args.command == "searches":
return run_searches(args)
if args.command == "agents":
return run_agents(args)
if args.command == "stages":
return run_stages(args)
if args.command == "seniority":
return run_seniority(args)
if args.command == "dupes":
return run_dupes(args)
if args.command == "prep":
return run_prep(args)
return init_db(args)
except (FileNotFoundError, config_mod.ConfigError) as exc:
print(exc, file=sys.stderr)
return 1
def serve(args: argparse.Namespace) -> int:
import uvicorn
config = config_mod.load(args.config) # a config error here, not in uvicorn
host = args.host or config.server.host
port = args.port or config.server.port
level = str(config.log_level).lower()
if args.reload:
# The reloader builds the app in a fresh process from an import
# string, and that process reads the config from $APP_CONFIG.
if args.config:
os.environ["APP_CONFIG"] = str(args.config)
uvicorn.run("app.main:app", host=host, port=port, reload=True, log_level=level)
return 0
from app.main import create_app
uvicorn.run(create_app(config), host=host, port=port, log_level=level)
return 0
def run_job(args: argparse.Namespace) -> int:
"""Queue `kind` (a cascade such as `pull`, a task from `pipeline.TASKS`,
or a job), run everything queued until the queue is idle, and print how
it ended: one line per task, from each task's final job. `pull` is a
cascade, so it prints a line for every stage it went through.
The queue fires no schedules: the command asked for one piece of work."""
from app import jobs
from app.storage import Store
from pipeline import FINAL, JOBS, TASKS, JobQueue, Jobs
cascade = jobs.cascade
try:
payload: dict[str, Any] = json.loads(args.payload or "{}")
except json.JSONDecodeError as exc:
print(f"--payload is not JSON: {exc}", file=sys.stderr)
return 2
chained = args.kind in cascade.CASCADES
if not chained and args.kind not in TASKS and args.kind not in JOBS:
known = sorted(set(JOBS) | set(TASKS))
print(f"no job kind {args.kind!r}. Have: {', '.join(known)}", file=sys.stderr)
return 2
config = config_mod.load(args.config)
store = Store(config.database)
queue = JobQueue(Jobs(str(config.database)), config, store,
workers=config.workers.model_dump(),
bootstrap=jobs.BOOTSTRAP, bootstrap_arg=str(config.path or ""),
tz=config.timezone, schedules=False)
if chained or args.kind not in JOBS:
first = cascade.chain(args.kind) if chained else TASKS[args.kind]()
if args.kind == "pull":
# The same claim the `pull` job makes, so a pull here and one in
# the server's worker cannot both fetch.
run = jobs.pull.claim_run(store)
if not run:
print(json.dumps({"task": "pull", "result": jobs.pull.ALREADY_RUNNING,
"running": jobs.pull.running(store)}))
return 1
first = cascade.pull_chain(run)
task_id = queue.start_task(first)
queue.drain()
started = queue.jobs.of_task(task_id)
if not started:
print(json.dumps({"task": args.kind, "result": "nothing to do"}))
return 0
whole = queue.jobs.of_cascade(started[0].cascade_id)
finals = [j for j in whole if j.kind == FINAL]
for final in finals:
# A task's own jobs say what the work did; the final job only
# tallies how they ended. Up to five, so a pull of twenty board
# batches stays one readable line.
work = [j.result for j in whole if j.task_id == final.task_id
and j.kind != FINAL][:5]
print(json.dumps({"id": final.id, "state": final.state,
"result": final.result, "work": work,
"error": final.error}))
return 0 if all(f.state == "done" for f in finals) else 1
queued = queue.enqueue(args.kind, payload)
queue.drain()
if queued is not None:
done = queue.jobs.get(queued.id)
if done is not None:
print(json.dumps({"id": done.id, "state": done.state, "result": done.result,
"error": done.error}))
return 0 if done.state == "done" else 1
return 0
def run_worker(args: argparse.Namespace) -> int:
"""`python -m pipeline.worker` with this app's settings filled in: the
bootstrap, the pools from `workers:`, and the timezone, without which a
`daily at 06:00` schedule would run at 06:00 on the machine's clock."""
from app.jobs import BOOTSTRAP
from pipeline.worker import main as worker_main
config = config_mod.load(args.config)
argv = ["--db", str(config.database), "--bootstrap", BOOTSTRAP,
"--bootstrap-arg", str(args.config or config.path or ""),
"--timezone", str(config.timezone)]
argv += ["--workers", args.workers or
",".join(f"{name}={n}" for name, n in config.workers.model_dump().items())]
if args.until_idle:
argv.append("--until-idle")
return worker_main(argv)
def run_companies(args: argparse.Namespace) -> int:
from app.storage import Store
config = config_mod.load(args.config)
store = Store(config.database)
if args.action == "seed":
return seed_companies(store, args.path)
if args.action == "add":
return add_company(store, args)
if args.action == "edit":
return edit_company(store, args)
if args.action == "checked":
return company_checked(store, args.key)
if args.action == "remove":
return remove_company(store, args.key)
if args.action == "by-hand":
return list_by_hand(store)
if args.action == "set-board":
return set_board(store, args.key, args.board)
if args.action == "detach":
return detach_board(store, args.key)
if args.action == "discover":
return discover_company(store, args.name, args.domain, add=args.add)
return check_companies(store)
def seed_companies(store: Store, path: str) -> int:
"""`app companies seed`: `app.jobs.sources.seed_companies`, its counts as
JSON and each entry that did not land whole on stderr."""
from app import jobs
result = jobs.sources.seed_companies(store, path)
for problem in result.pop("problems"):
print(f"companies seed: {problem}", file=sys.stderr)
print(json.dumps(result))
return 0
def discover_company(store: Store, name: str, domain: str, *, add: bool) -> int:
"""`app companies discover "Pleo" pleo.io [--add]`: what the company's
site says its board is, as JSON. With `--add`, the company is added or
updated: its board attached or replaced when one is readable, else the
host and slug it runs on recorded; see `add_discovered`."""
from app import jobs
found = jobs.discover.discover(name, domain)
out: dict[str, Any] = {"name": name, "domain": domain, "board": found.board,
"ats": found.ats, "ats_slug": found.ats_slug,
"careers_url": found.careers_url,
"evidence": found.evidence}
if add:
try:
out["key"], out["roles"] = jobs.sources.add_discovered(store, name, domain, found)
except ValueError as exc:
print(f"companies discover: {exc}", file=sys.stderr)
return 1
print(json.dumps(out))
return 0
def add_company(store: Store, args: argparse.Namespace) -> int:
"""`app companies add "Acme" [--careers-url URL] [--note TEXT] [--reason
REASON]` puts a company with no board on the list checked by hand;
`app companies add greenhouse:acme "Acme"` follows it by its board."""
if args.second is not None:
return add_company_by_board(store, args.first, args.second)
from app import jobs
try:
company = jobs.sources.add_by_hand(store, args.first, careers_url=args.careers_url,
notes=args.note, hand_reason=args.reason)
except ValueError as exc:
print(f"companies add: {exc}", file=sys.stderr)
return 1
print(json.dumps({"key": company.key, "hand_reason": company.hand_reason}))
return 0
def add_company_by_board(store: Store, board: str, name: str) -> int:
"""`app companies add greenhouse:acme "Acme"`: fetch the board first
and refuse if it does not answer or is not a known host. A company that
exists gets the board attached or replaced. Port of seedgen's `-add`."""
from app import jobs
from app.models import company_key
try:
sub, found = jobs.sources.add_board(store, board, name)
except ValueError as exc:
print(f"companies add: {exc}", file=sys.stderr)
return 1
print(json.dumps({"key": company_key(sub.company), "board": sub.target, "roles": found}))
return 0
def edit_company(store: Store, args: argparse.Namespace) -> int:
"""`app companies edit acme --careers-url https://acme.example/jobs`: only
the options given change; see `jobs.sources.edit_company`."""
from app import jobs
given = {"name": args.name, "careers_url": args.careers_url, "notes": args.note,
"hand_reason": args.reason}
try:
company = jobs.sources.edit_company(
store, args.key, **{f: v for f, v in given.items() if v is not None})
except KeyError:
print(f"companies edit: no company {args.key!r}", file=sys.stderr)
return 1
except ValueError as exc:
print(f"companies edit: {exc}", file=sys.stderr)
return 1
print(json.dumps({"key": company.key, "name": company.name,
"careers_url": company.careers_url, "notes": company.notes,
"hand_reason": company.hand_reason}))
return 0
def company_checked(store: Store, key: str) -> int:
"""`app companies checked acme`: its careers page was looked at just now,
which the Watchlist keeps as `careers_opened_at`, the same stamp opening
the page from there sets."""
if store.company(key) is None:
print(f"companies checked: no company {key!r}", file=sys.stderr)
return 1
company = store.update_company(key, careers_opened_at=store.now())
print(json.dumps({"key": key, "careers_opened_at": company.careers_opened_at}))
return 0
def remove_company(store: Store, key: str) -> int:
"""`app companies remove acme`: refused for a company still read, see
`jobs.sources.remove_company`."""
from app import jobs
try:
jobs.sources.remove_company(store, key)
except KeyError:
print(f"companies remove: no company {key!r}", file=sys.stderr)
return 1
except ValueError as exc:
print(f"companies remove: {exc}", file=sys.stderr)
return 1
print(json.dumps({"key": key, "removed": True}))
return 0
def list_by_hand(store: Store) -> int:
"""`app companies by-hand`: the companies with no board shown that are not
known from a search only, one line each, never opened first and then the
longest ago; `watch` is the Watchlist's rating, yes, no or a dash for
unrated. See `jobs.sources.by_hand`."""
from app import jobs
rows = jobs.sources.by_hand(store)
for c in rows:
watch = "-" if c.watch is None else "yes" if c.watch else "no"
opened = c.careers_opened_at[:10] or "never"
print(f"{c.name:30} {c.hand_reason or '-':8} {watch:4} {opened:10} {c.careers_url}")
print(f"{len(rows)} by hand")
return 0
def set_board(store: Store, key: str, board: str) -> int:
"""`app companies set-board trade-republic greenhouse:traderepublic`:
attach the board to an existing company or replace the one it has."""
from app import jobs
try:
sub, found, closed = jobs.sources.set_board(store, key, board)
except ValueError as exc:
print(f"companies set-board: {exc}", file=sys.stderr)
return 1
print(json.dumps({"key": key, "board": sub.target, "roles": found, "closed": closed}))
return 0
def detach_board(store: Store, key: str) -> int:
"""`app companies detach acme`: stop reading its board, close its roles."""
from app import jobs
try:
closed = jobs.sources.detach_board(store, key)
except KeyError:
print(f"companies detach: no company {key!r}", file=sys.stderr)
return 1
except ValueError as exc:
print(f"companies detach: {exc}", file=sys.stderr)
return 1
print(json.dumps({"key": key, "closed": closed}))
return 0
def check_companies(store: Store) -> int:
"""`app companies check`: every `ats` board, fetched once. Prints a
line per board that did not answer now, answered incompletely, or has
been failing in the pull, with what the pull recorded beside the live
result, and exits 1 if any did not answer now."""
from app import jobs
from plugins.ats import failure_reason, fetch_board
subs = [s for s in store.subscriptions(source="ats") if s.show]
failing = jobs.pull.failing_boards(store)
dead = 0
for sub in subs:
try:
rows, complete = fetch_board(sub.target)
live = f"{len(rows)} roles" + ("" if complete else ", incomplete")
except Exception as exc:
complete, live = False, f"not answering: {failure_reason(exc)}"
dead += 1
since, reason = failing.get(sub.target, ("", ""))
if complete and not since:
continue
stored = f"pull: failing since {since[:10]}: {reason}" if since else "pull: answering"
print(f"{sub.target} ({sub.label}): now {live}; {stored}")
print(f"checked {len(subs)} boards, {dead} dead, {len(failing)} failing in the pull")
return 1 if dead else 0
def run_searches(args: argparse.Namespace) -> int:
from app import jobs
from app.storage import Store
config = config_mod.load(args.config)
store = Store(config.database)
if args.action == "add":
return add_search(config, store, args)
if args.action == "remove":
sub = store.subscription_for("search", args.name)
if sub is None:
print(f"searches remove: no search named {args.name!r}", file=sys.stderr)
return 1
_, closed = jobs.sources.remove_search(store, sub.id)
print(json.dumps({"removed": sub.label, "roles_closed": closed}))
return 0
countries = store.open_countries_by_subscription()
for sub in store.subscriptions(source="search"):
removed = "" if sub.subscribed and sub.fetch else " (off)"
open_roles = sum(countries.get(sub.id, {}).values())
print(f"{sub.label:30} {sub.target:40} {open_roles:5} open "
f"{json.dumps(sub.params, ensure_ascii=False)}{removed}")
return 0
def add_search(config: config_mod.Config, store: Store, args: argparse.Namespace) -> int:
"""`app searches add jobindex "Copenhagen backend" --q '"backend developer"'
--area 1`: run the search once and save it only if it finds something.
A search of the same name is updated in place."""
from app import jobs
try:
params = jobs.sources.search_params(args.provider, args.q, area=args.area,
country=args.country, where=args.where,
region=args.region, pages=args.pages)
sub, found, complete = jobs.sources.save_search(config, store, args.provider, args.name,
params)
except ValueError as exc:
print(f"searches add: {exc}", file=sys.stderr)
return 1
print(json.dumps({"search": sub.label, "target": sub.target, "roles": found,
"complete": complete}, ensure_ascii=False))
return 0
def run_agents(args: argparse.Namespace) -> int:
"""`app agents list` prints every agent and its prompts. `app agents
prompt "Wolf, next role" "AI leadership" "Head of AI ..."` adds that
prompt at the end, or replaces the text of the one of that name in its
place; `--remove` removes it. The agent's other fields stay as they are.
Scores follow at the next `score`."""
from app.models import Prompt
from app.storage import Store
config = config_mod.load(args.config)
store = Store(config.database)
if args.action == "list":
for each in store.agents():
print(each.name + ("" if each.enabled else " (off)"))
for p in each.prompts:
text = " ".join(p.text.split())
print(f" {p.name}: {text[:100] + '…' if len(text) > 100 else text}")
return 0
agent = next((a for a in store.agents() if a.name == args.agent), None)
if agent is None:
print(f"agents prompt: no agent named {args.agent!r}", file=sys.stderr)
return 1
named = args.name.strip().casefold()
prompts = [p for p in agent.prompts if not (args.remove and p.name.casefold() == named)]
if args.remove and len(prompts) == len(agent.prompts):
print(f"agents prompt: {agent.name!r} has no prompt named {args.name!r}",
file=sys.stderr)
return 1
if not args.remove:
try:
new = Prompt(name=args.name, text=args.text)
except ValueError as exc:
print(f"agents prompt: {exc}", file=sys.stderr)
return 1
at = next((n for n, p in enumerate(prompts) if p.name.casefold() == named), None)
if at is None:
prompts.append(new)
else:
prompts[at] = new
saved = store.save_agent(agent.name, gates=agent.gates, weights=agent.weights,
enabled=agent.enabled, agent_id=agent.id, limits=agent.limits,
prompts=prompts)
print(json.dumps({"agent": saved.name, "prompts": [p.name for p in saved.prompts]},
ensure_ascii=False))
return 0
def run_stages(args: argparse.Namespace) -> int:
"""`app stages` prints what `GET /api/stages` returns; `app stages extract
on` stores the switch through the function `PUT /api/stages/extract`
calls, and prints the result."""
from app import jobs
from app.storage import Store
if bool(args.name) != bool(args.state):
print("stages: give a stage and on or off, or neither", file=sys.stderr)
return 1
config = config_mod.load(args.config)
store = Store(config.database)
if args.name:
jobs.stages.set_stage(store, args.name, args.state == "on")
found = [jobs.stages.describe_one(config, store, args.name)]
else:
found = jobs.stages.describe(config, store)
for stage in found:
where = "set by the switch" if stage.source == "stored" else "config.yaml's default"
print(f"{stage.name:10} {'on ' if stage.on else 'off'} {where}; "
f"{stage.model}; {stage.answer}")
return 0
def run_seniority(args: argparse.Namespace) -> int:
"""`app seniority`: per host, the roles that have both a stated and a
model level and how many of them agree, `Store.seniority_agreement`, as
a table. For deciding which host's stated level to trust."""
from app.storage import Store
config = config_mod.load(args.config)
found = Store(config.database).seniority_agreement()
if not found:
print("no role has a stated level yet")
return 0
print(f"{'host':20} {'compared':>8} {'agreed':>7} {'rate':>6} {'unread':>7}")
for h in found:
rate = "-" if h["rate"] is None else f"{h['rate']:.0%}"
print(f"{h['host'] or '(none)':20} {h['compared']:8} {h['agreed']:7} {rate:>6} "
f"{h['stated_only']:7}")
return 0
def run_dupes(args: argparse.Namespace) -> int:
"""`app dupes`: what the prep stage ends with, `Store.rebuild_dupes`,
now. Prints the groups (kept roles with a copy), the copies the feed
hides and the rows whose `dup_of` changed, as JSON."""
from app.storage import Store
config = config_mod.load(args.config)
counts = Store(config.database).rebuild_dupes()
print(json.dumps(counts._asdict()))
return 0
def run_prep(args: argparse.Namespace) -> int:
"""`app prep`: the prep stage now (`app.prep.run`), printing what it did
as JSON; `--remap` maps every role again. `app prep unmapped` lists the
host words no table maps, per field and host, most roles first; `app
prep unlinked` the place texts of open roles no place resolves, with the
countries and companies of their roles."""
from app import prep
from app.storage import Store
config = config_mod.load(args.config)
store = Store(config.database)
if args.action == "unmapped":
found = store.unmapped()
if not found:
print("every host word is mapped")
for w in found:
print(f"{w['field']:11} {w['host'] or '(none)':16} {w['roles']:6} {w['word']}")
return 0
if args.action == "unlinked":
linked = store.unlinked_places()
if not linked:
print("every place text is linked")
for p in linked:
where = " ".join(f"{c or '-'}:{n}" for c, n in p["countries"][:3])
print(f"{p['roles']:6} {p['text'][:40]:40} {where:20} {', '.join(p['companies'])}")
return 0
print(json.dumps(prep.run(store, remap=args.remap)._asdict()))
return 0
def init_db(args: argparse.Namespace) -> int:
"""Create the file, or bring it to the current schema."""
from app.storage import LATEST, Store
config = config_mod.load(args.config)
store = Store(config.database)
applied = store.migrations_applied
print(f"{store.path}: schema v{LATEST}"
+ (f", applied {applied}" if applied else ""))
return 0
if __name__ == "__main__":
raise SystemExit(main())