Seed 186 companies, add roledesk companies, and pull and close every board

seed/companies.json: 179 companies from jobhunt-hq's seed.json (MIT),
reduced to key/name/board/careers_url, plus a hand-added DACH set.
`roledesk companies seed|add|check` import it, add one company from its
board with a live check first, and report dead boards.

roledesk/jobs/pull.py is now a pipeline task: ten boards per pull_batch job
on the http resource, upserting every role the ats plugin yields. Its final
job's settle closes every board this run answered in full and no longer
lists a role from, reading the plugin's own self.state stamps rather than
each batch job's return value, so one bad board in a batch does not cost the
other nine their closure. `roledesk job pull` runs it, and it registers
itself in pipeline.TASKS so <desk-jobs> can start it too.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Sonnet 5 committed 2026-09-23 11:27:30 +02:00
1 parent 21ed472a87
commit 0f59452b2a
5 files changed
+1684 -10

No files matched your search

+141 -4
View File
@@ -1,4 +1,4 @@
"""`roledesk serve | job pull | db init`."""
"""`roledesk serve | job pull | companies seed|add|check | db init`."""
from __future__ import annotations
@@ -6,9 +6,24 @@ import argparse
import json
import logging
import sys
from pathlib import Path
from typing import TYPE_CHECKING
from roledesk import config as config_mod
if TYPE_CHECKING:
from roledesk.storage import Store
#: The one plugin this app ships lives outside the `roledesk` package, in
#: `plugins/`, so it is discovered by `desk.sources.registry` the same way a
#: user's own drop-in plugin would be. `companies` and `job pull` both want
#: its plain functions (`is_known_target`, `careers_url`, `fetch_board`) as
#: well as the `Source` class, so the repo root goes on `sys.path` once, here,
#: rather than each caller guessing where `plugins/` is relative to `cwd`.
_ROOT = Path(__file__).resolve().parents[1]
if str(_ROOT) not in sys.path:
sys.path.insert(0, str(_ROOT))
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(prog="roledesk",
@@ -21,9 +36,19 @@ def build_parser() -> argparse.ArgumentParser:
serve.add_argument("--host", default="127.0.0.1")
serve.add_argument("--port", type=int, default=8790)
job = sub.add_parser("job", help="queue a job and drain the queue once")
job = sub.add_parser("job", help="queue a job or task and drain the queue once")
job.add_argument("kind", choices=["pull"])
companies = sub.add_parser("companies", help="the target company list")
companies_sub = companies.add_subparsers(dest="action", required=True)
seed = companies_sub.add_parser(
"seed", help="import seed/companies.json, subscribing to each board")
seed.add_argument("--path", default="seed/companies.json")
add = companies_sub.add_parser("add", help="add one company from its board")
add.add_argument("board", help="host:slug, e.g. greenhouse:acme")
add.add_argument("name")
companies_sub.add_parser("check", help="report boards that do not answer")
db = sub.add_parser("db", help="database housekeeping")
db.add_argument("action", choices=["init"])
return parser
@@ -40,6 +65,8 @@ def main(argv: list[str] | None = None) -> int:
return serve(args.host, args.port, args.config)
if args.command == "job":
return run_job(args)
if args.command == "companies":
return run_companies(args)
return init_db(args)
except FileNotFoundError as exc:
print(exc, file=sys.stderr)
@@ -47,8 +74,11 @@ def main(argv: list[str] | None = None) -> int:
def run_job(args: argparse.Namespace) -> int:
"""Queue `kind`, then run everything queued until the queue is idle."""
from pipeline import JobQueue, Jobs
"""Queue `kind` — a job, or a task from `pipeline.TASKS` such as `pull` —
then run everything queued until the queue is idle, and print how it
ended. A task's own result is its final job's, which is the one whose
`task_id` equals the run this started."""
from pipeline import FINAL, JobQueue, Jobs, TASKS
from roledesk import jobs
from roledesk.storage import Store
@@ -57,6 +87,15 @@ def run_job(args: argparse.Namespace) -> int:
queue = JobQueue(Jobs(config.database), config, store,
workers=config.workers.model_dump(),
bootstrap=jobs.BOOTSTRAP, bootstrap_arg=config.path)
if args.kind in TASKS:
task_id = queue.start_task(TASKS[args.kind]())
queue.drain()
final = next((j for j in queue.jobs.of_task(task_id) if j.kind == FINAL), None)
if final is not None:
print(json.dumps({"id": final.id, "state": final.state,
"result": final.result, "error": final.error}))
return 0 if final.state == "done" else 1
return 0
queued = queue.enqueue(args.kind)
queue.drain()
if queued is not None:
@@ -68,6 +107,104 @@ def run_job(args: argparse.Namespace) -> int:
return 0
def run_companies(args: argparse.Namespace) -> int:
from roledesk.storage import Store
config = config_mod.load(args.config)
store = Store(config.database)
if args.action == "seed":
print(json.dumps(seed_companies(store, args.path)))
return 0
if args.action == "add":
return add_company(store, args.board, args.name)
return check_companies(store)
def seed_companies(store: Store, path: str) -> dict[str, int]:
"""Import `seed/companies.json`: one `companies` row and one `ats`
subscription per entry, skipping a key that already exists. Not
idempotent on edits to an existing row — re-running it only ever adds
what is missing."""
from plugins.ats import is_known_target
data = json.loads(Path(path).read_text(encoding="utf-8"))
added = skipped = 0
for row in data.get("companies", []):
key, name, board = row["key"], row["name"], row["board"]
if store.company(key) is not None:
skipped += 1
continue
if not is_known_target(board):
print(f"companies seed: {key}: {board!r} is not a known board; skipped",
file=sys.stderr)
skipped += 1
continue
store.add_company(name, key=key, affinity=int(row.get("affinity", 0)),
careers_url=row.get("careers_url", ""))
host, _, _slug = board.partition(":")
try:
store.add_subscription("ats", board, label=name, company=name, board=host)
except ValueError as exc:
# The company row landed; the subscription is the only thing
# missing, and worth saying rather than losing silently.
print(f"companies seed: {key}: {exc}", file=sys.stderr)
added += 1
return {"added": added, "skipped": skipped, "total": len(data.get("companies", []))}
def add_company(store: Store, board: str, name: str) -> int:
"""`roledesk companies add greenhouse:acme "Acme"`: fetch the board first
and refuse if it does not answer, is not a known host, or the company
already exists. Port of seedgen's `-add`."""
from plugins.ats import careers_url as careers_url_of, fetch_board, is_known_target
from roledesk.models import company_key
if not is_known_target(board):
print(f"companies add: {board!r} is not host:slug for a known ATS host",
file=sys.stderr)
return 1
key = company_key(name)
if store.company(key) is not None:
print(f"companies add: {key!r} already exists", file=sys.stderr)
return 1
try:
rows = fetch_board(board)
except Exception as exc:
print(f"companies add: {board} did not answer: {exc}", file=sys.stderr)
return 1
if not rows:
print(f"companies add: {board} answered but lists no roles; nothing added",
file=sys.stderr)
return 1
host, _, _slug = board.partition(":")
store.add_company(name, key=key, careers_url=careers_url_of(board))
try:
store.add_subscription("ats", board, label=name, company=name, board=host)
except ValueError as exc:
print(f"companies add: {exc}", file=sys.stderr)
return 1
print(json.dumps({"key": key, "board": board, "roles": len(rows)}))
return 0
def check_companies(store: Store) -> int:
"""`roledesk companies check`: every `ats` board, fetched once. Prints a
line per board that did not answer and exits 1 if any did not."""
from plugins.ats import fetch_board
subs = store.subscriptions(source="ats")
dead = []
for sub in subs:
try:
fetch_board(sub.target)
except Exception as exc:
dead.append((sub.target, sub.label, str(exc)))
for target, label, reason in dead:
print(f"{target} ({label}): {reason}")
print(f"checked {len(subs)} boards, {len(dead)} dead")
return 1 if dead else 0
def init_db(args: argparse.Namespace) -> int:
"""Create the file, or bring it to the current schema."""
from roledesk.storage import LATEST, Store
+101 -6
View File
@@ -1,18 +1,113 @@
"""`pull`: read every configured plugin. A no-op until the first plugin lands."""
"""`pull`: fetch every active `ats` board and store the roles it lists.
A task, not a single job: ten boards per `pull_batch` job on the `http`
resource pool, because a board is one HTTP round trip and that pool is sized
for exactly this. The final job's `settle` then closes every board a
complete fetch of it did not list again — see `close_boards`.
Closure is read from `self.state`, which the `ats` plugin itself stamps per
board on a successful parse (`ats:ok:<target>`, `ats:ids:<target>`), rather
than from what each `pull_batch` job happens to return. A batch job that
raises partway through still leaves the boards it had already finished
stamped, so one bad board in a batch of ten does not cost the other nine
their closure.
"""
from __future__ import annotations
import json
import logging
from datetime import UTC, datetime
from typing import Any
from pipeline import NULL, RunContext, job
from pipeline import NULL, RunContext, TASKS, Tally, Task, job
from roledesk.config import Config
from roledesk.storage import Store
log = logging.getLogger(__name__)
#: As many boards in the air per job as there are `http` workers by default
#: (see `roles.yaml`), so one pull keeps the pool busy without one job
#: holding all 190-odd boards' worth of retries against a single attempt.
BATCH_SIZE = 10
@job("pull", "Reading every board", resource="http", solo=True)
def run(config: Config, store: Store, *, context: RunContext = NULL) -> dict[str, Any]:
log.info("pull: no plugin is installed yet")
return {"sources": 0, "seen": 0, "new": 0}
SETTLE = "roledesk.jobs.pull:close_boards"
def _now() -> str:
return datetime.now(UTC).isoformat(timespec="seconds")
def _boards(config: Config, store: Store) -> list[dict[str, Any]]:
"""Every fetchable `ats` subscription, ten to a batch. Stamps the run
marker `close_boards` will look for, before any batch runs — finders run
synchronously, in whoever starts the task, ahead of every job it queues."""
subs = store.subscriptions(source="ats", fetchable=True)
if not subs:
return []
store.state["ats:run"] = _now()
return [{"targets": [s.target for s in subs[i:i + BATCH_SIZE]]}
for i in range(0, len(subs), BATCH_SIZE)]
@job("pull_batch", "Reading boards", resource="http")
def run_batch(config: Config, store: Store, targets: list[str], *,
context: RunContext = NULL) -> dict[str, Any]:
"""Fetch this batch's boards and `upsert_role` every item they list.
Builds the `ats` plugin scoped to just these targets — `Source.targets()`
reads `options[subscription_field]`, so handing it only this batch's
entries is what keeps one job from re-fetching every other job's boards.
"""
from plugins.ats import ATSSource
subs = {s.target: s for s in store.subscriptions(source="ats") if s.target in targets}
options = dict(config.sources.get("ats") or {})
options["boards"] = [{"url": target, "label": subs[target].company or subs[target].label}
for target in targets if target in subs]
source = ATSSource(options)
source.state = store.state
stored = 0
for item in source.fetch():
item.source = "ats"
target = f"{item.raw.get('board', '')}:{item.raw.get('slug', '')}"
store.upsert_role(item, subs.get(target))
stored += 1
return {"roles": stored}
def close_boards(config: Config, store: Store, tally: Tally, *,
context: RunContext = NULL) -> dict[str, Any]:
"""The run's settle: close every board this run answered in full and
stopped listing a role from, and tally what happened.
Reads `self.state` rather than the batch jobs' own results, because a
`pull_batch` job that raised partway through still leaves the boards it
had already parsed stamped — see the module docstring. A board whose
`ats:ok:<target>` was not written during *this* run — it failed, or its
whole batch job never got to it — is never closed.
"""
run_marker = store.state.get("ats:run", "")
answered = failed = closed = 0
for sub in store.subscriptions(source="ats", fetchable=True):
stamp = store.state.get(f"ats:ok:{sub.target}", "")
if stamp and stamp == run_marker:
answered += 1
seen_ids = json.loads(store.state.get(f"ats:ids:{sub.target}", "[]"))
closed += store.mark_closed("ats", seen_ids, sub.id)
else:
failed += 1
roles = sum(int(result.get("roles", 0)) for result in tally.results)
return {"boards": answered, "failed": failed, "roles": roles, "closed": closed}
def _pull_task() -> Task:
return Task(kind="pull_batch", finder=_boards, name="pull",
label="Reading every board", settle=SETTLE)
#: Offered as `roledesk job pull` and to `<desk-jobs>`'s Pull button, the same
#: way newsdesk's cascade stages register themselves in `TASKS`.
TASKS["pull"] = _pull_task