Seed 186 companies, add roledesk companies, and pull and close every board
seed/companies.json: 179 companies from jobhunt-hq's seed.json (MIT), reduced to key/name/board/careers_url, plus a hand-added DACH set. `roledesk companies seed|add|check` import it, add one company from its board with a live check first, and report dead boards. roledesk/jobs/pull.py is now a pipeline task: ten boards per pull_batch job on the http resource, upserting every role the ats plugin yields. Its final job's settle closes every board this run answered in full and no longer lists a role from, reading the plugin's own self.state stamps rather than each batch job's return value, so one bad board in a batch does not cost the other nine their closure. `roledesk job pull` runs it, and it registers itself in pipeline.TASKS so <desk-jobs> can start it too. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
21ed472a87
commit
0f59452b2a
5 files changed
+1684
-10
No files matched your search
+141
-4
@@ -1,4 +1,4 @@
|
||||
"""`roledesk serve | job pull | db init`."""
|
||||
"""`roledesk serve | job pull | companies seed|add|check | db init`."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -6,9 +6,24 @@ import argparse
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from roledesk import config as config_mod
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from roledesk.storage import Store
|
||||
|
||||
#: The one plugin this app ships lives outside the `roledesk` package, in
|
||||
#: `plugins/`, so it is discovered by `desk.sources.registry` the same way a
|
||||
#: user's own drop-in plugin would be. `companies` and `job pull` both want
|
||||
#: its plain functions (`is_known_target`, `careers_url`, `fetch_board`) as
|
||||
#: well as the `Source` class, so the repo root goes on `sys.path` once, here,
|
||||
#: rather than each caller guessing where `plugins/` is relative to `cwd`.
|
||||
_ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(_ROOT))
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(prog="roledesk",
|
||||
@@ -21,9 +36,19 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
serve.add_argument("--host", default="127.0.0.1")
|
||||
serve.add_argument("--port", type=int, default=8790)
|
||||
|
||||
job = sub.add_parser("job", help="queue a job and drain the queue once")
|
||||
job = sub.add_parser("job", help="queue a job or task and drain the queue once")
|
||||
job.add_argument("kind", choices=["pull"])
|
||||
|
||||
companies = sub.add_parser("companies", help="the target company list")
|
||||
companies_sub = companies.add_subparsers(dest="action", required=True)
|
||||
seed = companies_sub.add_parser(
|
||||
"seed", help="import seed/companies.json, subscribing to each board")
|
||||
seed.add_argument("--path", default="seed/companies.json")
|
||||
add = companies_sub.add_parser("add", help="add one company from its board")
|
||||
add.add_argument("board", help="host:slug, e.g. greenhouse:acme")
|
||||
add.add_argument("name")
|
||||
companies_sub.add_parser("check", help="report boards that do not answer")
|
||||
|
||||
db = sub.add_parser("db", help="database housekeeping")
|
||||
db.add_argument("action", choices=["init"])
|
||||
return parser
|
||||
@@ -40,6 +65,8 @@ def main(argv: list[str] | None = None) -> int:
|
||||
return serve(args.host, args.port, args.config)
|
||||
if args.command == "job":
|
||||
return run_job(args)
|
||||
if args.command == "companies":
|
||||
return run_companies(args)
|
||||
return init_db(args)
|
||||
except FileNotFoundError as exc:
|
||||
print(exc, file=sys.stderr)
|
||||
@@ -47,8 +74,11 @@ def main(argv: list[str] | None = None) -> int:
|
||||
|
||||
|
||||
def run_job(args: argparse.Namespace) -> int:
|
||||
"""Queue `kind`, then run everything queued until the queue is idle."""
|
||||
from pipeline import JobQueue, Jobs
|
||||
"""Queue `kind` — a job, or a task from `pipeline.TASKS` such as `pull` —
|
||||
then run everything queued until the queue is idle, and print how it
|
||||
ended. A task's own result is its final job's, which is the one whose
|
||||
`task_id` equals the run this started."""
|
||||
from pipeline import FINAL, JobQueue, Jobs, TASKS
|
||||
from roledesk import jobs
|
||||
from roledesk.storage import Store
|
||||
|
||||
@@ -57,6 +87,15 @@ def run_job(args: argparse.Namespace) -> int:
|
||||
queue = JobQueue(Jobs(config.database), config, store,
|
||||
workers=config.workers.model_dump(),
|
||||
bootstrap=jobs.BOOTSTRAP, bootstrap_arg=config.path)
|
||||
if args.kind in TASKS:
|
||||
task_id = queue.start_task(TASKS[args.kind]())
|
||||
queue.drain()
|
||||
final = next((j for j in queue.jobs.of_task(task_id) if j.kind == FINAL), None)
|
||||
if final is not None:
|
||||
print(json.dumps({"id": final.id, "state": final.state,
|
||||
"result": final.result, "error": final.error}))
|
||||
return 0 if final.state == "done" else 1
|
||||
return 0
|
||||
queued = queue.enqueue(args.kind)
|
||||
queue.drain()
|
||||
if queued is not None:
|
||||
@@ -68,6 +107,104 @@ def run_job(args: argparse.Namespace) -> int:
|
||||
return 0
|
||||
|
||||
|
||||
def run_companies(args: argparse.Namespace) -> int:
|
||||
from roledesk.storage import Store
|
||||
|
||||
config = config_mod.load(args.config)
|
||||
store = Store(config.database)
|
||||
if args.action == "seed":
|
||||
print(json.dumps(seed_companies(store, args.path)))
|
||||
return 0
|
||||
if args.action == "add":
|
||||
return add_company(store, args.board, args.name)
|
||||
return check_companies(store)
|
||||
|
||||
|
||||
def seed_companies(store: Store, path: str) -> dict[str, int]:
|
||||
"""Import `seed/companies.json`: one `companies` row and one `ats`
|
||||
subscription per entry, skipping a key that already exists. Not
|
||||
idempotent on edits to an existing row — re-running it only ever adds
|
||||
what is missing."""
|
||||
from plugins.ats import is_known_target
|
||||
|
||||
data = json.loads(Path(path).read_text(encoding="utf-8"))
|
||||
added = skipped = 0
|
||||
for row in data.get("companies", []):
|
||||
key, name, board = row["key"], row["name"], row["board"]
|
||||
if store.company(key) is not None:
|
||||
skipped += 1
|
||||
continue
|
||||
if not is_known_target(board):
|
||||
print(f"companies seed: {key}: {board!r} is not a known board; skipped",
|
||||
file=sys.stderr)
|
||||
skipped += 1
|
||||
continue
|
||||
store.add_company(name, key=key, affinity=int(row.get("affinity", 0)),
|
||||
careers_url=row.get("careers_url", ""))
|
||||
host, _, _slug = board.partition(":")
|
||||
try:
|
||||
store.add_subscription("ats", board, label=name, company=name, board=host)
|
||||
except ValueError as exc:
|
||||
# The company row landed; the subscription is the only thing
|
||||
# missing, and worth saying rather than losing silently.
|
||||
print(f"companies seed: {key}: {exc}", file=sys.stderr)
|
||||
added += 1
|
||||
return {"added": added, "skipped": skipped, "total": len(data.get("companies", []))}
|
||||
|
||||
|
||||
def add_company(store: Store, board: str, name: str) -> int:
|
||||
"""`roledesk companies add greenhouse:acme "Acme"`: fetch the board first
|
||||
and refuse if it does not answer, is not a known host, or the company
|
||||
already exists. Port of seedgen's `-add`."""
|
||||
from plugins.ats import careers_url as careers_url_of, fetch_board, is_known_target
|
||||
from roledesk.models import company_key
|
||||
|
||||
if not is_known_target(board):
|
||||
print(f"companies add: {board!r} is not host:slug for a known ATS host",
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
key = company_key(name)
|
||||
if store.company(key) is not None:
|
||||
print(f"companies add: {key!r} already exists", file=sys.stderr)
|
||||
return 1
|
||||
try:
|
||||
rows = fetch_board(board)
|
||||
except Exception as exc:
|
||||
print(f"companies add: {board} did not answer: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
if not rows:
|
||||
print(f"companies add: {board} answered but lists no roles; nothing added",
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
host, _, _slug = board.partition(":")
|
||||
store.add_company(name, key=key, careers_url=careers_url_of(board))
|
||||
try:
|
||||
store.add_subscription("ats", board, label=name, company=name, board=host)
|
||||
except ValueError as exc:
|
||||
print(f"companies add: {exc}", file=sys.stderr)
|
||||
return 1
|
||||
print(json.dumps({"key": key, "board": board, "roles": len(rows)}))
|
||||
return 0
|
||||
|
||||
|
||||
def check_companies(store: Store) -> int:
|
||||
"""`roledesk companies check`: every `ats` board, fetched once. Prints a
|
||||
line per board that did not answer and exits 1 if any did not."""
|
||||
from plugins.ats import fetch_board
|
||||
|
||||
subs = store.subscriptions(source="ats")
|
||||
dead = []
|
||||
for sub in subs:
|
||||
try:
|
||||
fetch_board(sub.target)
|
||||
except Exception as exc:
|
||||
dead.append((sub.target, sub.label, str(exc)))
|
||||
for target, label, reason in dead:
|
||||
print(f"{target} ({label}): {reason}")
|
||||
print(f"checked {len(subs)} boards, {len(dead)} dead")
|
||||
return 1 if dead else 0
|
||||
|
||||
|
||||
def init_db(args: argparse.Namespace) -> int:
|
||||
"""Create the file, or bring it to the current schema."""
|
||||
from roledesk.storage import LATEST, Store
|
||||
|
||||
+101
-6
@@ -1,18 +1,113 @@
|
||||
"""`pull`: read every configured plugin. A no-op until the first plugin lands."""
|
||||
"""`pull`: fetch every active `ats` board and store the roles it lists.
|
||||
|
||||
A task, not a single job: ten boards per `pull_batch` job on the `http`
|
||||
resource pool, because a board is one HTTP round trip and that pool is sized
|
||||
for exactly this. The final job's `settle` then closes every board a
|
||||
complete fetch of it did not list again — see `close_boards`.
|
||||
|
||||
Closure is read from `self.state`, which the `ats` plugin itself stamps per
|
||||
board on a successful parse (`ats:ok:<target>`, `ats:ids:<target>`), rather
|
||||
than from what each `pull_batch` job happens to return. A batch job that
|
||||
raises partway through still leaves the boards it had already finished
|
||||
stamped, so one bad board in a batch of ten does not cost the other nine
|
||||
their closure.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from pipeline import NULL, RunContext, job
|
||||
from pipeline import NULL, RunContext, TASKS, Tally, Task, job
|
||||
|
||||
from roledesk.config import Config
|
||||
from roledesk.storage import Store
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
#: As many boards in the air per job as there are `http` workers by default
|
||||
#: (see `roles.yaml`), so one pull keeps the pool busy without one job
|
||||
#: holding all 190-odd boards' worth of retries against a single attempt.
|
||||
BATCH_SIZE = 10
|
||||
|
||||
@job("pull", "Reading every board", resource="http", solo=True)
|
||||
def run(config: Config, store: Store, *, context: RunContext = NULL) -> dict[str, Any]:
|
||||
log.info("pull: no plugin is installed yet")
|
||||
return {"sources": 0, "seen": 0, "new": 0}
|
||||
SETTLE = "roledesk.jobs.pull:close_boards"
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(UTC).isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def _boards(config: Config, store: Store) -> list[dict[str, Any]]:
|
||||
"""Every fetchable `ats` subscription, ten to a batch. Stamps the run
|
||||
marker `close_boards` will look for, before any batch runs — finders run
|
||||
synchronously, in whoever starts the task, ahead of every job it queues."""
|
||||
subs = store.subscriptions(source="ats", fetchable=True)
|
||||
if not subs:
|
||||
return []
|
||||
store.state["ats:run"] = _now()
|
||||
return [{"targets": [s.target for s in subs[i:i + BATCH_SIZE]]}
|
||||
for i in range(0, len(subs), BATCH_SIZE)]
|
||||
|
||||
|
||||
@job("pull_batch", "Reading boards", resource="http")
|
||||
def run_batch(config: Config, store: Store, targets: list[str], *,
|
||||
context: RunContext = NULL) -> dict[str, Any]:
|
||||
"""Fetch this batch's boards and `upsert_role` every item they list.
|
||||
|
||||
Builds the `ats` plugin scoped to just these targets — `Source.targets()`
|
||||
reads `options[subscription_field]`, so handing it only this batch's
|
||||
entries is what keeps one job from re-fetching every other job's boards.
|
||||
"""
|
||||
from plugins.ats import ATSSource
|
||||
|
||||
subs = {s.target: s for s in store.subscriptions(source="ats") if s.target in targets}
|
||||
options = dict(config.sources.get("ats") or {})
|
||||
options["boards"] = [{"url": target, "label": subs[target].company or subs[target].label}
|
||||
for target in targets if target in subs]
|
||||
source = ATSSource(options)
|
||||
source.state = store.state
|
||||
|
||||
stored = 0
|
||||
for item in source.fetch():
|
||||
item.source = "ats"
|
||||
target = f"{item.raw.get('board', '')}:{item.raw.get('slug', '')}"
|
||||
store.upsert_role(item, subs.get(target))
|
||||
stored += 1
|
||||
return {"roles": stored}
|
||||
|
||||
|
||||
def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||
context: RunContext = NULL) -> dict[str, Any]:
|
||||
"""The run's settle: close every board this run answered in full and
|
||||
stopped listing a role from, and tally what happened.
|
||||
|
||||
Reads `self.state` rather than the batch jobs' own results, because a
|
||||
`pull_batch` job that raised partway through still leaves the boards it
|
||||
had already parsed stamped — see the module docstring. A board whose
|
||||
`ats:ok:<target>` was not written during *this* run — it failed, or its
|
||||
whole batch job never got to it — is never closed.
|
||||
"""
|
||||
run_marker = store.state.get("ats:run", "")
|
||||
answered = failed = closed = 0
|
||||
for sub in store.subscriptions(source="ats", fetchable=True):
|
||||
stamp = store.state.get(f"ats:ok:{sub.target}", "")
|
||||
if stamp and stamp == run_marker:
|
||||
answered += 1
|
||||
seen_ids = json.loads(store.state.get(f"ats:ids:{sub.target}", "[]"))
|
||||
closed += store.mark_closed("ats", seen_ids, sub.id)
|
||||
else:
|
||||
failed += 1
|
||||
roles = sum(int(result.get("roles", 0)) for result in tally.results)
|
||||
return {"boards": answered, "failed": failed, "roles": roles, "closed": closed}
|
||||
|
||||
|
||||
def _pull_task() -> Task:
|
||||
return Task(kind="pull_batch", finder=_boards, name="pull",
|
||||
label="Reading every board", settle=SETTLE)
|
||||
|
||||
|
||||
#: Offered as `roledesk job pull` and to `<desk-jobs>`'s Pull button, the same
|
||||
#: way newsdesk's cascade stages register themselves in `TASKS`.
|
||||
TASKS["pull"] = _pull_task
|
||||
Reference in new issue
Block a user