embed stores one vector per open role; score rates every open role for every enabled agent from require and prefer gates, fit against the agent's intent, recency and company affinity, keeping gated rows with a reason. Schema v3 adds vectors and the cached intent vector. The page gets an agent order, the score on each row, and an Agents tab. Two starter agents in roles.yaml are imported once. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
111 lines
4.5 KiB
Python
111 lines
4.5 KiB
Python
"""`pull`: fetch every active `ats` board and store the roles it lists.
|
|
|
|
A task, not a single job: ten boards per `pull_batch` job on the `http`
|
|
resource pool, because a board is one HTTP round trip and that pool is sized
|
|
for exactly this. The final job's `settle` then closes every board a
|
|
complete fetch of it did not list again — see `close_boards`.
|
|
|
|
Closure is read from `self.state`, which the `ats` plugin itself stamps per
|
|
board on a successful parse (`ats:ok:<target>`, `ats:ids:<target>`), rather
|
|
than from what each `pull_batch` job happens to return. A batch job that
|
|
raises partway through still leaves the boards it had already finished
|
|
stamped, so one bad board in a batch of ten does not cost the other nine
|
|
their closure.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
from datetime import UTC, datetime
|
|
from typing import Any
|
|
|
|
from pipeline import NULL, RunContext, Tally, Task, job
|
|
|
|
from roledesk.config import Config
|
|
from roledesk.storage import Store
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
#: As many boards in the air per job as there are `http` workers by default
|
|
#: (see `roles.yaml`), so one pull keeps the pool busy without one job
|
|
#: holding all 190-odd boards' worth of retries against a single attempt.
|
|
BATCH_SIZE = 10
|
|
|
|
SETTLE = "roledesk.jobs.pull:close_boards"
|
|
|
|
|
|
def _now() -> str:
|
|
return datetime.now(UTC).isoformat(timespec="seconds")
|
|
|
|
|
|
def _boards(config: Config, store: Store) -> list[dict[str, Any]]:
|
|
"""Every fetchable `ats` subscription, ten to a batch. Stamps the run
|
|
marker `close_boards` will look for, before any batch runs — finders run
|
|
synchronously, in whoever starts the task, ahead of every job it queues."""
|
|
subs = store.subscriptions(source="ats", fetchable=True)
|
|
if not subs:
|
|
return []
|
|
store.state["ats:run"] = _now()
|
|
return [{"targets": [s.target for s in subs[i:i + BATCH_SIZE]]}
|
|
for i in range(0, len(subs), BATCH_SIZE)]
|
|
|
|
|
|
@job("pull_batch", "Reading boards", resource="http")
|
|
def run_batch(config: Config, store: Store, targets: list[str], *,
|
|
context: RunContext = NULL) -> dict[str, Any]:
|
|
"""Fetch this batch's boards and `upsert_role` every item they list.
|
|
|
|
Builds the `ats` plugin scoped to just these targets — `Source.targets()`
|
|
reads `options[subscription_field]`, so handing it only this batch's
|
|
entries is what keeps one job from re-fetching every other job's boards.
|
|
"""
|
|
from plugins.ats import ATSSource
|
|
|
|
subs = {s.target: s for s in store.subscriptions(source="ats") if s.target in targets}
|
|
options = dict(config.sources.get("ats") or {})
|
|
options["boards"] = [{"url": target, "label": subs[target].company or subs[target].label}
|
|
for target in targets if target in subs]
|
|
source = ATSSource(options)
|
|
source.state = store.state
|
|
|
|
stored = 0
|
|
for item in source.fetch():
|
|
item.source = "ats"
|
|
target = f"{item.raw.get('board', '')}:{item.raw.get('slug', '')}"
|
|
store.upsert_role(item, subs.get(target))
|
|
stored += 1
|
|
return {"roles": stored}
|
|
|
|
|
|
def close_boards(config: Config, store: Store, tally: Tally, *,
|
|
context: RunContext = NULL) -> dict[str, Any]:
|
|
"""The run's settle: close every board this run answered in full and
|
|
stopped listing a role from, and tally what happened.
|
|
|
|
Reads `self.state` rather than the batch jobs' own results, because a
|
|
`pull_batch` job that raised partway through still leaves the boards it
|
|
had already parsed stamped — see the module docstring. A board whose
|
|
`ats:ok:<target>` was not written during *this* run — it failed, or its
|
|
whole batch job never got to it — is never closed.
|
|
"""
|
|
run_marker = store.state.get("ats:run", "")
|
|
answered = failed = closed = 0
|
|
for sub in store.subscriptions(source="ats", fetchable=True):
|
|
stamp = store.state.get(f"ats:ok:{sub.target}", "")
|
|
if stamp and stamp == run_marker:
|
|
answered += 1
|
|
seen_ids = json.loads(store.state.get(f"ats:ids:{sub.target}", "[]"))
|
|
closed += store.mark_closed("ats", seen_ids, sub.id)
|
|
else:
|
|
failed += 1
|
|
roles = sum(int(result.get("roles", 0)) for result in tally.results)
|
|
return {"boards": answered, "failed": failed, "roles": roles, "closed": closed}
|
|
|
|
|
|
def pull_task() -> Task:
|
|
"""The pull on its own. `roledesk.jobs.cascade` makes it the first stage
|
|
of the chain that `TASKS["pull"]` starts."""
|
|
return Task(kind="pull_batch", finder=_boards, name="pull",
|
|
label="Reading every board", settle=SETTLE)
|