Files
jobboard/roledesk/jobs/pull.py
T
WolfandClaude Sonnet 5 0f59452b2a Seed 186 companies, add roledesk companies, and pull and close every board
seed/companies.json: 179 companies from jobhunt-hq's seed.json (MIT),
reduced to key/name/board/careers_url, plus a hand-added DACH set.
`roledesk companies seed|add|check` import it, add one company from its
board with a live check first, and report dead boards.

roledesk/jobs/pull.py is now a pipeline task: ten boards per pull_batch job
on the http resource, upserting every role the ats plugin yields. Its final
job's settle closes every board this run answered in full and no longer
lists a role from, reading the plugin's own self.state stamps rather than
each batch job's return value, so one bad board in a batch does not cost the
other nine their closure. `roledesk job pull` runs it, and it registers
itself in pipeline.TASKS so <desk-jobs> can start it too.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
2026-09-23 11:27:30 +02:00

114 lines
4.6 KiB
Python

"""`pull`: fetch every active `ats` board and store the roles it lists.
A task, not a single job: ten boards per `pull_batch` job on the `http`
resource pool, because a board is one HTTP round trip and that pool is sized
for exactly this. The final job's `settle` then closes every board a
complete fetch of it did not list again — see `close_boards`.
Closure is read from `self.state`, which the `ats` plugin itself stamps per
board on a successful parse (`ats:ok:<target>`, `ats:ids:<target>`), rather
than from what each `pull_batch` job happens to return. A batch job that
raises partway through still leaves the boards it had already finished
stamped, so one bad board in a batch of ten does not cost the other nine
their closure.
"""
from __future__ import annotations
import json
import logging
from datetime import UTC, datetime
from typing import Any
from pipeline import NULL, RunContext, TASKS, Tally, Task, job
from roledesk.config import Config
from roledesk.storage import Store
log = logging.getLogger(__name__)
#: As many boards in the air per job as there are `http` workers by default
#: (see `roles.yaml`), so one pull keeps the pool busy without one job
#: holding all 190-odd boards' worth of retries against a single attempt.
BATCH_SIZE = 10
SETTLE = "roledesk.jobs.pull:close_boards"
def _now() -> str:
return datetime.now(UTC).isoformat(timespec="seconds")
def _boards(config: Config, store: Store) -> list[dict[str, Any]]:
"""Every fetchable `ats` subscription, ten to a batch. Stamps the run
marker `close_boards` will look for, before any batch runs — finders run
synchronously, in whoever starts the task, ahead of every job it queues."""
subs = store.subscriptions(source="ats", fetchable=True)
if not subs:
return []
store.state["ats:run"] = _now()
return [{"targets": [s.target for s in subs[i:i + BATCH_SIZE]]}
for i in range(0, len(subs), BATCH_SIZE)]
@job("pull_batch", "Reading boards", resource="http")
def run_batch(config: Config, store: Store, targets: list[str], *,
context: RunContext = NULL) -> dict[str, Any]:
"""Fetch this batch's boards and `upsert_role` every item they list.
Builds the `ats` plugin scoped to just these targets — `Source.targets()`
reads `options[subscription_field]`, so handing it only this batch's
entries is what keeps one job from re-fetching every other job's boards.
"""
from plugins.ats import ATSSource
subs = {s.target: s for s in store.subscriptions(source="ats") if s.target in targets}
options = dict(config.sources.get("ats") or {})
options["boards"] = [{"url": target, "label": subs[target].company or subs[target].label}
for target in targets if target in subs]
source = ATSSource(options)
source.state = store.state
stored = 0
for item in source.fetch():
item.source = "ats"
target = f"{item.raw.get('board', '')}:{item.raw.get('slug', '')}"
store.upsert_role(item, subs.get(target))
stored += 1
return {"roles": stored}
def close_boards(config: Config, store: Store, tally: Tally, *,
context: RunContext = NULL) -> dict[str, Any]:
"""The run's settle: close every board this run answered in full and
stopped listing a role from, and tally what happened.
Reads `self.state` rather than the batch jobs' own results, because a
`pull_batch` job that raised partway through still leaves the boards it
had already parsed stamped — see the module docstring. A board whose
`ats:ok:<target>` was not written during *this* run — it failed, or its
whole batch job never got to it — is never closed.
"""
run_marker = store.state.get("ats:run", "")
answered = failed = closed = 0
for sub in store.subscriptions(source="ats", fetchable=True):
stamp = store.state.get(f"ats:ok:{sub.target}", "")
if stamp and stamp == run_marker:
answered += 1
seen_ids = json.loads(store.state.get(f"ats:ids:{sub.target}", "[]"))
closed += store.mark_closed("ats", seen_ids, sub.id)
else:
failed += 1
roles = sum(int(result.get("roles", 0)) for result in tally.results)
return {"boards": answered, "failed": failed, "roles": roles, "closed": closed}
def _pull_task() -> Task:
return Task(kind="pull_batch", finder=_boards, name="pull",
label="Reading every board", settle=SETTLE)
#: Offered as `roledesk job pull` and to `<desk-jobs>`'s Pull button, the same
#: way newsdesk's cascade stages register themselves in `TASKS`.
TASKS["pull"] = _pull_task