diff --git a/docs/decisions.md b/docs/decisions.md index 3c2b9e4..0de6807 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -30,6 +30,14 @@ What was decided for roledesk and the one reason for each. The longer reasoning **`subscriptions` names its labels `company` and `board`.** newsdesk's `outlet` and `section` are news words; roledesk has no rows to migrate, so it uses its own names from the first migration. +## Ingestion + +**Closure is decided from `self.state`, not from what a `pull_batch` job returns.** The `ats` plugin stamps `ats:ok:` with the run's own marker and `ats:ids:` with the ids it saw on every board it parses without raising. The `pull` task's settle then closes a board only when its stamp matches the marker the finder wrote before any batch ran. A batch job that raises part way through a ten-board chunk still leaves the boards it had already finished stamped, so one bad board does not cost its batch-mates their closure — and a board a batch never got to (the whole job crashed, or was never claimed) stays open rather than being closed on an empty id list. + +**SmartRecruiters' board list has no description.** Its `/postings` endpoint gives title, location and department but not the job text; that is only on the per-posting detail endpoint, and fetching one extra request per role would multiply a board's request count by however many roles it lists. `description` stays empty for this host; title, location, department and the derived `function` are still filled. + +**Workable's shape is confirmed, not a populated example.** Every slug tried against `apply.workable.com/api/v3/accounts/{slug}/jobs` during this build returned the documented `{"total": 0, "results": []}` before Cloudflare rate-limited further requests from this session, so the parser is written from Workable's long-published job object rather than from a fetched row. Worth a fixture-level recheck against a live board with open roles the next time this plugin is touched. + ## Ranking **Scoring is per search agent.** A role is scored as `(uid, agent) -> score`, because a role is only good for a given search. Each gate has a mode: `require` becomes a SQL `WHERE`, `prefer` becomes a weighted term in the score. Gated roles keep their row with a reason. `companies.affinity = 2` skips the gates and floors the score, `-1` mutes. diff --git a/roledesk/cli.py b/roledesk/cli.py index ff7d4b9..c15992c 100644 --- a/roledesk/cli.py +++ b/roledesk/cli.py @@ -1,4 +1,4 @@ -"""`roledesk serve | job pull | db init`.""" +"""`roledesk serve | job pull | companies seed|add|check | db init`.""" from __future__ import annotations @@ -6,9 +6,24 @@ import argparse import json import logging import sys +from pathlib import Path +from typing import TYPE_CHECKING from roledesk import config as config_mod +if TYPE_CHECKING: + from roledesk.storage import Store + +#: The one plugin this app ships lives outside the `roledesk` package, in +#: `plugins/`, so it is discovered by `desk.sources.registry` the same way a +#: user's own drop-in plugin would be. `companies` and `job pull` both want +#: its plain functions (`is_known_target`, `careers_url`, `fetch_board`) as +#: well as the `Source` class, so the repo root goes on `sys.path` once, here, +#: rather than each caller guessing where `plugins/` is relative to `cwd`. +_ROOT = Path(__file__).resolve().parents[1] +if str(_ROOT) not in sys.path: + sys.path.insert(0, str(_ROOT)) + def build_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(prog="roledesk", @@ -21,9 +36,19 @@ def build_parser() -> argparse.ArgumentParser: serve.add_argument("--host", default="127.0.0.1") serve.add_argument("--port", type=int, default=8790) - job = sub.add_parser("job", help="queue a job and drain the queue once") + job = sub.add_parser("job", help="queue a job or task and drain the queue once") job.add_argument("kind", choices=["pull"]) + companies = sub.add_parser("companies", help="the target company list") + companies_sub = companies.add_subparsers(dest="action", required=True) + seed = companies_sub.add_parser( + "seed", help="import seed/companies.json, subscribing to each board") + seed.add_argument("--path", default="seed/companies.json") + add = companies_sub.add_parser("add", help="add one company from its board") + add.add_argument("board", help="host:slug, e.g. greenhouse:acme") + add.add_argument("name") + companies_sub.add_parser("check", help="report boards that do not answer") + db = sub.add_parser("db", help="database housekeeping") db.add_argument("action", choices=["init"]) return parser @@ -40,6 +65,8 @@ def main(argv: list[str] | None = None) -> int: return serve(args.host, args.port, args.config) if args.command == "job": return run_job(args) + if args.command == "companies": + return run_companies(args) return init_db(args) except FileNotFoundError as exc: print(exc, file=sys.stderr) @@ -47,8 +74,11 @@ def main(argv: list[str] | None = None) -> int: def run_job(args: argparse.Namespace) -> int: - """Queue `kind`, then run everything queued until the queue is idle.""" - from pipeline import JobQueue, Jobs + """Queue `kind` — a job, or a task from `pipeline.TASKS` such as `pull` — + then run everything queued until the queue is idle, and print how it + ended. A task's own result is its final job's, which is the one whose + `task_id` equals the run this started.""" + from pipeline import FINAL, JobQueue, Jobs, TASKS from roledesk import jobs from roledesk.storage import Store @@ -57,6 +87,15 @@ def run_job(args: argparse.Namespace) -> int: queue = JobQueue(Jobs(config.database), config, store, workers=config.workers.model_dump(), bootstrap=jobs.BOOTSTRAP, bootstrap_arg=config.path) + if args.kind in TASKS: + task_id = queue.start_task(TASKS[args.kind]()) + queue.drain() + final = next((j for j in queue.jobs.of_task(task_id) if j.kind == FINAL), None) + if final is not None: + print(json.dumps({"id": final.id, "state": final.state, + "result": final.result, "error": final.error})) + return 0 if final.state == "done" else 1 + return 0 queued = queue.enqueue(args.kind) queue.drain() if queued is not None: @@ -68,6 +107,104 @@ def run_job(args: argparse.Namespace) -> int: return 0 +def run_companies(args: argparse.Namespace) -> int: + from roledesk.storage import Store + + config = config_mod.load(args.config) + store = Store(config.database) + if args.action == "seed": + print(json.dumps(seed_companies(store, args.path))) + return 0 + if args.action == "add": + return add_company(store, args.board, args.name) + return check_companies(store) + + +def seed_companies(store: Store, path: str) -> dict[str, int]: + """Import `seed/companies.json`: one `companies` row and one `ats` + subscription per entry, skipping a key that already exists. Not + idempotent on edits to an existing row — re-running it only ever adds + what is missing.""" + from plugins.ats import is_known_target + + data = json.loads(Path(path).read_text(encoding="utf-8")) + added = skipped = 0 + for row in data.get("companies", []): + key, name, board = row["key"], row["name"], row["board"] + if store.company(key) is not None: + skipped += 1 + continue + if not is_known_target(board): + print(f"companies seed: {key}: {board!r} is not a known board; skipped", + file=sys.stderr) + skipped += 1 + continue + store.add_company(name, key=key, affinity=int(row.get("affinity", 0)), + careers_url=row.get("careers_url", "")) + host, _, _slug = board.partition(":") + try: + store.add_subscription("ats", board, label=name, company=name, board=host) + except ValueError as exc: + # The company row landed; the subscription is the only thing + # missing, and worth saying rather than losing silently. + print(f"companies seed: {key}: {exc}", file=sys.stderr) + added += 1 + return {"added": added, "skipped": skipped, "total": len(data.get("companies", []))} + + +def add_company(store: Store, board: str, name: str) -> int: + """`roledesk companies add greenhouse:acme "Acme"`: fetch the board first + and refuse if it does not answer, is not a known host, or the company + already exists. Port of seedgen's `-add`.""" + from plugins.ats import careers_url as careers_url_of, fetch_board, is_known_target + from roledesk.models import company_key + + if not is_known_target(board): + print(f"companies add: {board!r} is not host:slug for a known ATS host", + file=sys.stderr) + return 1 + key = company_key(name) + if store.company(key) is not None: + print(f"companies add: {key!r} already exists", file=sys.stderr) + return 1 + try: + rows = fetch_board(board) + except Exception as exc: + print(f"companies add: {board} did not answer: {exc}", file=sys.stderr) + return 1 + if not rows: + print(f"companies add: {board} answered but lists no roles; nothing added", + file=sys.stderr) + return 1 + host, _, _slug = board.partition(":") + store.add_company(name, key=key, careers_url=careers_url_of(board)) + try: + store.add_subscription("ats", board, label=name, company=name, board=host) + except ValueError as exc: + print(f"companies add: {exc}", file=sys.stderr) + return 1 + print(json.dumps({"key": key, "board": board, "roles": len(rows)})) + return 0 + + +def check_companies(store: Store) -> int: + """`roledesk companies check`: every `ats` board, fetched once. Prints a + line per board that did not answer and exits 1 if any did not.""" + from plugins.ats import fetch_board + + subs = store.subscriptions(source="ats") + dead = [] + for sub in subs: + try: + fetch_board(sub.target) + except Exception as exc: + dead.append((sub.target, sub.label, str(exc))) + for target, label, reason in dead: + print(f"{target} ({label}): {reason}") + print(f"checked {len(subs)} boards, {len(dead)} dead") + return 1 if dead else 0 + + def init_db(args: argparse.Namespace) -> int: """Create the file, or bring it to the current schema.""" from roledesk.storage import LATEST, Store diff --git a/roledesk/jobs/pull.py b/roledesk/jobs/pull.py index b433b83..1a6c556 100644 --- a/roledesk/jobs/pull.py +++ b/roledesk/jobs/pull.py @@ -1,18 +1,113 @@ -"""`pull`: read every configured plugin. A no-op until the first plugin lands.""" +"""`pull`: fetch every active `ats` board and store the roles it lists. + +A task, not a single job: ten boards per `pull_batch` job on the `http` +resource pool, because a board is one HTTP round trip and that pool is sized +for exactly this. The final job's `settle` then closes every board a +complete fetch of it did not list again — see `close_boards`. + +Closure is read from `self.state`, which the `ats` plugin itself stamps per +board on a successful parse (`ats:ok:`, `ats:ids:`), rather +than from what each `pull_batch` job happens to return. A batch job that +raises partway through still leaves the boards it had already finished +stamped, so one bad board in a batch of ten does not cost the other nine +their closure. +""" from __future__ import annotations +import json import logging +from datetime import UTC, datetime from typing import Any -from pipeline import NULL, RunContext, job +from pipeline import NULL, RunContext, TASKS, Tally, Task, job + from roledesk.config import Config from roledesk.storage import Store log = logging.getLogger(__name__) +#: As many boards in the air per job as there are `http` workers by default +#: (see `roles.yaml`), so one pull keeps the pool busy without one job +#: holding all 190-odd boards' worth of retries against a single attempt. +BATCH_SIZE = 10 -@job("pull", "Reading every board", resource="http", solo=True) -def run(config: Config, store: Store, *, context: RunContext = NULL) -> dict[str, Any]: - log.info("pull: no plugin is installed yet") - return {"sources": 0, "seen": 0, "new": 0} +SETTLE = "roledesk.jobs.pull:close_boards" + + +def _now() -> str: + return datetime.now(UTC).isoformat(timespec="seconds") + + +def _boards(config: Config, store: Store) -> list[dict[str, Any]]: + """Every fetchable `ats` subscription, ten to a batch. Stamps the run + marker `close_boards` will look for, before any batch runs — finders run + synchronously, in whoever starts the task, ahead of every job it queues.""" + subs = store.subscriptions(source="ats", fetchable=True) + if not subs: + return [] + store.state["ats:run"] = _now() + return [{"targets": [s.target for s in subs[i:i + BATCH_SIZE]]} + for i in range(0, len(subs), BATCH_SIZE)] + + +@job("pull_batch", "Reading boards", resource="http") +def run_batch(config: Config, store: Store, targets: list[str], *, + context: RunContext = NULL) -> dict[str, Any]: + """Fetch this batch's boards and `upsert_role` every item they list. + + Builds the `ats` plugin scoped to just these targets — `Source.targets()` + reads `options[subscription_field]`, so handing it only this batch's + entries is what keeps one job from re-fetching every other job's boards. + """ + from plugins.ats import ATSSource + + subs = {s.target: s for s in store.subscriptions(source="ats") if s.target in targets} + options = dict(config.sources.get("ats") or {}) + options["boards"] = [{"url": target, "label": subs[target].company or subs[target].label} + for target in targets if target in subs] + source = ATSSource(options) + source.state = store.state + + stored = 0 + for item in source.fetch(): + item.source = "ats" + target = f"{item.raw.get('board', '')}:{item.raw.get('slug', '')}" + store.upsert_role(item, subs.get(target)) + stored += 1 + return {"roles": stored} + + +def close_boards(config: Config, store: Store, tally: Tally, *, + context: RunContext = NULL) -> dict[str, Any]: + """The run's settle: close every board this run answered in full and + stopped listing a role from, and tally what happened. + + Reads `self.state` rather than the batch jobs' own results, because a + `pull_batch` job that raised partway through still leaves the boards it + had already parsed stamped — see the module docstring. A board whose + `ats:ok:` was not written during *this* run — it failed, or its + whole batch job never got to it — is never closed. + """ + run_marker = store.state.get("ats:run", "") + answered = failed = closed = 0 + for sub in store.subscriptions(source="ats", fetchable=True): + stamp = store.state.get(f"ats:ok:{sub.target}", "") + if stamp and stamp == run_marker: + answered += 1 + seen_ids = json.loads(store.state.get(f"ats:ids:{sub.target}", "[]")) + closed += store.mark_closed("ats", seen_ids, sub.id) + else: + failed += 1 + roles = sum(int(result.get("roles", 0)) for result in tally.results) + return {"boards": answered, "failed": failed, "roles": roles, "closed": closed} + + +def _pull_task() -> Task: + return Task(kind="pull_batch", finder=_boards, name="pull", + label="Reading every board", settle=SETTLE) + + +#: Offered as `roledesk job pull` and to ``'s Pull button, the same +#: way newsdesk's cascade stages register themselves in `TASKS`. +TASKS["pull"] = _pull_task diff --git a/seed/companies.json b/seed/companies.json new file mode 100644 index 0000000..5735525 --- /dev/null +++ b/seed/companies.json @@ -0,0 +1,1307 @@ +{ + "_note": "179 companies from jobhunt-hq's seed.json (https://github.com/rembrandtreyes/jobhunt-hq, MIT), reduced to key/name/board/careers_url, plus a hand-added DACH set. Generated 2026-09-23.", + "companies": [ + { + "key": "andurilindustries", + "name": "Anduril", + "board": "greenhouse:andurilindustries", + "careers_url": "https://job-boards.greenhouse.io/andurilindustries", + "affinity": 0 + }, + { + "key": "carvana", + "name": "Carvana", + "board": "greenhouse:carvana", + "careers_url": "https://job-boards.greenhouse.io/carvana", + "affinity": 0 + }, + { + "key": "databricks", + "name": "Databricks", + "board": "greenhouse:databricks", + "careers_url": "https://job-boards.greenhouse.io/databricks", + "affinity": 0 + }, + { + "key": "openai", + "name": "OpenAI", + "board": "ashby:openai", + "careers_url": "https://jobs.ashbyhq.com/openai", + "affinity": 0 + }, + { + "key": "gopuff", + "name": "Gopuff", + "board": "lever:gopuff", + "careers_url": "https://jobs.lever.co/gopuff", + "affinity": 0 + }, + { + "key": "stripe", + "name": "Stripe", + "board": "greenhouse:stripe", + "careers_url": "https://job-boards.greenhouse.io/stripe", + "affinity": 0 + }, + { + "key": "anthropic", + "name": "Anthropic", + "board": "greenhouse:anthropic", + "careers_url": "https://job-boards.greenhouse.io/anthropic", + "affinity": 0 + }, + { + "key": "airwallex", + "name": "Airwallex", + "board": "ashby:airwallex", + "careers_url": "https://jobs.ashbyhq.com/airwallex", + "affinity": 0 + }, + { + "key": "axon", + "name": "Axon", + "board": "greenhouse:axon", + "careers_url": "https://job-boards.greenhouse.io/axon", + "affinity": 0 + }, + { + "key": "datadog", + "name": "Datadog", + "board": "greenhouse:datadog", + "careers_url": "https://job-boards.greenhouse.io/datadog", + "affinity": 0 + }, + { + "key": "mongodb", + "name": "MongoDB", + "board": "greenhouse:mongodb", + "careers_url": "https://job-boards.greenhouse.io/mongodb", + "affinity": 0 + }, + { + "key": "nebius", + "name": "Nebius", + "board": "greenhouse:nebius", + "careers_url": "https://job-boards.greenhouse.io/nebius", + "affinity": 0 + }, + { + "key": "cloudflare", + "name": "Cloudflare", + "board": "greenhouse:cloudflare", + "careers_url": "https://job-boards.greenhouse.io/cloudflare", + "affinity": 0 + }, + { + "key": "zscaler", + "name": "Zscaler", + "board": "greenhouse:zscaler", + "careers_url": "https://job-boards.greenhouse.io/zscaler", + "affinity": 0 + }, + { + "key": "elastic", + "name": "Elastic", + "board": "greenhouse:elastic", + "careers_url": "https://job-boards.greenhouse.io/elastic", + "affinity": 0 + }, + { + "key": "purestorage", + "name": "Everpure", + "board": "greenhouse:purestorage", + "careers_url": "https://job-boards.greenhouse.io/purestorage", + "affinity": 0 + }, + { + "key": "crusoe", + "name": "Crusoe", + "board": "ashby:crusoe", + "careers_url": "https://jobs.ashbyhq.com/crusoe", + "affinity": 0 + }, + { + "key": "toast", + "name": "Toast", + "board": "greenhouse:toast", + "careers_url": "https://job-boards.greenhouse.io/toast", + "affinity": 0 + }, + { + "key": "okta", + "name": "Okta", + "board": "greenhouse:okta", + "careers_url": "https://job-boards.greenhouse.io/okta", + "affinity": 0 + }, + { + "key": "palantir", + "name": "Palantir", + "board": "lever:palantir", + "careers_url": "https://jobs.lever.co/palantir", + "affinity": 0 + }, + { + "key": "braze", + "name": "Braze", + "board": "greenhouse:braze", + "careers_url": "https://job-boards.greenhouse.io/braze", + "affinity": 0 + }, + { + "key": "canonical", + "name": "Canonical", + "board": "greenhouse:canonical", + "careers_url": "https://job-boards.greenhouse.io/canonical", + "affinity": 0 + }, + { + "key": "harvey", + "name": "Harvey", + "board": "ashby:harvey", + "careers_url": "https://jobs.ashbyhq.com/harvey", + "affinity": 0 + }, + { + "key": "verkada", + "name": "Verkada", + "board": "greenhouse:verkada", + "careers_url": "https://job-boards.greenhouse.io/verkada", + "affinity": 0 + }, + { + "key": "coreweave", + "name": "CoreWeave", + "board": "greenhouse:coreweave", + "careers_url": "https://job-boards.greenhouse.io/coreweave", + "affinity": 0 + }, + { + "key": "oscar", + "name": "Oscar Health", + "board": "greenhouse:oscar", + "careers_url": "https://job-boards.greenhouse.io/oscar", + "affinity": 0 + }, + { + "key": "xai", + "name": "xAI", + "board": "greenhouse:xai", + "careers_url": "https://job-boards.greenhouse.io/xai", + "affinity": 0 + }, + { + "key": "samsara", + "name": "Samsara", + "board": "greenhouse:samsara", + "careers_url": "https://job-boards.greenhouse.io/samsara", + "affinity": 0 + }, + { + "key": "roku", + "name": "Roku", + "board": "greenhouse:roku", + "careers_url": "https://job-boards.greenhouse.io/roku", + "affinity": 0 + }, + { + "key": "brex", + "name": "Brex", + "board": "greenhouse:brex", + "careers_url": "https://job-boards.greenhouse.io/brex", + "affinity": 0 + }, + { + "key": "roblox", + "name": "Roblox", + "board": "greenhouse:roblox", + "careers_url": "https://job-boards.greenhouse.io/roblox", + "affinity": 0 + }, + { + "key": "zoox", + "name": "Zoox", + "board": "lever:zoox", + "careers_url": "https://jobs.lever.co/zoox", + "affinity": 0 + }, + { + "key": "elevenlabs", + "name": "ElevenLabs", + "board": "ashby:elevenlabs", + "careers_url": "https://jobs.ashbyhq.com/elevenlabs", + "affinity": 0 + }, + { + "key": "scaleai", + "name": "Scale AI", + "board": "greenhouse:scaleai", + "careers_url": "https://job-boards.greenhouse.io/scaleai", + "affinity": 0 + }, + { + "key": "block", + "name": "Block", + "board": "greenhouse:block", + "careers_url": "https://job-boards.greenhouse.io/block", + "affinity": 0 + }, + { + "key": "adyen", + "name": "Adyen", + "board": "greenhouse:adyen", + "careers_url": "https://job-boards.greenhouse.io/adyen", + "affinity": 0 + }, + { + "key": "coinbase", + "name": "Coinbase", + "board": "greenhouse:coinbase", + "careers_url": "https://job-boards.greenhouse.io/coinbase", + "affinity": 0 + }, + { + "key": "sierra", + "name": "Sierra", + "board": "ashby:sierra", + "careers_url": "https://jobs.ashbyhq.com/sierra", + "affinity": 0 + }, + { + "key": "gitlab", + "name": "GitLab", + "board": "greenhouse:gitlab", + "careers_url": "https://job-boards.greenhouse.io/gitlab", + "affinity": 0 + }, + { + "key": "affirm", + "name": "Affirm", + "board": "greenhouse:affirm", + "careers_url": "https://job-boards.greenhouse.io/affirm", + "affinity": 0 + }, + { + "key": "fivetran", + "name": "Fivetran", + "board": "greenhouse:fivetran", + "careers_url": "https://job-boards.greenhouse.io/fivetran", + "affinity": 0 + }, + { + "key": "clickhouse", + "name": "ClickHouse", + "board": "ashby:clickhouse", + "careers_url": "https://jobs.ashbyhq.com/clickhouse", + "affinity": 0 + }, + { + "key": "flexport", + "name": "Flexport", + "board": "greenhouse:flexport", + "careers_url": "https://job-boards.greenhouse.io/flexport", + "affinity": 0 + }, + { + "key": "lyft", + "name": "Lyft", + "board": "greenhouse:lyft", + "careers_url": "https://job-boards.greenhouse.io/lyft", + "affinity": 0 + }, + { + "key": "airbnb", + "name": "Airbnb", + "board": "greenhouse:airbnb", + "careers_url": "https://job-boards.greenhouse.io/airbnb", + "affinity": 0 + }, + { + "key": "pinterest", + "name": "Pinterest", + "board": "greenhouse:pinterest", + "careers_url": "https://job-boards.greenhouse.io/pinterest", + "affinity": 0 + }, + { + "key": "robinhood", + "name": "Robinhood", + "board": "greenhouse:robinhood", + "careers_url": "https://job-boards.greenhouse.io/robinhood", + "affinity": 0 + }, + { + "key": "reddit", + "name": "Reddit", + "board": "greenhouse:reddit", + "careers_url": "https://job-boards.greenhouse.io/reddit", + "affinity": 0 + }, + { + "key": "figma", + "name": "Figma", + "board": "greenhouse:figma", + "careers_url": "https://job-boards.greenhouse.io/figma", + "affinity": 0 + }, + { + "key": "whoop", + "name": "WHOOP", + "board": "ashby:whoop", + "careers_url": "https://jobs.ashbyhq.com/whoop", + "affinity": 0 + }, + { + "key": "ramp", + "name": "Ramp", + "board": "ashby:ramp", + "careers_url": "https://jobs.ashbyhq.com/ramp", + "affinity": 0 + }, + { + "key": "epicgames", + "name": "Epic Games", + "board": "greenhouse:epicgames", + "careers_url": "https://job-boards.greenhouse.io/epicgames", + "affinity": 0 + }, + { + "key": "netskope", + "name": "Netskope", + "board": "greenhouse:netskope", + "careers_url": "https://job-boards.greenhouse.io/netskope", + "affinity": 0 + }, + { + "key": "decagon", + "name": "Decagon", + "board": "ashby:decagon", + "careers_url": "https://jobs.ashbyhq.com/decagon", + "affinity": 0 + }, + { + "key": "klaviyo", + "name": "Klaviyo", + "board": "greenhouse:klaviyo", + "careers_url": "https://job-boards.greenhouse.io/klaviyo", + "affinity": 0 + }, + { + "key": "twilio", + "name": "Twilio", + "board": "greenhouse:twilio", + "careers_url": "https://job-boards.greenhouse.io/twilio", + "affinity": 0 + }, + { + "key": "cohere", + "name": "Cohere", + "board": "ashby:cohere", + "careers_url": "https://jobs.ashbyhq.com/cohere", + "affinity": 0 + }, + { + "key": "hubspotjobs", + "name": "HubSpot", + "board": "greenhouse:hubspotjobs", + "careers_url": "https://job-boards.greenhouse.io/hubspotjobs", + "affinity": 0 + }, + { + "key": "grafanalabs", + "name": "Grafana Labs", + "board": "greenhouse:grafanalabs", + "careers_url": "https://job-boards.greenhouse.io/grafanalabs", + "affinity": 0 + }, + { + "key": "rubrik", + "name": "Rubrik", + "board": "greenhouse:rubrik", + "careers_url": "https://job-boards.greenhouse.io/rubrik", + "affinity": 0 + }, + { + "key": "notion", + "name": "Notion", + "board": "ashby:notion", + "careers_url": "https://jobs.ashbyhq.com/notion", + "affinity": 0 + }, + { + "key": "nubank", + "name": "Nubank", + "board": "ashby:nubank", + "careers_url": "https://jobs.ashbyhq.com/nubank", + "affinity": 0 + }, + { + "key": "cursor", + "name": "Cursor", + "board": "ashby:cursor", + "careers_url": "https://jobs.ashbyhq.com/cursor", + "affinity": 0 + }, + { + "key": "perplexity", + "name": "Perplexity", + "board": "ashby:perplexity", + "careers_url": "https://jobs.ashbyhq.com/perplexity", + "affinity": 0 + }, + { + "key": "five9", + "name": "Five9", + "board": "greenhouse:five9", + "careers_url": "https://job-boards.greenhouse.io/five9", + "affinity": 0 + }, + { + "key": "intercom", + "name": "Intercom", + "board": "greenhouse:intercom", + "careers_url": "https://job-boards.greenhouse.io/intercom", + "affinity": 0 + }, + { + "key": "plaid", + "name": "Plaid", + "board": "ashby:plaid", + "careers_url": "https://jobs.ashbyhq.com/plaid", + "affinity": 0 + }, + { + "key": "cerebras", + "name": "Cerebras", + "board": "ashby:cerebras", + "careers_url": "https://jobs.ashbyhq.com/cerebras", + "affinity": 0 + }, + { + "key": "zoominfo", + "name": "ZoomInfo", + "board": "greenhouse:zoominfo", + "careers_url": "https://job-boards.greenhouse.io/zoominfo", + "affinity": 0 + }, + { + "key": "instacart", + "name": "Instacart", + "board": "greenhouse:instacart", + "careers_url": "https://job-boards.greenhouse.io/instacart", + "affinity": 0 + }, + { + "key": "mercor", + "name": "Mercor", + "board": "ashby:mercor", + "careers_url": "https://jobs.ashbyhq.com/mercor", + "affinity": 0 + }, + { + "key": "nuro", + "name": "Nuro", + "board": "greenhouse:nuro", + "careers_url": "https://job-boards.greenhouse.io/nuro", + "affinity": 0 + }, + { + "key": "asana", + "name": "Asana", + "board": "greenhouse:asana", + "careers_url": "https://job-boards.greenhouse.io/asana", + "affinity": 0 + }, + { + "key": "langchain", + "name": "LangChain", + "board": "ashby:langchain", + "careers_url": "https://jobs.ashbyhq.com/langchain", + "affinity": 0 + }, + { + "key": "baseten", + "name": "Baseten", + "board": "ashby:baseten", + "careers_url": "https://jobs.ashbyhq.com/baseten", + "affinity": 0 + }, + { + "key": "cognition", + "name": "Cognition", + "board": "ashby:cognition", + "careers_url": "https://jobs.ashbyhq.com/cognition", + "affinity": 0 + }, + { + "key": "cresta", + "name": "Cresta", + "board": "greenhouse:cresta", + "careers_url": "https://job-boards.greenhouse.io/cresta", + "affinity": 0 + }, + { + "key": "smartsheet", + "name": "Smartsheet", + "board": "greenhouse:smartsheet", + "careers_url": "https://job-boards.greenhouse.io/smartsheet", + "affinity": 0 + }, + { + "key": "vanta", + "name": "Vanta", + "board": "ashby:vanta", + "careers_url": "https://jobs.ashbyhq.com/vanta", + "affinity": 0 + }, + { + "key": "deepgram", + "name": "Deepgram", + "board": "ashby:deepgram", + "careers_url": "https://jobs.ashbyhq.com/deepgram", + "affinity": 0 + }, + { + "key": "gusto", + "name": "Gusto", + "board": "greenhouse:gusto", + "careers_url": "https://job-boards.greenhouse.io/gusto", + "affinity": 0 + }, + { + "key": "justworks", + "name": "Justworks", + "board": "greenhouse:justworks", + "careers_url": "https://job-boards.greenhouse.io/justworks", + "affinity": 0 + }, + { + "key": "oura", + "name": "Oura", + "board": "greenhouse:oura", + "careers_url": "https://job-boards.greenhouse.io/oura", + "affinity": 0 + }, + { + "key": "upstart", + "name": "Upstart", + "board": "greenhouse:upstart", + "careers_url": "https://job-boards.greenhouse.io/upstart", + "affinity": 0 + }, + { + "key": "lambda", + "name": "Lambda", + "board": "ashby:lambda", + "careers_url": "https://jobs.ashbyhq.com/lambda", + "affinity": 0 + }, + { + "key": "headway", + "name": "Headway", + "board": "ashby:headway", + "careers_url": "https://jobs.ashbyhq.com/headway", + "affinity": 0 + }, + { + "key": "saviynt", + "name": "Saviynt", + "board": "lever:saviynt", + "careers_url": "https://jobs.lever.co/saviynt", + "affinity": 0 + }, + { + "key": "vercel", + "name": "Vercel", + "board": "greenhouse:vercel", + "careers_url": "https://job-boards.greenhouse.io/vercel", + "affinity": 0 + }, + { + "key": "hightouch", + "name": "Hightouch", + "board": "greenhouse:hightouch", + "careers_url": "https://job-boards.greenhouse.io/hightouch", + "affinity": 0 + }, + { + "key": "mixpanel", + "name": "Mixpanel", + "board": "greenhouse:mixpanel", + "careers_url": "https://job-boards.greenhouse.io/mixpanel", + "affinity": 0 + }, + { + "key": "socure", + "name": "Socure", + "board": "ashby:socure", + "careers_url": "https://jobs.ashbyhq.com/socure", + "affinity": 0 + }, + { + "key": "abnormalsecurity", + "name": "Abnormal Security", + "board": "greenhouse:abnormalsecurity", + "careers_url": "https://job-boards.greenhouse.io/abnormalsecurity", + "affinity": 0 + }, + { + "key": "chainguard", + "name": "Chainguard", + "board": "greenhouse:chainguard", + "careers_url": "https://job-boards.greenhouse.io/chainguard", + "affinity": 0 + }, + { + "key": "duolingo", + "name": "Duolingo", + "board": "greenhouse:duolingo", + "careers_url": "https://job-boards.greenhouse.io/duolingo", + "affinity": 0 + }, + { + "key": "fireworks", + "name": "Fireworks", + "board": "ashby:fireworks", + "careers_url": "https://jobs.ashbyhq.com/fireworks", + "affinity": 0 + }, + { + "key": "lovable", + "name": "Lovable", + "board": "ashby:lovable", + "careers_url": "https://jobs.ashbyhq.com/lovable", + "affinity": 0 + }, + { + "key": "replit", + "name": "Replit", + "board": "ashby:replit", + "careers_url": "https://jobs.ashbyhq.com/replit", + "affinity": 0 + }, + { + "key": "togetherai", + "name": "Together AI", + "board": "greenhouse:togetherai", + "careers_url": "https://job-boards.greenhouse.io/togetherai", + "affinity": 0 + }, + { + "key": "workato", + "name": "Workato", + "board": "greenhouse:workato", + "careers_url": "https://job-boards.greenhouse.io/workato", + "affinity": 0 + }, + { + "key": "crunchyroll", + "name": "Crunchyroll", + "board": "greenhouse:crunchyroll", + "careers_url": "https://job-boards.greenhouse.io/crunchyroll", + "affinity": 0 + }, + { + "key": "humaninterest", + "name": "Human Interest", + "board": "greenhouse:humaninterest", + "careers_url": "https://job-boards.greenhouse.io/humaninterest", + "affinity": 0 + }, + { + "key": "faire", + "name": "Faire", + "board": "greenhouse:faire", + "careers_url": "https://job-boards.greenhouse.io/faire", + "affinity": 0 + }, + { + "key": "kong", + "name": "Kong", + "board": "ashby:kong", + "careers_url": "https://jobs.ashbyhq.com/kong", + "affinity": 0 + }, + { + "key": "ashby", + "name": "Ashby", + "board": "ashby:ashby", + "careers_url": "https://jobs.ashbyhq.com/ashby", + "affinity": 0 + }, + { + "key": "mozilla", + "name": "Mozilla", + "board": "greenhouse:mozilla", + "careers_url": "https://job-boards.greenhouse.io/mozilla", + "affinity": 0 + }, + { + "key": "fireblocks", + "name": "Fireblocks", + "board": "greenhouse:fireblocks", + "careers_url": "https://job-boards.greenhouse.io/fireblocks", + "affinity": 0 + }, + { + "key": "spotify", + "name": "Spotify", + "board": "lever:spotify", + "careers_url": "https://jobs.lever.co/spotify", + "affinity": 0 + }, + { + "key": "chime", + "name": "Chime", + "board": "greenhouse:chime", + "careers_url": "https://job-boards.greenhouse.io/chime", + "affinity": 0 + }, + { + "key": "carta", + "name": "Carta", + "board": "greenhouse:carta", + "careers_url": "https://job-boards.greenhouse.io/carta", + "affinity": 0 + }, + { + "key": "dialpad", + "name": "Dialpad", + "board": "greenhouse:dialpad", + "careers_url": "https://job-boards.greenhouse.io/dialpad", + "affinity": 0 + }, + { + "key": "suno", + "name": "Suno", + "board": "ashby:suno", + "careers_url": "https://jobs.ashbyhq.com/suno", + "affinity": 0 + }, + { + "key": "mercury", + "name": "Mercury", + "board": "greenhouse:mercury", + "careers_url": "https://job-boards.greenhouse.io/mercury", + "affinity": 0 + }, + { + "key": "temporal", + "name": "Temporal", + "board": "ashby:temporal", + "careers_url": "https://jobs.ashbyhq.com/temporal", + "affinity": 0 + }, + { + "key": "harnessinc", + "name": "Harness", + "board": "greenhouse:harnessinc", + "careers_url": "https://job-boards.greenhouse.io/harnessinc", + "affinity": 0 + }, + { + "key": "1password", + "name": "1Password", + "board": "ashby:1password", + "careers_url": "https://jobs.ashbyhq.com/1password", + "affinity": 0 + }, + { + "key": "supabase", + "name": "Supabase", + "board": "ashby:supabase", + "careers_url": "https://jobs.ashbyhq.com/supabase", + "affinity": 0 + }, + { + "key": "clickup", + "name": "ClickUp", + "board": "ashby:clickup", + "careers_url": "https://jobs.ashbyhq.com/clickup", + "affinity": 0 + }, + { + "key": "peloton", + "name": "Peloton", + "board": "greenhouse:peloton", + "careers_url": "https://job-boards.greenhouse.io/peloton", + "affinity": 0 + }, + { + "key": "tailscale", + "name": "Tailscale", + "board": "greenhouse:tailscale", + "careers_url": "https://job-boards.greenhouse.io/tailscale", + "affinity": 0 + }, + { + "key": "cribl", + "name": "Cribl", + "board": "greenhouse:cribl", + "careers_url": "https://job-boards.greenhouse.io/cribl", + "affinity": 0 + }, + { + "key": "sofi", + "name": "SoFi", + "board": "greenhouse:sofi", + "careers_url": "https://job-boards.greenhouse.io/sofi", + "affinity": 0 + }, + { + "key": "beyondtrust", + "name": "BeyondTrust", + "board": "greenhouse:beyondtrust", + "careers_url": "https://job-boards.greenhouse.io/beyondtrust", + "affinity": 0 + }, + { + "key": "tanium", + "name": "Tanium", + "board": "greenhouse:tanium", + "careers_url": "https://job-boards.greenhouse.io/tanium", + "affinity": 0 + }, + { + "key": "lessen", + "name": "Lessen", + "board": "lever:lessen", + "careers_url": "https://jobs.lever.co/lessen", + "affinity": 0 + }, + { + "key": "logicmonitor", + "name": "LogicMonitor", + "board": "greenhouse:logicmonitor", + "careers_url": "https://job-boards.greenhouse.io/logicmonitor", + "affinity": 0 + }, + { + "key": "launchdarkly", + "name": "LaunchDarkly", + "board": "greenhouse:launchdarkly", + "careers_url": "https://job-boards.greenhouse.io/launchdarkly", + "affinity": 0 + }, + { + "key": "synthesia", + "name": "Synthesia", + "board": "ashby:synthesia", + "careers_url": "https://jobs.ashbyhq.com/synthesia", + "affinity": 0 + }, + { + "key": "benchling", + "name": "Benchling", + "board": "ashby:benchling", + "careers_url": "https://jobs.ashbyhq.com/benchling", + "affinity": 0 + }, + { + "key": "writer", + "name": "Writer", + "board": "ashby:writer", + "careers_url": "https://jobs.ashbyhq.com/writer", + "affinity": 0 + }, + { + "key": "newrelic", + "name": "New Relic", + "board": "greenhouse:newrelic", + "careers_url": "https://job-boards.greenhouse.io/newrelic", + "affinity": 0 + }, + { + "key": "pagerduty", + "name": "PagerDuty", + "board": "greenhouse:pagerduty", + "careers_url": "https://job-boards.greenhouse.io/pagerduty", + "affinity": 0 + }, + { + "key": "checkr", + "name": "Checkr", + "board": "greenhouse:checkr", + "careers_url": "https://job-boards.greenhouse.io/checkr", + "affinity": 0 + }, + { + "key": "twitch", + "name": "Twitch", + "board": "greenhouse:twitch", + "careers_url": "https://job-boards.greenhouse.io/twitch", + "affinity": 0 + }, + { + "key": "apolloio", + "name": "Apollo.io", + "board": "greenhouse:apolloio", + "careers_url": "https://job-boards.greenhouse.io/apolloio", + "affinity": 0 + }, + { + "key": "discord", + "name": "Discord", + "board": "greenhouse:discord", + "careers_url": "https://job-boards.greenhouse.io/discord", + "affinity": 0 + }, + { + "key": "sentry", + "name": "Sentry", + "board": "ashby:sentry", + "careers_url": "https://jobs.ashbyhq.com/sentry", + "affinity": 0 + }, + { + "key": "dropbox", + "name": "Dropbox", + "board": "greenhouse:dropbox", + "careers_url": "https://job-boards.greenhouse.io/dropbox", + "affinity": 0 + }, + { + "key": "thumbtack", + "name": "Thumbtack", + "board": "ashby:thumbtack", + "careers_url": "https://jobs.ashbyhq.com/thumbtack", + "affinity": 0 + }, + { + "key": "fastly", + "name": "Fastly", + "board": "greenhouse:fastly", + "careers_url": "https://job-boards.greenhouse.io/fastly", + "affinity": 0 + }, + { + "key": "singlestore", + "name": "SingleStore", + "board": "greenhouse:singlestore", + "careers_url": "https://job-boards.greenhouse.io/singlestore", + "affinity": 0 + }, + { + "key": "amplitude", + "name": "Amplitude", + "board": "greenhouse:amplitude", + "careers_url": "https://job-boards.greenhouse.io/amplitude", + "affinity": 0 + }, + { + "key": "cultureamp", + "name": "Culture Amp", + "board": "greenhouse:cultureamp", + "careers_url": "https://job-boards.greenhouse.io/cultureamp", + "affinity": 0 + }, + { + "key": "gemini", + "name": "Gemini", + "board": "greenhouse:gemini", + "careers_url": "https://job-boards.greenhouse.io/gemini", + "affinity": 0 + }, + { + "key": "sardine", + "name": "Sardine", + "board": "ashby:sardine", + "careers_url": "https://jobs.ashbyhq.com/sardine", + "affinity": 0 + }, + { + "key": "drata", + "name": "Drata", + "board": "ashby:drata", + "careers_url": "https://jobs.ashbyhq.com/drata", + "affinity": 0 + }, + { + "key": "godaddy", + "name": "GoDaddy", + "board": "greenhouse:godaddy", + "careers_url": "https://job-boards.greenhouse.io/godaddy", + "affinity": 0 + }, + { + "key": "render", + "name": "Render", + "board": "ashby:render", + "careers_url": "https://jobs.ashbyhq.com/render", + "affinity": 0 + }, + { + "key": "ziprecruiter", + "name": "ZipRecruiter", + "board": "greenhouse:ziprecruiter", + "careers_url": "https://job-boards.greenhouse.io/ziprecruiter", + "affinity": 0 + }, + { + "key": "modal", + "name": "Modal", + "board": "ashby:modal", + "careers_url": "https://jobs.ashbyhq.com/modal", + "affinity": 0 + }, + { + "key": "n8n", + "name": "n8n", + "board": "ashby:n8n", + "careers_url": "https://jobs.ashbyhq.com/n8n", + "affinity": 0 + }, + { + "key": "salesloft", + "name": "Salesloft", + "board": "greenhouse:salesloft", + "careers_url": "https://job-boards.greenhouse.io/salesloft", + "affinity": 0 + }, + { + "key": "watershed", + "name": "Watershed", + "board": "ashby:watershed", + "careers_url": "https://jobs.ashbyhq.com/watershed", + "affinity": 0 + }, + { + "key": "attentive", + "name": "Attentive", + "board": "greenhouse:attentive", + "careers_url": "https://job-boards.greenhouse.io/attentive", + "affinity": 0 + }, + { + "key": "backblaze", + "name": "Backblaze", + "board": "greenhouse:backblaze", + "careers_url": "https://job-boards.greenhouse.io/backblaze", + "affinity": 0 + }, + { + "key": "livekit", + "name": "LiveKit", + "board": "ashby:livekit", + "careers_url": "https://jobs.ashbyhq.com/livekit", + "affinity": 0 + }, + { + "key": "hudl", + "name": "Hudl", + "board": "greenhouse:hudl", + "careers_url": "https://job-boards.greenhouse.io/hudl", + "affinity": 0 + }, + { + "key": "linear", + "name": "Linear", + "board": "ashby:linear", + "careers_url": "https://jobs.ashbyhq.com/linear", + "affinity": 0 + }, + { + "key": "seatgeek", + "name": "SeatGeek", + "board": "greenhouse:seatgeek", + "careers_url": "https://job-boards.greenhouse.io/seatgeek", + "affinity": 0 + }, + { + "key": "squarespace", + "name": "Squarespace", + "board": "greenhouse:squarespace", + "careers_url": "https://job-boards.greenhouse.io/squarespace", + "affinity": 0 + }, + { + "key": "6sense", + "name": "6sense", + "board": "greenhouse:6sense", + "careers_url": "https://job-boards.greenhouse.io/6sense", + "affinity": 0 + }, + { + "key": "axios", + "name": "Axios", + "board": "greenhouse:axios", + "careers_url": "https://job-boards.greenhouse.io/axios", + "affinity": 0 + }, + { + "key": "coder", + "name": "Coder", + "board": "ashby:coder", + "careers_url": "https://jobs.ashbyhq.com/coder", + "affinity": 0 + }, + { + "key": "vapi", + "name": "Vapi", + "board": "ashby:vapi", + "careers_url": "https://jobs.ashbyhq.com/vapi", + "affinity": 0 + }, + { + "key": "outreach", + "name": "Outreach", + "board": "lever:outreach", + "careers_url": "https://jobs.lever.co/outreach", + "affinity": 0 + }, + { + "key": "nextiva", + "name": "Nextiva", + "board": "greenhouse:nextiva", + "careers_url": "https://job-boards.greenhouse.io/nextiva", + "affinity": 0 + }, + { + "key": "baidu", + "name": "Baidu USA", + "board": "greenhouse:baidu", + "careers_url": "https://job-boards.greenhouse.io/baidu", + "affinity": 0 + }, + { + "key": "braintrust", + "name": "Braintrust", + "board": "ashby:braintrust", + "careers_url": "https://jobs.ashbyhq.com/braintrust", + "affinity": 0 + }, + { + "key": "toptal", + "name": "Toptal", + "board": "lever:toptal", + "careers_url": "https://jobs.lever.co/toptal", + "affinity": 0 + }, + { + "key": "zuora", + "name": "Zuora", + "board": "greenhouse:zuora", + "careers_url": "https://job-boards.greenhouse.io/zuora", + "affinity": 0 + }, + { + "key": "alloy", + "name": "Alloy", + "board": "greenhouse:alloy", + "careers_url": "https://job-boards.greenhouse.io/alloy", + "affinity": 0 + }, + { + "key": "melio", + "name": "Melio", + "board": "greenhouse:melio", + "careers_url": "https://job-boards.greenhouse.io/melio", + "affinity": 0 + }, + { + "key": "angellist", + "name": "AngelList", + "board": "lever:angellist", + "careers_url": "https://jobs.lever.co/angellist", + "affinity": 0 + }, + { + "key": "bandwidth", + "name": "Bandwidth", + "board": "greenhouse:bandwidth", + "careers_url": "https://job-boards.greenhouse.io/bandwidth", + "affinity": 0 + }, + { + "key": "betterment", + "name": "Betterment", + "board": "greenhouse:betterment", + "careers_url": "https://job-boards.greenhouse.io/betterment", + "affinity": 0 + }, + { + "key": "wealthfront", + "name": "Wealthfront", + "board": "lever:wealthfront", + "careers_url": "https://jobs.lever.co/wealthfront", + "affinity": 0 + }, + { + "key": "pendo", + "name": "Pendo", + "board": "greenhouse:pendo", + "careers_url": "https://job-boards.greenhouse.io/pendo", + "affinity": 0 + }, + { + "key": "planetscale", + "name": "PlanetScale", + "board": "greenhouse:planetscale", + "careers_url": "https://job-boards.greenhouse.io/planetscale", + "affinity": 0 + }, + { + "key": "posthog", + "name": "PostHog", + "board": "ashby:posthog", + "careers_url": "https://jobs.ashbyhq.com/posthog", + "affinity": 0 + }, + { + "key": "hex", + "name": "Hex", + "board": "ashby:hex", + "careers_url": "https://jobs.ashbyhq.com/hex", + "affinity": 0 + }, + { + "key": "celonis", + "name": "Celonis", + "board": "greenhouse:celonis", + "careers_url": "https://job-boards.greenhouse.io/celonis", + "affinity": 0 + }, + { + "key": "n26", + "name": "N26", + "board": "greenhouse:n26", + "careers_url": "https://job-boards.greenhouse.io/n26", + "affinity": 0 + }, + { + "key": "traderepublic", + "name": "Trade Republic", + "board": "greenhouse:traderepublic", + "careers_url": "https://job-boards.greenhouse.io/traderepublic", + "affinity": 0 + }, + { + "key": "helsing", + "name": "Helsing", + "board": "greenhouse:helsing", + "careers_url": "https://job-boards.greenhouse.io/helsing", + "affinity": 0 + }, + { + "key": "deepl", + "name": "DeepL", + "board": "ashby:deepl", + "careers_url": "https://jobs.ashbyhq.com/deepl", + "affinity": 0 + }, + { + "key": "alephalpha", + "name": "Aleph Alpha", + "board": "ashby:alephalpha", + "careers_url": "https://jobs.ashbyhq.com/alephalpha", + "affinity": 0 + }, + { + "key": "parloa", + "name": "Parloa", + "board": "greenhouse:parloa", + "careers_url": "https://job-boards.greenhouse.io/parloa", + "affinity": 0 + } + ] +} diff --git a/tests/test_pull.py b/tests/test_pull.py new file mode 100644 index 0000000..9a0c389 --- /dev/null +++ b/tests/test_pull.py @@ -0,0 +1,127 @@ +"""The `pull` task: batching, the batch job's upsert, and the settle's +closure — with the `ats` plugin's own `fetch()` stubbed out, so nothing here +touches the network.""" + +from __future__ import annotations + +import sys +from datetime import UTC, datetime +from pathlib import Path +from typing import ClassVar + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from desk.models import Item # noqa: E402 +from plugins.ats import ATSSource # noqa: E402 +from roledesk.jobs import pull # noqa: E402 +from roledesk.storage import Store # noqa: E402 + + +class FakeConfig: + sources: ClassVar[dict[str, object]] = {} + + +def _item(board: str, slug: str, job_id: str) -> Item: + item = Item( + source_id=f"{board}:{slug}:{job_id}", location=f"https://x.example/{job_id}", + title="Engineer", body="", published_at=datetime(2026, 9, 20, tzinfo=UTC), + raw={"company": "Acme", "place": "Berlin", "department": "", + "function": "engineering", "remote": None, "salary": "", + "board": board, "slug": slug}) + item.source = "ats" + return item + + +@pytest.fixture +def store(tmp_path: Path) -> Store: + return Store(tmp_path / "roledesk.db") + + +def test_boards_batches_fetchable_subscriptions_and_stamps_a_run(store: Store) -> None: + for i in range(3): + store.add_subscription("ats", f"greenhouse:c{i}", label=f"C{i}", + company=f"C{i}", board="greenhouse") + off = store.add_subscription("ats", "greenhouse:off", label="Off", + company="Off", board="greenhouse") + store.update_subscription(off.id, fetch=False) + + batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type] + assert sum(len(b["targets"]) for b in batches) == 3 + assert store.state.get("ats:run", "") != "" + + +def test_boards_is_empty_and_stamps_nothing_with_no_subscriptions(store: Store) -> None: + assert pull._boards(FakeConfig(), store) == [] # type: ignore[arg-type] + assert store.state.get("ats:run", "") == "" + + +def test_run_batch_upserts_every_item_the_plugin_yields( + store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + store.add_subscription("ats", "greenhouse:acme", label="Acme", + company="Acme", board="greenhouse") + + def fake_fetch(self: ATSSource) -> object: + yield _item("greenhouse", "acme", "1") + yield _item("greenhouse", "acme", "2") + self.state["ats:ok:greenhouse:acme"] = self.state.get("ats:run", "") + self.state["ats:ids:greenhouse:acme"] = '["1", "2"]' + + monkeypatch.setattr(ATSSource, "fetch", fake_fetch) + store.state["ats:run"] = "run-1" + + result = pull.run_batch(FakeConfig(), store, targets=["greenhouse:acme"]) + + assert result == {"roles": 2} + assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:acme:2"} + assert store.roles()[0].subscription_id is not None + + +def test_close_boards_closes_what_a_completed_board_no_longer_lists( + store: Store) -> None: + sub = store.add_subscription("ats", "greenhouse:acme", label="Acme", + company="Acme", board="greenhouse") + other = store.add_subscription("ats", "greenhouse:other", label="Other", + company="Other", board="greenhouse") + + store.upsert_role(_item("greenhouse", "acme", "1"), sub) + store.upsert_role(_item("greenhouse", "acme", "2"), sub) + store.upsert_role(_item("greenhouse", "other", "9"), other) + + # acme answered this run and now lists only "1"; other's board failed + # (or its batch never got to it) and stamped nothing. + store.state["ats:run"] = "run-1" + store.state["ats:ok:greenhouse:acme"] = "run-1" + store.state["ats:ids:greenhouse:acme"] = '["greenhouse:acme:1"]' + + class FakeTally: + results = ({"roles": 1},) + + result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type] + + assert result == {"boards": 1, "failed": 1, "roles": 1, "closed": 1} + assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"} + + +def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run( + store: Store) -> None: + sub = store.add_subscription("ats", "greenhouse:acme", label="Acme", + company="Acme", board="greenhouse") + store.upsert_role(_item("greenhouse", "acme", "1"), sub) + + # A stale stamp from a run that is not this one — e.g. this run's batch + # for this board never finished. + store.state["ats:run"] = "run-2" + store.state["ats:ok:greenhouse:acme"] = "run-1" + store.state["ats:ids:greenhouse:acme"] = "[]" + + class FakeTally: + results: tuple[dict[str, int], ...] = () + + result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type] + + assert result == {"boards": 0, "failed": 1, "roles": 0, "closed": 0} + assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"}