Seed 186 companies, add roledesk companies, and pull and close every board
seed/companies.json: 179 companies from jobhunt-hq's seed.json (MIT), reduced to key/name/board/careers_url, plus a hand-added DACH set. `roledesk companies seed|add|check` import it, add one company from its board with a live check first, and report dead boards. roledesk/jobs/pull.py is now a pipeline task: ten boards per pull_batch job on the http resource, upserting every role the ats plugin yields. Its final job's settle closes every board this run answered in full and no longer lists a role from, reading the plugin's own self.state stamps rather than each batch job's return value, so one bad board in a batch does not cost the other nine their closure. `roledesk job pull` runs it, and it registers itself in pipeline.TASKS so <desk-jobs> can start it too. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
21ed472a87
commit
0f59452b2a
5 files changed
+1684
-10
No files matched your search
@@ -0,0 +1,127 @@
|
||||
"""The `pull` task: batching, the batch job's upsert, and the settle's
|
||||
closure — with the `ats` plugin's own `fetch()` stubbed out, so nothing here
|
||||
touches the network."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import ClassVar
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from desk.models import Item # noqa: E402
|
||||
from plugins.ats import ATSSource # noqa: E402
|
||||
from roledesk.jobs import pull # noqa: E402
|
||||
from roledesk.storage import Store # noqa: E402
|
||||
|
||||
|
||||
class FakeConfig:
|
||||
sources: ClassVar[dict[str, object]] = {}
|
||||
|
||||
|
||||
def _item(board: str, slug: str, job_id: str) -> Item:
|
||||
item = Item(
|
||||
source_id=f"{board}:{slug}:{job_id}", location=f"https://x.example/{job_id}",
|
||||
title="Engineer", body="", published_at=datetime(2026, 9, 20, tzinfo=UTC),
|
||||
raw={"company": "Acme", "place": "Berlin", "department": "",
|
||||
"function": "engineering", "remote": None, "salary": "",
|
||||
"board": board, "slug": slug})
|
||||
item.source = "ats"
|
||||
return item
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def store(tmp_path: Path) -> Store:
|
||||
return Store(tmp_path / "roledesk.db")
|
||||
|
||||
|
||||
def test_boards_batches_fetchable_subscriptions_and_stamps_a_run(store: Store) -> None:
|
||||
for i in range(3):
|
||||
store.add_subscription("ats", f"greenhouse:c{i}", label=f"C{i}",
|
||||
company=f"C{i}", board="greenhouse")
|
||||
off = store.add_subscription("ats", "greenhouse:off", label="Off",
|
||||
company="Off", board="greenhouse")
|
||||
store.update_subscription(off.id, fetch=False)
|
||||
|
||||
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
|
||||
assert sum(len(b["targets"]) for b in batches) == 3
|
||||
assert store.state.get("ats:run", "") != ""
|
||||
|
||||
|
||||
def test_boards_is_empty_and_stamps_nothing_with_no_subscriptions(store: Store) -> None:
|
||||
assert pull._boards(FakeConfig(), store) == [] # type: ignore[arg-type]
|
||||
assert store.state.get("ats:run", "") == ""
|
||||
|
||||
|
||||
def test_run_batch_upserts_every_item_the_plugin_yields(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
store.add_subscription("ats", "greenhouse:acme", label="Acme",
|
||||
company="Acme", board="greenhouse")
|
||||
|
||||
def fake_fetch(self: ATSSource) -> object:
|
||||
yield _item("greenhouse", "acme", "1")
|
||||
yield _item("greenhouse", "acme", "2")
|
||||
self.state["ats:ok:greenhouse:acme"] = self.state.get("ats:run", "")
|
||||
self.state["ats:ids:greenhouse:acme"] = '["1", "2"]'
|
||||
|
||||
monkeypatch.setattr(ATSSource, "fetch", fake_fetch)
|
||||
store.state["ats:run"] = "run-1"
|
||||
|
||||
result = pull.run_batch(FakeConfig(), store, targets=["greenhouse:acme"])
|
||||
|
||||
assert result == {"roles": 2}
|
||||
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:acme:2"}
|
||||
assert store.roles()[0].subscription_id is not None
|
||||
|
||||
|
||||
def test_close_boards_closes_what_a_completed_board_no_longer_lists(
|
||||
store: Store) -> None:
|
||||
sub = store.add_subscription("ats", "greenhouse:acme", label="Acme",
|
||||
company="Acme", board="greenhouse")
|
||||
other = store.add_subscription("ats", "greenhouse:other", label="Other",
|
||||
company="Other", board="greenhouse")
|
||||
|
||||
store.upsert_role(_item("greenhouse", "acme", "1"), sub)
|
||||
store.upsert_role(_item("greenhouse", "acme", "2"), sub)
|
||||
store.upsert_role(_item("greenhouse", "other", "9"), other)
|
||||
|
||||
# acme answered this run and now lists only "1"; other's board failed
|
||||
# (or its batch never got to it) and stamped nothing.
|
||||
store.state["ats:run"] = "run-1"
|
||||
store.state["ats:ok:greenhouse:acme"] = "run-1"
|
||||
store.state["ats:ids:greenhouse:acme"] = '["greenhouse:acme:1"]'
|
||||
|
||||
class FakeTally:
|
||||
results = ({"roles": 1},)
|
||||
|
||||
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
||||
|
||||
assert result == {"boards": 1, "failed": 1, "roles": 1, "closed": 1}
|
||||
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"}
|
||||
|
||||
|
||||
def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run(
|
||||
store: Store) -> None:
|
||||
sub = store.add_subscription("ats", "greenhouse:acme", label="Acme",
|
||||
company="Acme", board="greenhouse")
|
||||
store.upsert_role(_item("greenhouse", "acme", "1"), sub)
|
||||
|
||||
# A stale stamp from a run that is not this one — e.g. this run's batch
|
||||
# for this board never finished.
|
||||
store.state["ats:run"] = "run-2"
|
||||
store.state["ats:ok:greenhouse:acme"] = "run-1"
|
||||
store.state["ats:ids:greenhouse:acme"] = "[]"
|
||||
|
||||
class FakeTally:
|
||||
results: tuple[dict[str, int], ...] = ()
|
||||
|
||||
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
||||
|
||||
assert result == {"boards": 0, "failed": 1, "roles": 0, "closed": 0}
|
||||
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"}
|
||||
Reference in new issue
Block a user