773 lines
33 KiB
Python
773 lines
33 KiB
Python
"""The `pull` task: batching, the batch job's upsert, and the settle's
|
|
closure — with the `ats` plugin's own `fetch()` stubbed out, so nothing here
|
|
touches the network."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
import threading
|
|
import types
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from datetime import UTC, datetime, timedelta
|
|
from pathlib import Path
|
|
from typing import ClassVar
|
|
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from desk.models import Item # noqa: E402
|
|
from pipeline import Cancelled, RunContext, Tally # noqa: E402
|
|
from app.models import Application # noqa: E402
|
|
from plugins import ats, readers # noqa: E402
|
|
from plugins.readers import Filters, Reads, _runner # noqa: E402
|
|
from plugins.ats import ATSSource # noqa: E402
|
|
from app import prep # noqa: E402
|
|
from app.jobs import cascade, pull # noqa: E402
|
|
from app.storage import Store # noqa: E402
|
|
|
|
|
|
class FakeConfig:
|
|
sources: ClassVar[dict[str, object]] = {}
|
|
searches: ClassVar[list[object]] = []
|
|
|
|
|
|
def _item(board: str, slug: str, job_id: str, company: str = "Acme") -> Item:
|
|
item = Item(
|
|
source_id=f"{board}:{slug}:{job_id}", location=f"https://x.example/{job_id}",
|
|
title="Engineer", body="", published_at=datetime(2026, 9, 20, tzinfo=UTC),
|
|
raw={"company": company, "place": "Berlin", "department": "",
|
|
"function": "engineering", "remote": None, "salary_text": "",
|
|
"board": board, "slug": slug})
|
|
item.source = "ats"
|
|
return item
|
|
|
|
|
|
@pytest.fixture
|
|
def store(tmp_path: Path) -> Store:
|
|
return Store(tmp_path / "roledesk.db")
|
|
|
|
|
|
def test_boards_batches_fetchable_subscriptions_and_stamps_a_run(store: Store) -> None:
|
|
for i in range(3):
|
|
store.add_subscription("ats", f"greenhouse:c{i}", label=f"C{i}",
|
|
company=f"C{i}", board="greenhouse")
|
|
off = store.add_subscription("ats", "greenhouse:off", label="Off",
|
|
company="Off", board="greenhouse")
|
|
store.update_subscription(off.id, fetch=False)
|
|
|
|
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
|
|
assert sum(len(b["targets"]) for b in batches) == 3
|
|
run = store.state.get(pull.RUN, "")
|
|
assert run and {b["run"] for b in batches} == {run}
|
|
assert pull.running(store) == run
|
|
|
|
|
|
def test_boards_is_empty_and_stamps_nothing_with_no_subscriptions(store: Store) -> None:
|
|
assert pull._boards(FakeConfig(), store) == [] # type: ignore[arg-type]
|
|
assert store.state.get(pull.RUN, "") == ""
|
|
|
|
|
|
def test_run_batch_upserts_every_item_the_plugin_yields(
|
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
store.add_subscription("ats", "greenhouse:acme", label="Acme",
|
|
company="Acme", board="greenhouse")
|
|
|
|
def fake_fetch(self: ATSSource) -> object:
|
|
yield _item("greenhouse", "acme", "1")
|
|
yield _item("greenhouse", "acme", "2")
|
|
self.state["ats:ok:greenhouse:acme"] = self.options["run"]
|
|
self.state["ats:ids:greenhouse:acme"] = '["1", "2"]'
|
|
|
|
monkeypatch.setattr(ATSSource, "fetch", fake_fetch)
|
|
|
|
result = pull.run_batch(FakeConfig(), store, targets=["greenhouse:acme"], run="run-1")
|
|
|
|
assert result == {"roles": 2, "run": "run-1"}
|
|
assert store.state["ats:ok:greenhouse:acme"] == "run-1"
|
|
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:acme:2"}
|
|
assert store.roles()[0].subscription_id is not None
|
|
|
|
|
|
def test_a_kill_stops_the_batch_before_the_next_board(
|
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""`check()` before each board: a Kill pressed while the first is read
|
|
ends the job before the second is fetched, and keeps what the first
|
|
stored."""
|
|
for slug in ("acme", "beta"):
|
|
store.add_subscription("ats", f"greenhouse:{slug}", label=slug.title(),
|
|
company=slug.title(), board="greenhouse")
|
|
fetched: list[str] = []
|
|
killed = False
|
|
|
|
def fake_fetch(self: ATSSource) -> object:
|
|
nonlocal killed
|
|
[(target, _)] = self.targets()
|
|
fetched.append(target)
|
|
yield _item("greenhouse", target.partition(":")[2], "1")
|
|
killed = True
|
|
|
|
monkeypatch.setattr(ATSSource, "fetch", fake_fetch)
|
|
context = RunContext(is_cancelled=lambda: killed)
|
|
|
|
with pytest.raises(Cancelled):
|
|
pull.run_batch(FakeConfig(), store, targets=["greenhouse:acme", "greenhouse:beta"],
|
|
context=context)
|
|
|
|
assert fetched == ["greenhouse:acme"]
|
|
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"}
|
|
|
|
|
|
def test_close_boards_closes_what_a_completed_board_no_longer_lists(
|
|
store: Store) -> None:
|
|
sub = store.add_subscription("ats", "greenhouse:acme", label="Acme",
|
|
company="Acme", board="greenhouse")
|
|
other = store.add_subscription("ats", "greenhouse:other", label="Other",
|
|
company="Other", board="greenhouse")
|
|
|
|
store.upsert_role(_item("greenhouse", "acme", "1"), sub)
|
|
store.upsert_role(_item("greenhouse", "acme", "2"), sub)
|
|
store.upsert_role(_item("greenhouse", "other", "9", company="Other"), other)
|
|
|
|
# acme answered this run and now lists only "1"; other's board failed
|
|
# (or its batch never got to it) and stamped nothing.
|
|
store.state["ats:ok:greenhouse:acme"] = "run-1"
|
|
store.state["ats:complete:greenhouse:acme"] = "1"
|
|
store.state["ats:ids:greenhouse:acme"] = '["greenhouse:acme:1"]'
|
|
|
|
class FakeTally:
|
|
results = ({"roles": 1, "run": "run-1"},)
|
|
|
|
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
|
|
|
assert result == {"boards": 1, "searches": 0, "unanswered": 1, "incomplete": 0,
|
|
"roles": 1, "closed": 1}
|
|
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1", "greenhouse:other:9"}
|
|
|
|
|
|
def test_close_boards_never_closes_a_board_stamped_by_an_earlier_run(
|
|
store: Store) -> None:
|
|
sub = store.add_subscription("ats", "greenhouse:acme", label="Acme",
|
|
company="Acme", board="greenhouse")
|
|
store.upsert_role(_item("greenhouse", "acme", "1"), sub)
|
|
|
|
# A stale stamp from a run that is not this one — e.g. this run's batch
|
|
# for this board never finished.
|
|
store.state["ats:ok:greenhouse:acme"] = "run-1"
|
|
store.state["ats:ids:greenhouse:acme"] = "[]"
|
|
|
|
class FakeTally:
|
|
results = ({"roles": 0, "run": "run-2"},)
|
|
|
|
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
|
|
|
assert result == {"boards": 0, "searches": 0, "unanswered": 1, "incomplete": 0,
|
|
"roles": 0, "closed": 0}
|
|
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"}
|
|
|
|
|
|
def test_the_prep_after_the_settle_rebuilds_the_copies(store: Store) -> None:
|
|
"""The prep that follows the settle folds: a board's repeats of one job
|
|
are one role, and when the kept one closes the next copy is kept in its
|
|
place."""
|
|
sub = store.add_subscription("ats", "greenhouse:acme", label="Acme",
|
|
company="Acme", board="greenhouse")
|
|
for job_id in ("1", "2", "3"):
|
|
store.upsert_role(_item("greenhouse", "acme", job_id), sub)
|
|
store.state["ats:ok:greenhouse:acme"] = "run-1"
|
|
store.state["ats:complete:greenhouse:acme"] = "1"
|
|
store.state["ats:ids:greenhouse:acme"] = json.dumps(
|
|
["greenhouse:acme:1", "greenhouse:acme:2", "greenhouse:acme:3"])
|
|
|
|
class FakeTally:
|
|
results = ({"roles": 3, "run": "run-1"},)
|
|
|
|
pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
|
assert prep.run(store).copies == 2
|
|
[kept] = store.roles_page()
|
|
assert kept.copies == 2
|
|
|
|
gone = store.role(kept.uid).source_id # type: ignore[union-attr]
|
|
store.state["ats:ids:greenhouse:acme"] = json.dumps(
|
|
[f"greenhouse:acme:{n}" for n in "123" if f"greenhouse:acme:{n}" != gone])
|
|
result = pull.close_boards(FakeConfig(), store, FakeTally()) # type: ignore[arg-type]
|
|
assert (result["closed"], prep.run(store).copies) == (1, 1)
|
|
[after] = store.roles_page()
|
|
assert after.uid != kept.uid and after.copies == 1
|
|
|
|
|
|
# ---------------------------------------------- whole pulls, fetch stubbed
|
|
|
|
def _row(job_id: str) -> dict[str, object]:
|
|
return {"source_id": job_id, "title": f"Engineer {job_id}", "location": "Berlin",
|
|
"url": f"https://x.example/{job_id}", "posted": None, "department": "",
|
|
"description": "", "remote": None, "salary_text": ""}
|
|
|
|
|
|
def _answer(monkeypatch: pytest.MonkeyPatch, answer: object, filtered: bool = False,
|
|
swept: bool = False) -> None:
|
|
"""Every board's listing is `answer`, `(rows, complete)`, or raises it;
|
|
with `filtered`, a read the board's filter narrowed, with `swept` one that
|
|
finished a sweep across pulls."""
|
|
def fake(*args: object, **kwargs: object) -> _runner.Listing:
|
|
if isinstance(answer, Exception):
|
|
raise answer
|
|
rows, complete = answer # type: ignore[misc]
|
|
return _runner.Listing(rows, complete, [r["source_id"] for r in rows], filtered,
|
|
swept=swept)
|
|
|
|
monkeypatch.setattr(_runner, "read_listing", fake)
|
|
|
|
|
|
def _pull(store: Store) -> dict[str, object]:
|
|
results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type]
|
|
for batch in pull._boards(FakeConfig(), store)] # type: ignore[arg-type]
|
|
return pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
|
|
|
|
|
|
def _open(store: Store) -> set[str]:
|
|
return {r.source_id for r in store.roles()}
|
|
|
|
|
|
@pytest.fixture
|
|
def acme(store: Store) -> Store:
|
|
store.add_subscription("ats", "workable:acme", label="Acme", company="Acme",
|
|
board="workable")
|
|
return store
|
|
|
|
|
|
def test_an_incomplete_board_stores_what_it_read_and_closes_nothing(
|
|
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_answer(monkeypatch, ([_row("1"), _row("2")], True))
|
|
_pull(acme)
|
|
# A 429 on page two: page one's role is stored again, the other stays open.
|
|
_answer(monkeypatch, ([_row("1"), _row("3")], False))
|
|
result = _pull(acme)
|
|
assert _open(acme) == {"workable:acme:1", "workable:acme:2", "workable:acme:3"}
|
|
assert (result["boards"], result["incomplete"], result["closed"]) == (1, 1, 0)
|
|
|
|
|
|
def _aged(store: Store, source_id: str) -> None:
|
|
"""Make a role look unlisted for longer than `STALE_DAYS`."""
|
|
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat(timespec="seconds")
|
|
with store.transaction() as conn:
|
|
conn.execute("UPDATE roles SET last_seen = ? WHERE source_id = ?", (old, source_id))
|
|
|
|
|
|
def test_an_incomplete_read_closes_what_no_read_has_listed_for_stale_days(
|
|
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""A board past its ceiling or page cap is never read complete; a role
|
|
it has not listed for three weeks still closes (ADR 1.3)."""
|
|
_answer(monkeypatch, ([_row("1"), _row("2"), _row("3")], True))
|
|
_pull(acme)
|
|
_aged(acme, "workable:acme:3")
|
|
_answer(monkeypatch, ([_row("1")], False))
|
|
result = _pull(acme)
|
|
# 2 was listed two pulls ago and stays; 3 has been unlisted for 22 days.
|
|
assert _open(acme) == {"workable:acme:1", "workable:acme:2"}
|
|
assert (result["incomplete"], result["closed"]) == (1, 1)
|
|
|
|
|
|
def test_a_filtered_read_closes_a_spared_role_once_it_is_stale(
|
|
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""A filtered read spares a role acted on from its list; after three
|
|
weeks unlisted the role closes by age like any other."""
|
|
_answer(monkeypatch, ([_row("1"), _row("2")], True))
|
|
_pull(acme)
|
|
uid = {r.source_id: r.uid for r in acme.roles()}
|
|
agent = acme.save_agent("Hunt")
|
|
acme.set_vote(agent.id, uid["workable:acme:2"], 1)
|
|
_answer(monkeypatch, ([_row("1")], True), filtered=True)
|
|
assert _pull(acme)["closed"] == 0
|
|
assert _open(acme) == {"workable:acme:1", "workable:acme:2"}
|
|
_aged(acme, "workable:acme:2")
|
|
assert _pull(acme)["closed"] == 1
|
|
assert _open(acme) == {"workable:acme:1"}
|
|
|
|
|
|
def test_a_failing_board_keeps_its_roles_before_stale_days(
|
|
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_answer(monkeypatch, ([_row("1")], True))
|
|
_pull(acme)
|
|
_answer(monkeypatch, ValueError("gone"))
|
|
result = _pull(acme)
|
|
assert _open(acme) == {"workable:acme:1"}
|
|
assert (result["unanswered"], result["closed"]) == (1, 0)
|
|
since, reason = pull.failing_boards(acme)["workable:acme"]
|
|
assert since and reason == "ValueError: gone"
|
|
|
|
|
|
def test_a_board_failing_for_stale_days_closes_and_its_answer_reopens(
|
|
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_answer(monkeypatch, ([_row("1"), _row("2")], True))
|
|
_pull(acme)
|
|
_answer(monkeypatch, ValueError("404 for good"))
|
|
_pull(acme)
|
|
old = (datetime.now(UTC) - timedelta(days=pull.STALE_DAYS + 1)).isoformat(
|
|
timespec="seconds")
|
|
acme.state[f"{pull.FAILING}workable:acme"] = json.dumps(
|
|
{"since": old, "reason": "404 Not Found"})
|
|
|
|
result = _pull(acme)
|
|
assert _open(acme) == set()
|
|
assert result["closed"] == 2
|
|
# Still failing, and still the date the failures began.
|
|
assert pull.failing_boards(acme)["workable:acme"][0] == old
|
|
|
|
_answer(monkeypatch, ([_row("2")], True))
|
|
_pull(acme)
|
|
assert _open(acme) == {"workable:acme:2"}
|
|
assert pull.failing_boards(acme) == {}
|
|
|
|
|
|
def test_a_filtered_read_closes_what_it_left_out_except_roles_acted_on(
|
|
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_answer(monkeypatch, ([_row(i) for i in "1234"], True))
|
|
_pull(acme)
|
|
uid = {r.source_id: r.uid for r in acme.roles()}
|
|
agent = acme.save_agent("Hunt")
|
|
acme.set_vote(agent.id, uid["workable:acme:2"], 0)
|
|
acme.save_application(Application(role_uid=uid["workable:acme:3"], role_title="Engineer"))
|
|
|
|
_answer(monkeypatch, ([_row("1")], True), filtered=True)
|
|
result = _pull(acme)
|
|
assert _open(acme) == {"workable:acme:1", "workable:acme:2", "workable:acme:3"}
|
|
assert result["closed"] == 1
|
|
assert acme.state["ats:filtered:workable:acme"] == "1"
|
|
|
|
# Read without the filter again, the board closes as before.
|
|
_answer(monkeypatch, ([_row("1")], True))
|
|
_pull(acme)
|
|
assert _open(acme) == {"workable:acme:1"}
|
|
|
|
|
|
def test_a_swept_read_closes_only_what_two_sweeps_in_a_row_missed(
|
|
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_answer(monkeypatch, ([_row(i) for i in "123"], True))
|
|
_pull(acme)
|
|
|
|
_answer(monkeypatch, ([_row("1")], True), swept=True)
|
|
result = _pull(acme)
|
|
assert _open(acme) == {"workable:acme:1", "workable:acme:2", "workable:acme:3"}
|
|
assert result["closed"] == 0
|
|
assert json.loads(acme.state["ats:missing:workable:acme"]) == [
|
|
"workable:acme:2", "workable:acme:3"]
|
|
|
|
# 3 came back, 2 did not: 2 has two strikes, 3 starts over.
|
|
_answer(monkeypatch, ([_row("1"), _row("3")], True), swept=True)
|
|
result = _pull(acme)
|
|
assert _open(acme) == {"workable:acme:1", "workable:acme:3"}
|
|
assert result["closed"] == 1
|
|
assert json.loads(acme.state["ats:missing:workable:acme"]) == []
|
|
|
|
|
|
def test_an_incomplete_read_between_two_sweeps_keeps_the_strike(
|
|
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_answer(monkeypatch, ([_row(i) for i in "12"], True))
|
|
_pull(acme)
|
|
_answer(monkeypatch, ([_row("1")], True), swept=True)
|
|
_pull(acme)
|
|
_answer(monkeypatch, ([_row("1")], False))
|
|
_pull(acme)
|
|
_answer(monkeypatch, ValueError("down"))
|
|
_pull(acme)
|
|
assert json.loads(acme.state["ats:missing:workable:acme"]) == ["workable:acme:2"]
|
|
|
|
_answer(monkeypatch, ([_row("1")], True), swept=True)
|
|
_pull(acme)
|
|
assert _open(acme) == {"workable:acme:1"}
|
|
|
|
|
|
def test_a_filtered_swept_read_keeps_the_roles_acted_on(
|
|
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_answer(monkeypatch, ([_row(i) for i in "1234"], True))
|
|
_pull(acme)
|
|
uid = {r.source_id: r.uid for r in acme.roles()}
|
|
agent = acme.save_agent("Hunt")
|
|
acme.set_vote(agent.id, uid["workable:acme:2"], 0)
|
|
acme.save_application(Application(role_uid=uid["workable:acme:3"], role_title="Engineer"))
|
|
|
|
_answer(monkeypatch, ([_row("1")], True), filtered=True, swept=True)
|
|
assert _pull(acme)["closed"] == 0
|
|
assert _open(acme) == {f"workable:acme:{i}" for i in "1234"}
|
|
|
|
result = _pull(acme)
|
|
assert result["closed"] == 1
|
|
assert _open(acme) == {"workable:acme:1", "workable:acme:2", "workable:acme:3"}
|
|
assert json.loads(acme.state["ats:missing:workable:acme"]) == [
|
|
"workable:acme:2", "workable:acme:3"]
|
|
|
|
|
|
def test_a_narrowed_swept_read_closes_nothing_and_keeps_the_record(
|
|
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_answer(monkeypatch, ([_row(i) for i in "12"], True))
|
|
_pull(acme)
|
|
_answer(monkeypatch, ([_row("1")], True), swept=True)
|
|
_pull(acme)
|
|
|
|
def fake(*args: object, **kwargs: object) -> _runner.Listing:
|
|
return _runner.Listing([_row("1")], True, ["1"], swept=True, narrowed=True)
|
|
|
|
monkeypatch.setattr(_runner, "read_listing", fake)
|
|
assert _pull(acme)["closed"] == 0
|
|
assert json.loads(acme.state["ats:missing:workable:acme"]) == ["workable:acme:2"]
|
|
|
|
|
|
def test_a_board_subscriptions_params_reach_the_runner_as_its_ask(
|
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
store.add_subscription("ats", "workable:acme", label="Acme", company="Acme",
|
|
board="workable", params={"countries": ["dk"], "remote": True})
|
|
asked: list[object] = []
|
|
|
|
def fake(module: object, slug: str, get: object, ask: object, **kwargs: object
|
|
) -> _runner.Listing:
|
|
asked.append(ask)
|
|
return _runner.Listing([_row("1")], True, ["1"], _runner.filtered(ask)) # type: ignore[arg-type]
|
|
|
|
monkeypatch.setattr(_runner, "read_listing", fake)
|
|
_pull(store)
|
|
[ask] = asked
|
|
declared = readers.HOSTS["workable"].READS.filters.declared()
|
|
assert ask.countries == (("DK",) if "country" in declared else ()) # type: ignore[attr-defined]
|
|
|
|
|
|
# ------------------------------------------------------ one pull at a time
|
|
|
|
def _ago(**delta: float) -> str:
|
|
return (datetime.now(UTC) - timedelta(**delta)).isoformat(timespec="microseconds")
|
|
|
|
|
|
def test_a_pull_during_a_running_one_is_refused_and_queues_nothing(acme: Store) -> None:
|
|
first = pull.claim_run(acme)
|
|
assert first and pull.running(acme) == first
|
|
|
|
asked: list[object] = []
|
|
said: list[object] = []
|
|
context = RunContext(on_task=lambda task: asked.append(task) or True,
|
|
on_notice=said.append)
|
|
result = cascade.run_pull(FakeConfig(), acme, context=context)
|
|
|
|
assert result == {"skipped": pull.ALREADY_RUNNING, "running": first}
|
|
assert asked == []
|
|
assert [n.code for n in said] == ["pull.running"] # type: ignore[attr-defined]
|
|
# A chain started without the job's claim (a test, a script) finds
|
|
# nothing to queue either, and the running run's marker stands.
|
|
assert pull._boards(FakeConfig(), acme) == [] # type: ignore[arg-type]
|
|
assert acme.state[pull.RUN] == first
|
|
|
|
|
|
def test_a_pull_starts_once_the_last_one_settled(acme: Store,
|
|
monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_answer(monkeypatch, ([_row("1")], True))
|
|
_pull(acme)
|
|
assert pull.running(acme) == ""
|
|
|
|
asked: list[object] = []
|
|
context = RunContext(on_task=lambda task: asked.append(task) or True)
|
|
result = cascade.run_pull(FakeConfig(), acme, context=context)
|
|
assert result["run"] == acme.state[pull.RUN] and len(asked) == 1
|
|
# The chain's finder reads under the job's claim instead of claiming again.
|
|
[batch] = asked[0].finder(FakeConfig(), acme) # type: ignore[attr-defined]
|
|
assert batch["run"] == result["run"]
|
|
|
|
|
|
def test_starters_claiming_at_once_give_one_run(tmp_path: Path) -> None:
|
|
"""Eight starters, each on its own connection to one file, as two
|
|
processes would be, released together: one claim wins."""
|
|
Store(tmp_path / "shared.db")
|
|
stores = [Store(tmp_path / "shared.db") for _ in range(8)]
|
|
barrier = threading.Barrier(len(stores))
|
|
|
|
def claim(store: Store) -> str:
|
|
barrier.wait()
|
|
return pull.claim_run(store)
|
|
|
|
with ThreadPoolExecutor(len(stores)) as pool:
|
|
claims = list(pool.map(claim, stores))
|
|
assert sum(bool(c) for c in claims) == 1
|
|
|
|
|
|
def test_an_abandoned_run_is_taken_over_after_the_threshold(acme: Store) -> None:
|
|
hours = pull.ABANDONED.total_seconds() / 3600
|
|
acme.state[pull.RUN] = _ago(hours=hours / 2)
|
|
assert pull.claim_run(acme) == ""
|
|
acme.state[pull.RUN] = _ago(hours=hours + 0.1)
|
|
taken = pull.claim_run(acme)
|
|
assert taken and acme.state[pull.RUN] == taken
|
|
|
|
|
|
def test_a_batch_stamps_with_the_marker_it_was_handed(
|
|
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_answer(monkeypatch, ([_row("1")], True))
|
|
[batch] = pull._boards(FakeConfig(), acme) # type: ignore[arg-type]
|
|
acme.state[pull.RUN] = "a later run"
|
|
pull.run_batch(FakeConfig(), acme, **batch) # type: ignore[arg-type]
|
|
assert acme.state["ats:ok:workable:acme"] == batch["run"]
|
|
|
|
|
|
def test_run_a_settle_does_not_close_on_run_b_ids(
|
|
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
"""Run A read the board in full; run B, which took over, read it again and
|
|
listed less before A's settle ran. A's settle compares with A's marker, so
|
|
B's stamp and B's ids close nothing, and A's settle leaves B's claim held;
|
|
B's own settle is the one that closes."""
|
|
_answer(monkeypatch, ([_row("1"), _row("2")], True))
|
|
[a] = pull._boards(FakeConfig(), acme) # type: ignore[arg-type]
|
|
a_result = pull.run_batch(FakeConfig(), acme, **a) # type: ignore[arg-type]
|
|
|
|
acme.state[pull.RUN] = _ago(hours=2) # A looks abandoned
|
|
_answer(monkeypatch, ([_row("1")], True))
|
|
[b] = pull._boards(FakeConfig(), acme) # type: ignore[arg-type]
|
|
assert b["run"] != a["run"]
|
|
b_result = pull.run_batch(FakeConfig(), acme, **b) # type: ignore[arg-type]
|
|
|
|
result = pull.close_boards(FakeConfig(), acme, Tally(results=(a_result,))) # type: ignore[arg-type]
|
|
assert (result["unanswered"], result["closed"]) == (1, 0)
|
|
assert _open(acme) == {"workable:acme:1", "workable:acme:2"}
|
|
assert pull.running(acme) == b["run"]
|
|
|
|
result = pull.close_boards(FakeConfig(), acme, Tally(results=(b_result,))) # type: ignore[arg-type]
|
|
assert result["closed"] == 1
|
|
assert _open(acme) == {"workable:acme:1"}
|
|
assert pull.running(acme) == ""
|
|
|
|
|
|
def test_the_settle_reads_its_marker_off_its_batches_payloads(acme: Store) -> None:
|
|
"""A batch that raised has no result, but its payload still names the run."""
|
|
run = pull.claim_run(acme)
|
|
acme.state["ats:ok:workable:acme"] = run
|
|
acme.state["ats:complete:workable:acme"] = "1"
|
|
acme.state["ats:ids:workable:acme"] = "[]"
|
|
failed = type("Job", (), {"payload": {"targets": ["workable:acme"], "run": run},
|
|
"result": None, "state": "failed"})()
|
|
context = RunContext(waited_for=(failed,))
|
|
result = pull.close_boards(FakeConfig(), acme, Tally(), context=context) # type: ignore[arg-type]
|
|
assert result["boards"] == 1 and pull.running(acme) == ""
|
|
|
|
|
|
# -------------------------------------------------------- losses, guarded
|
|
|
|
def _days_ago(days: float) -> str:
|
|
return (datetime.now(UTC) - timedelta(days=days)).isoformat(timespec="seconds")
|
|
|
|
|
|
def _set(store: Store, column: str, value: str, *source_ids: str) -> None:
|
|
with store.transaction() as conn:
|
|
conn.executemany(f"UPDATE roles SET {column} = ? WHERE source_id = ?",
|
|
[(value, i) for i in source_ids])
|
|
|
|
|
|
@pytest.fixture
|
|
def big(store: Store) -> Store:
|
|
store.add_subscription("ats", "greenhouse:big", label="Big", company="Big",
|
|
board="greenhouse")
|
|
return store
|
|
|
|
|
|
def test_a_whole_read_of_nothing_after_forty_keeps_the_roles_and_fails(
|
|
big: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_answer(monkeypatch, ([_row(str(n)) for n in range(40)], True))
|
|
_pull(big)
|
|
_answer(monkeypatch, ([], True))
|
|
result = _pull(big)
|
|
assert len(_open(big)) == 40
|
|
assert (result["unanswered"], result["closed"]) == (1, 0)
|
|
assert "under 10% of the 40" in pull.failing_boards(big)["greenhouse:big"][1]
|
|
|
|
|
|
def test_a_whole_read_of_nothing_after_five_closes_the_board(
|
|
big: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_answer(monkeypatch, ([_row(str(n)) for n in range(5)], True))
|
|
_pull(big)
|
|
_answer(monkeypatch, ([], True))
|
|
assert _pull(big)["closed"] == 5
|
|
assert _open(big) == set()
|
|
|
|
|
|
def test_a_suspect_answer_is_accepted_after_stale_days(
|
|
big: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_answer(monkeypatch, ([_row(str(n)) for n in range(40)], True))
|
|
_pull(big)
|
|
_answer(monkeypatch, ([_row("1")], True))
|
|
_pull(big)
|
|
assert len(_open(big)) == 40
|
|
record = json.loads(big.state[f"{pull.FAILING}greenhouse:big"])
|
|
big.state[f"{pull.FAILING}greenhouse:big"] = json.dumps(
|
|
record | {"since": _days_ago(pull.STALE_DAYS + 1)})
|
|
result = _pull(big)
|
|
assert _open(big) == {"greenhouse:big:1"}
|
|
assert (result["unanswered"], result["closed"]) == (0, 39)
|
|
assert pull.failing_boards(big) == {}
|
|
assert json.loads(big.state["ats:count:greenhouse:big"])["count"] == 1
|
|
|
|
|
|
def _remote_host(monkeypatch: pytest.MonkeyPatch, remote: str) -> list[str]:
|
|
"""A whole-listing host `fake` that filters by country and, per `remote`,
|
|
on remote; its feed lists the ids in the returned list."""
|
|
listed: list[str] = []
|
|
module: object = types.ModuleType("fake")
|
|
module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", () # type: ignore[attr-defined]
|
|
module.slug_ok = lambda slug: slug.isalnum() # type: ignore[attr-defined]
|
|
module.allowed = lambda slug: lambda h: h == "jobs.fake.example" # type: ignore[attr-defined]
|
|
module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}" # type: ignore[attr-defined]
|
|
module.READS = Reads( # type: ignore[attr-defined]
|
|
scope="board", listing="whole", text="listing", complete="whole", places="one",
|
|
robots="test", filters=Filters(country="many", remote=remote)) # type: ignore[arg-type]
|
|
module.feed_url = lambda slug, ask: f"https://jobs.fake.example/{slug}" # type: ignore[attr-defined]
|
|
module.parse = lambda body, slug: [_row(i) for i in json.loads(body)] # type: ignore[attr-defined]
|
|
monkeypatch.setitem(readers.HOSTS, "fake", module)
|
|
monkeypatch.setattr(ats, "_get", lambda client, url, host, slug, **kw: json.dumps(
|
|
listed).encode())
|
|
return listed
|
|
|
|
|
|
def test_a_filtered_read_that_could_not_ask_remote_keeps_remote_roles_until_stale(
|
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
listed = _remote_host(monkeypatch, remote="none")
|
|
store.add_subscription("ats", "fake:acme", label="Acme", company="Acme", board="fake")
|
|
reach = {"fake:acme": {"countries": ["DK"], "remote": True}}
|
|
|
|
def pull_once() -> dict[str, object]:
|
|
run = pull.claim_run(store)
|
|
result = pull.run_batch(FakeConfig(), store, targets=["fake:acme"], run=run, # type: ignore[arg-type]
|
|
reach=reach)
|
|
return pull.close_boards(FakeConfig(), store, Tally(results=(result,))) # type: ignore[arg-type]
|
|
|
|
listed[:] = ["1", "2", "3", "4"]
|
|
pull_once()
|
|
_set(store, "remote", "remote", "fake:acme:3", "fake:acme:4")
|
|
listed[:] = ["1"]
|
|
assert pull_once()["closed"] == 1
|
|
assert _open(store) == {"fake:acme:1", "fake:acme:3", "fake:acme:4"}
|
|
assert store.state["ats:remote_unasked:fake:acme"] == "1"
|
|
|
|
_set(store, "last_seen", _days_ago(pull.STALE_DAYS + 1), "fake:acme:4")
|
|
assert pull_once()["closed"] == 1
|
|
assert _open(store) == {"fake:acme:1", "fake:acme:3"}
|
|
|
|
|
|
def test_a_narrowed_read_closes_what_it_has_not_listed_for_stale_days(
|
|
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_answer(monkeypatch, ([_row(i) for i in "123"], True))
|
|
_pull(acme)
|
|
|
|
def fake(*args: object, **kwargs: object) -> _runner.Listing:
|
|
return _runner.Listing([_row("1")], True, ["1"], narrowed=True)
|
|
|
|
monkeypatch.setattr(_runner, "read_listing", fake)
|
|
assert _pull(acme)["closed"] == 0
|
|
_set(acme, "last_seen", _days_ago(pull.STALE_DAYS + 1), "workable:acme:3")
|
|
assert _pull(acme)["closed"] == 1
|
|
assert _open(acme) == {"workable:acme:1", "workable:acme:2"}
|
|
|
|
|
|
def test_a_target_that_raises_after_its_read_is_recorded_and_the_batch_goes_on(
|
|
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
targets = [f"greenhouse:{slug}" for slug in ("a", "b", "c")]
|
|
for target in targets:
|
|
slug = target.partition(":")[2]
|
|
store.add_subscription("ats", target, label=slug, company=slug, board="greenhouse")
|
|
|
|
def fake_fetch(self: ATSSource) -> object:
|
|
[(target, _)] = self.targets()
|
|
yield _item("greenhouse", target.partition(":")[2], "1")
|
|
self.state[f"ats:ok:{target}"] = self.options["run"]
|
|
|
|
real = Store.upsert_role
|
|
|
|
def upsert(self: Store, item: Item, sub: object = None) -> bool:
|
|
if item.source_id.startswith("greenhouse:b:"):
|
|
raise ValueError("a row the store refuses")
|
|
return real(self, item, sub) # type: ignore[arg-type]
|
|
|
|
monkeypatch.setattr(ATSSource, "fetch", fake_fetch)
|
|
monkeypatch.setattr(Store, "upsert_role", upsert)
|
|
result = pull.run_batch(FakeConfig(), store, targets=targets, run="r") # type: ignore[arg-type]
|
|
|
|
assert result["roles"] == 2
|
|
assert _open(store) == {"greenhouse:a:1", "greenhouse:c:1"}
|
|
assert "ats:ok:greenhouse:b" not in store.state
|
|
since, reason = pull.failing_boards(store)["greenhouse:b"]
|
|
assert since and reason == "ValueError: a row the store refuses"
|
|
assert set(pull.failing_boards(store)) == {"greenhouse:b"}
|
|
|
|
|
|
def test_a_board_whose_rows_do_not_store_keeps_its_failing_record(
|
|
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
since = _days_ago(2)
|
|
acme.state[f"{pull.FAILING}workable:acme"] = json.dumps(
|
|
{"since": since, "reason": "404 Not Found"})
|
|
_answer(monkeypatch, ([_row("1")], True))
|
|
|
|
def refuse(self: Store, item: Item, sub: object = None) -> bool:
|
|
raise ValueError("no")
|
|
|
|
monkeypatch.setattr(Store, "upsert_role", refuse)
|
|
_pull(acme)
|
|
assert pull.failing_boards(acme)["workable:acme"] == (since, "ValueError: no")
|
|
assert "ats:ok:workable:acme" not in acme.state
|
|
|
|
|
|
# ------------------------------------------------------- the settle speaks
|
|
|
|
def test_a_settle_names_the_unanswered_board_and_the_one_that_closed_many(
|
|
store: Store) -> None:
|
|
sub = store.add_subscription("ats", "greenhouse:big", label="Big", company="Big",
|
|
board="greenhouse")
|
|
store.add_subscription("ats", "greenhouse:gone", label="Gone", company="Gone",
|
|
board="greenhouse")
|
|
for n in range(80):
|
|
store.upsert_role(_item("greenhouse", "big", str(n), company="Big"), sub)
|
|
run = pull.claim_run(store)
|
|
store.state["ats:ok:greenhouse:big"] = run
|
|
store.state["ats:complete:greenhouse:big"] = "1"
|
|
store.state["ats:ids:greenhouse:big"] = json.dumps(
|
|
[f"greenhouse:big:{n}" for n in range(20)])
|
|
store.state[f"{pull.FAILING}greenhouse:gone"] = json.dumps(
|
|
{"since": _days_ago(1), "reason": "404 Not Found"})
|
|
|
|
said: list[object] = []
|
|
result = pull.close_boards(FakeConfig(), store, Tally(results=({"roles": 20, "run": run},)), # type: ignore[arg-type]
|
|
context=RunContext(on_notice=said.append))
|
|
assert (result["unanswered"], result["closed"]) == (1, 60)
|
|
[notice] = said
|
|
assert notice.code == "pull.settled" # type: ignore[attr-defined]
|
|
assert "greenhouse:gone (404 Not Found)" in notice.text # type: ignore[attr-defined]
|
|
assert "greenhouse:big 60 of 80" in notice.text # type: ignore[attr-defined]
|
|
assert "closed 60 roles" in notice.text # type: ignore[attr-defined]
|
|
|
|
|
|
def test_a_clean_settle_says_nothing(acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
_answer(monkeypatch, ([_row(i) for i in "12"], True))
|
|
said: list[object] = []
|
|
results = [pull.run_batch(FakeConfig(), acme, **batch) # type: ignore[arg-type]
|
|
for batch in pull._boards(FakeConfig(), acme)] # type: ignore[arg-type]
|
|
pull.close_boards(FakeConfig(), acme, Tally(results=tuple(results)), # type: ignore[arg-type]
|
|
context=RunContext(on_notice=said.append))
|
|
assert said == []
|
|
|
|
|
|
def test_a_finder_that_raises_releases_the_pull_claim(
|
|
acme: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
|
def broken(store: Store, subs: object) -> dict[str, object]:
|
|
raise RuntimeError("reach broke")
|
|
|
|
monkeypatch.setattr(pull, "reaches", broken)
|
|
# The worker runs the chain's finder once the job is done; here at once.
|
|
context = RunContext(on_task=lambda task: task.finder(FakeConfig(), acme))
|
|
with pytest.raises(RuntimeError, match="reach broke"):
|
|
cascade.run_pull(FakeConfig(), acme, context=context)
|
|
assert pull.running(acme) == ""
|
|
|
|
# And the job itself lets go when the chain cannot be built.
|
|
def no_chain(run: str) -> object:
|
|
raise RuntimeError("no chain")
|
|
|
|
monkeypatch.setattr(cascade, "pull_chain", no_chain)
|
|
with pytest.raises(RuntimeError, match="no chain"):
|
|
cascade.run_pull(FakeConfig(), acme, context=RunContext(on_task=lambda task: True))
|
|
assert pull.running(acme) == ""
|