Size a board once with board_size, before a pull reads it with no size, and name what a site's ids leave out

set_board sizes a board from page 1's total, or reads to the end on a host that states none, instead of reading it whole up to page 100. A pull no longer rewrites ats:size; a board with none is handed the agents' reach marked unsized and is sized in its batch first, or by its whole read on a host that cannot narrow the reach. reach.sizes drops the open-role fallback. The Companies line names countries a Workday or Oracle site does not list and a missing remote option.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 13:43:52 +02:00
1 parent a19c528e3f
commit a0882e9659
8 files changed
+579 -67

No files matched your search

+343
View File
@@ -0,0 +1,343 @@
"""A board's size, set once: `ats.board_size` per kind of host, `set_board`
that keeps it, the pull that sizes a board
with none before it reads it, and the Companies line that names what a
site's own ids leave out. No network: `ats._get` or the runner's listing is
stubbed."""
from __future__ import annotations
import json
import sys
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, ClassVar
from urllib.parse import parse_qs, urlsplit
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from pipeline import Tally # noqa: E402
from app.jobs import pull, reach, sources # noqa: E402
from app.models import Gate # noqa: E402
from app.storage import Store # noqa: E402
from plugins import ats # noqa: E402
from plugins.readers import Ask, _runner, greenhouse, workday # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
BOSCH = "smartrecruiters:BoschGroup"
TODAY = datetime.now(UTC).date().isoformat()
class FakeConfig:
sources: ClassVar[dict[str, object]] = {}
searches: ClassVar[list[object]] = []
@pytest.fixture
def store(tmp_path: Path) -> Store:
return Store(tmp_path / "t.db")
@pytest.fixture(autouse=True)
def _no_pause(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
def _query(url: str) -> dict[str, list[str]]:
return parse_qs(urlsplit(url).query)
def _posting(job_id: str, country: str = "de") -> dict[str, Any]:
return {"id": job_id, "name": f"Engineer {job_id}",
"location": {"city": "Berlin", "country": country}}
class SmartRecruiters:
"""A SmartRecruiters board of `total` roles that answers each request
with one page of three and the stated total, or, per country, the roles
there. Remembers every URL asked."""
def __init__(self, total: int | None, *, down: bool = False) -> None:
self.total, self.down = total, down
self.asked: list[str] = []
def __call__(self, client: Any, url: str, host: str, slug: str, **kwargs: Any) -> bytes:
self.asked.append(url)
if self.down:
raise OSError("connection refused")
query = _query(url)
if int(query.get("offset", ["0"])[0]):
return json.dumps({"content": []}).encode()
country = query.get("country", [""])[0]
content = [_posting(f"{country}{n}", country or "de") for n in range(3)]
answer: dict[str, Any] = {"content": content}
if self.total is not None:
answer["totalFound"] = 3 if country else self.total
return json.dumps(answer).encode()
def _teamtailor(count: int) -> Any:
"""A Teamtailor site of `count` roles, served 50 a page by offset."""
asked: list[str] = []
def get(client: Any, url: str, host: str, slug: str, **kwargs: Any) -> bytes:
asked.append(url)
offset = int(_query(url)["offset"][0])
items = "".join(
f"<item><title>Role {n}</title><link>https://acme.teamtailor.com/jobs/{n}</link>"
f"<guid>{n}</guid></item>" for n in range(offset, min(offset + 50, count)))
return f"<rss><channel>{items}</channel></rss>".encode()
get.asked = asked # type: ignore[attr-defined]
return get
def _wolf(store: Store, countries: tuple[str, ...] = ("DK", "SE"), remote: bool = True) -> None:
gates = [Gate(field="country", values=list(countries), mode="any")]
if remote:
gates.append(Gate(field="remote", values=["remote"], mode="any"))
store.save_agent("Wolf", gates=gates)
def _board(store: Store, target: str, company: str, size: int | None = None,
params: dict[str, object] | None = None) -> None:
store.add_subscription("ats", target, label=company, company=company,
board=target.partition(":")[0], params=params or {})
store.add_company(company)
if size is not None:
store.state[f"ats:size:{target}"] = str(size)
def _sub(store: Store, target: str) -> Any:
return next(s for s in store.subscriptions(source="ats") if s.target == target)
def _pull(store: Store) -> list[dict[str, Any]]:
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
results = [pull.run_batch(FakeConfig(), store, **batch)
for batch in batches]
pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
return batches
# ------------------------------------------------------------- board_size
def test_a_host_that_states_a_total_is_sized_by_page_one(
monkeypatch: pytest.MonkeyPatch) -> None:
board = SmartRecruiters(999)
monkeypatch.setattr(ats, "_get", board)
assert ats.board_size(BOSCH) == 999
# One unfiltered request: no country, no terms.
[url] = board.asked
assert "country" not in _query(url) and "q" not in _query(url)
def test_a_host_that_states_no_total_is_read_to_the_end(
monkeypatch: pytest.MonkeyPatch) -> None:
get = _teamtailor(120)
monkeypatch.setattr(ats, "_get", get)
assert ats.board_size("teamtailor:acme") == 120
assert [_query(u)["offset"][0] for u in get.asked] == ["0", "50", "100", "120"]
def test_a_stated_host_that_leaves_out_its_total_is_read_to_the_end(
monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(ats, "_get", SmartRecruiters(None))
assert ats.board_size(BOSCH) == 3
def test_a_whole_host_is_sized_by_its_one_answer(monkeypatch: pytest.MonkeyPatch) -> None:
body = (FIXTURES / "greenhouse.json").read_bytes()
monkeypatch.setattr(ats, "_get", lambda *args, **kwargs: body)
assert ats.board_size("greenhouse:acme") == len(greenhouse.parse(body, "acme"))
def test_board_size_refuses_an_unknown_board_and_raises_on_no_answer(
monkeypatch: pytest.MonkeyPatch) -> None:
with pytest.raises(ValueError, match="host:slug"):
ats.board_size("taleo:acme")
monkeypatch.setattr(ats, "_get", SmartRecruiters(5, down=True))
with pytest.raises(OSError):
ats.board_size(BOSCH)
# -------------------------------------------------------------- set_board
def test_set_board_keeps_the_size_and_the_day_without_reading_the_board_whole(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
board = SmartRecruiters(4830)
monkeypatch.setattr(ats, "_get", board)
sub, found, _closed = sources.set_board(store, "bosch", BOSCH, name="Bosch")
assert found == 4830 and len(board.asked) == 1
assert store.state[f"ats:size:{BOSCH}"] == "4830"
assert store.state[f"ats:sizechecked:{BOSCH}"] == TODAY
assert reach.plan(store).why(sub)[1] == "agents"
def test_the_seed_keeps_no_size_and_a_detached_board_forgets_it(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(ats, "_get", SmartRecruiters(4830, down=True))
sources.set_board(store, "bosch", BOSCH, name="Bosch", check=False)
assert f"ats:size:{BOSCH}" not in store.state
ats.keep_size(store.state, BOSCH, 4830)
sources.detach_board(store, "bosch")
assert f"ats:size:{BOSCH}" not in store.state
assert f"ats:sizechecked:{BOSCH}" not in store.state
# ------------------------------------------------------ the unsized reach
def test_a_board_with_no_size_is_handed_the_agents_reach_marked_unsized(store: Store) -> None:
_wolf(store)
_board(store, BOSCH, "Bosch")
_board(store, "workday:acme.wd3/Jobs", "Acme", params={"countries": ["DE"]})
_board(store, "smartrecruiters:tiny", "Tiny", size=12)
plan = reach.plan(store)
assert plan.for_subscription(_sub(store, BOSCH)) == {
"countries": ["DK", "SE"], "remote": True, "unsized": True, "small": reach.SMALL}
# A board with its own reach carries only the marks; a sized one none.
assert plan.for_subscription(_sub(store, "workday:acme.wd3/Jobs")) == {
"unsized": True, "small": reach.SMALL}
assert plan.for_subscription(_sub(store, "smartrecruiters:tiny")) is None
assert reach.line(plan, _sub(store, BOSCH)) == "size not known yet: sized on the next pull"
def test_a_big_board_with_no_size_is_sized_then_read_with_the_reach(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_wolf(store)
_board(store, BOSCH, "Bosch")
board = SmartRecruiters(4830)
monkeypatch.setattr(ats, "_get", board)
_pull(store)
assert store.state[f"ats:size:{BOSCH}"] == "4830"
assert store.state[f"ats:sizechecked:{BOSCH}"] == TODAY
# The first request sizes the board; the read then asks for DK and SE.
assert "country" not in _query(board.asked[0])
assert sorted(_query(u).get("country", [""])[0] for u in board.asked[1:]) == ["dk", "se"]
assert store.state[f"ats:filtered:{BOSCH}"] == "1"
# The next pull finds the size and does not ask for it again.
board.asked.clear()
_pull(store)
assert all("country" in _query(u) for u in board.asked)
def test_a_small_board_with_no_size_is_sized_then_read_whole(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_wolf(store)
_board(store, BOSCH, "Bosch")
board = SmartRecruiters(3)
monkeypatch.setattr(ats, "_get", board)
_pull(store)
assert store.state[f"ats:size:{BOSCH}"] == "3"
assert all("country" not in _query(u) for u in board.asked)
assert store.state[f"ats:filtered:{BOSCH}"] == ""
def test_a_board_that_does_not_answer_its_sizing_fails_as_any_read(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_wolf(store)
_board(store, BOSCH, "Bosch")
monkeypatch.setattr(ats, "_get", SmartRecruiters(4830, down=True))
_pull(store)
assert f"ats:size:{BOSCH}" not in store.state
assert "connection refused" in json.loads(store.state[f"ats:failing:{BOSCH}"])["reason"]
def test_a_board_read_whole_anyway_is_sized_by_that_read_once_it_is_complete(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_wolf(store)
_board(store, "greenhouse:acme", "Acme")
rows = [{"source_id": str(n), "title": f"Role {n}", "url": f"https://x.example/{n}",
"places": [], "description": ""} for n in range(4)]
answer = {"complete": False}
asked: list[Ask] = []
def fake(module: object, slug: str, get: object, ask: Ask, **kwargs: object
) -> _runner.Listing:
asked.append(ask)
return _runner.Listing(rows, answer["complete"], [r["source_id"] for r in rows]) # type: ignore[misc]
monkeypatch.setattr(_runner, "read_listing", fake)
monkeypatch.setattr(ats, "board_size", lambda *a, **k: pytest.fail("sized apart"))
_pull(store)
# Greenhouse cannot filter: one read, and an incomplete one sizes nothing.
assert asked == [Ask()]
assert "ats:size:greenhouse:acme" not in store.state
answer["complete"] = True
_pull(store)
assert store.state["ats:size:greenhouse:acme"] == "4"
assert store.state["ats:sizechecked:greenhouse:acme"] == TODAY
def test_a_pull_leaves_a_kept_size_alone(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_wolf(store)
_board(store, "greenhouse:acme", "Acme", size=250)
def fake(module: object, slug: str, get: object, ask: Ask, **kwargs: object
) -> _runner.Listing:
return _runner.Listing([], True, [])
monkeypatch.setattr(_runner, "read_listing", fake)
_pull(store)
assert store.state["ats:size:greenhouse:acme"] == "250"
# ---------------------------------------------------- the line and facets
WORKDAY = "workday:acme.wd3/Jobs"
@pytest.mark.parametrize("facets, line", [
(None, "follows agents: DK, SE, remote · all roles"),
({"country": {"DK": "c=dk", "SE": "c=se"}, "remote": {"remote": "r=1"}},
"follows agents: DK, SE, remote · all roles"),
({"country": {"DK": "c=dk"}},
"follows agents: DK · SE not on this site · no remote option · all roles"),
({"country": {"DK": "c=dk"}, "remote": {"remote": "r=1"}},
"follows agents: DK, remote · SE not on this site · all roles"),
({"country": {"DE": "c=de"}, "remote": {"remote": "r=1"}},
"follows agents: remote · DK, SE not on this site · all roles"),
({"country": {"DE": "c=de"}}, "site lists none of DK, SE, remote: nothing asked"),
], ids=["none kept", "all known", "one country, no remote", "one country and remote",
"remote only", "none"])
def test_the_line_names_what_a_site_does_not_know(
store: Store, facets: dict[str, dict[str, str]] | None, line: str) -> None:
_wolf(store)
_board(store, WORKDAY, "Acme", size=1000)
if facets is not None:
store.state[f"ats:facets:{WORKDAY}"] = json.dumps(
{"checked": "2026-09-30T06:00:00+00:00", "facets": facets})
plan = reach.plan(store)
sub = _sub(store, WORKDAY)
assert reach.line(plan, sub) == line
# What the line says is left to ask is what the runner asks.
ask = _runner.ask_for(workday, sub.params, plan.for_subscription(sub))
assert (_runner.series(workday, ask, facets) == []) == line.endswith("nothing asked")
def test_the_line_says_nothing_is_asked_in_the_plans_words(store: Store) -> None:
_wolf(store, remote=False)
_board(store, WORKDAY, "Acme", size=1000)
store.state[f"ats:facets:{WORKDAY}"] = json.dumps(
{"checked": "2026-09-30T06:00:00+00:00", "facets": {"country": {"DE": "c=de"}}})
assert reach.line(reach.plan(store), _sub(store, WORKDAY)) == \
"site lists none of DK, SE: nothing asked"
def test_a_boards_own_reach_is_named_the_same_and_remote_alone_is_read_whole(
store: Store) -> None:
_wolf(store)
_board(store, WORKDAY, "Acme", size=1000, params={"countries": ["DK", "NO"]})
_board(store, "workday:other.wd3/Jobs", "Other", size=1000, params={"remote": True})
for target in (WORKDAY, "workday:other.wd3/Jobs"):
store.state[f"ats:facets:{target}"] = json.dumps(
{"checked": "2026-09-30T06:00:00+00:00", "facets": {"country": {"DK": "c=dk"}}})
plan = reach.plan(store)
assert reach.line(plan, _sub(store, WORKDAY)) == \
"set here: DK · NO not on this site · all roles"
assert reach.line(plan, _sub(store, "workday:other.wd3/Jobs")) == \
"all places, no remote option on this site · all roles"
+4 -7
View File
@@ -127,8 +127,8 @@ def test_the_narrow_reach_takes_no_terms(store: Store) -> None:
@pytest.mark.parametrize("size, params", [
(299, {}), (None, {}), (5000, {"countries": ["DE"]}),
], ids=["small", "never read whole", "own reach"])
(299, {}), (5000, {"countries": ["DE"]}),
], ids=["small", "own reach"])
def test_small_unknown_overridden_and_switched_off_boards_take_no_terms(
store: Store, size: int | None, params: dict[str, object]) -> None:
_wolf(store, TERMS)
@@ -394,10 +394,10 @@ def test_a_term_read_writes_no_size_and_closes_nothing(
Tally(results=tuple(results)))
# No terms: Europe is too many countries for SmartRecruiters, so whole.
# A whole read writes no size either: the board keeps the one it has.
run()
assert [_query(u).get("q") for u in asked] == [None]
assert store.state["ats:size:smartrecruiters:bfl"] == "3"
store.state["ats:size:smartrecruiters:bfl"] = "1000"
assert store.state["ats:size:smartrecruiters:bfl"] == "1000"
open_before = {r.source_id for r in store.roles(limit=50)}
assert open_before == {f"smartrecruiters:bfl:{n}" for n in ("1", "2", "9")}
@@ -413,9 +413,6 @@ def test_a_term_read_writes_no_size_and_closes_nothing(
assert {r.source_id for r in store.roles(limit=50)} == open_before
assert settled["closed"] == 0
assert store.state["ats:size:smartrecruiters:bfl"] == "1000"
# A board sized by its open roles is never sized by a term read.
del store.state["ats:size:smartrecruiters:bfl"]
assert reach.sizes(store, store.subscriptions(source="ats")) == {}
# ------------------------------------------------------- storage and the API
+6 -6
View File
@@ -185,14 +185,14 @@ def test_small_unknown_and_overridden_boards_get_no_reach(
assert (found.params() is None) == (why != "agents")
def test_a_board_read_before_sizes_were_kept_is_sized_by_its_open_roles(store: Store) -> None:
def test_a_board_has_only_the_size_it_was_given(store: Store) -> None:
_board(store, "workday:acme.wd3/Jobs", "Acme", size=None)
[sub] = store.subscriptions(source="ats")
# A board read whole before sizes were kept is not sized by its open roles.
store.state["ats:ok:workday:acme.wd3/Jobs"] = "run-0"
assert reach.sizes(store, [sub]) == {"workday:acme.wd3/Jobs": 0}
# A filtered board's open roles are not its size.
store.state["ats:filtered:workday:acme.wd3/Jobs"] = "1"
assert reach.sizes(store, [sub]) == {}
store.state["ats:size:workday:acme.wd3/Jobs"] = "12"
assert reach.sizes(store, [sub]) == {"workday:acme.wd3/Jobs": 12}
def test_describe_is_short() -> None:
@@ -237,8 +237,8 @@ def test_the_pull_hands_each_board_its_reach_and_the_runner_narrows_it(
assert asked["BoschGroup"] == Ask(countries=("DK", "SE"))
assert asked["tiny"] == Ask()
assert store.state["ats:filtered:smartrecruiters:BoschGroup"] == "1"
# The small board, read whole, keeps its size; the filtered one keeps its last.
assert store.state["ats:size:smartrecruiters:tiny"] == "1"
# A pull rewrites no size: the small board read whole keeps its 12.
assert store.state["ats:size:smartrecruiters:tiny"] == "12"
assert store.state["ats:size:smartrecruiters:BoschGroup"] == "1000"
+5 -5
View File
@@ -29,24 +29,24 @@ def store(tmp_path: Path) -> Store:
class Boards:
"""A fake `fetch_board`: `listed` maps a target to how many roles it
lists; any other target does not answer. Remembers what was fetched."""
"""A fake `board_size`: `listed` maps a target to how many roles it
lists; any other target does not answer. Remembers what was sized."""
def __init__(self, **listed: int) -> None:
self.listed = {k.replace("_", ":", 1): v for k, v in listed.items()}
self.fetched: list[str] = []
def __call__(self, target: str) -> tuple[list[dict[str, Any]], bool]:
def __call__(self, target: str, **kwargs: Any) -> int:
self.fetched.append(target)
if target not in self.listed:
raise OSError("connection refused")
return [{"source_id": str(i)} for i in range(self.listed[target])], True
return self.listed[target]
@pytest.fixture
def boards(monkeypatch: pytest.MonkeyPatch) -> Boards:
fake = Boards(greenhouse_acme=3, ashby_acme=2, lever_quiet=0, lever_acme=1)
monkeypatch.setattr(ats, "fetch_board", fake)
monkeypatch.setattr(ats, "board_size", fake)
return fake
+7 -7
View File
@@ -603,7 +603,7 @@ def test_companies_lists_board_and_open_role_count(
"careers_opened_at": "", "failing_since": "", "failing_reason": "",
"from_search": False, "hand_reason": "", "by_hand": False,
"all_countries": False, "all_roles": False,
"reach_params": {}, "reach": "size not known yet: read whole"}
"reach_params": {}, "reach": "size not known yet: sized on the next pull"}
[other] = [c for c in res.json() if c["key"] == "other"]
assert (other["board"], other["subscription_id"], other["on"], other["ats"]) == (
"", None, None, "")
@@ -1371,7 +1371,7 @@ def test_a_board_is_followed_only_if_it_answers(
client: TestClient, store: Store, queue: Any, monkeypatch: pytest.MonkeyPatch) -> None:
import plugins.ats as ats
monkeypatch.setattr(ats, "fetch_board", lambda target: ([{"source_id": "1"}], True))
monkeypatch.setattr(ats, "board_size", lambda target, **kw: 1)
res = client.post("/api/boards", json={"board": "ashby:acme", "name": "Acme"})
assert res.status_code == 201
assert res.json()["found"] == 1 and res.json()["source"]["target"] == "ashby:acme"
@@ -1389,10 +1389,10 @@ def test_a_board_is_followed_only_if_it_answers(
bad = client.post("/api/boards", json={"board": "acme", "name": "Other"})
assert bad.status_code == 422 and "host:slug" in bad.json()["detail"]
def down(target: str) -> list[dict[str, str]]:
def down(target: str, **kw: object) -> int:
raise OSError("connection refused")
monkeypatch.setattr(ats, "fetch_board", down)
monkeypatch.setattr(ats, "board_size", down)
dead = client.post("/api/boards", json={"board": "ashby:other", "name": "Other"})
assert dead.status_code == 422 and "did not answer" in dead.json()["detail"]
assert store.company("other") is None
@@ -1402,7 +1402,7 @@ def test_a_companys_board_is_set_replaced_and_detached(
client: TestClient, store: Store, queue: Any, monkeypatch: pytest.MonkeyPatch) -> None:
import plugins.ats as ats
monkeypatch.setattr(ats, "fetch_board", lambda target: ([], True)) # between openings
monkeypatch.setattr(ats, "board_size", lambda target, **kw: 0) # between openings
store.add_company("Acme", key="acme", ats="teamtailor", ats_slug="acme")
res = client.post("/api/companies/acme/board", json={"board": "greenhouse:acme"})
@@ -1522,7 +1522,7 @@ def test_a_company_with_a_board_keeps_its_name_and_takes_the_other_edits(
client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
import plugins.ats as ats
monkeypatch.setattr(ats, "fetch_board", lambda target: ([], True))
monkeypatch.setattr(ats, "board_size", lambda target, **kw: 0)
store.add_company("Acme", key="acme")
assert client.post("/api/companies/acme/board", json={"board": "greenhouse:acme"}
).status_code == 200
@@ -1563,7 +1563,7 @@ def test_a_company_is_deleted_unless_it_is_read(
client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
import plugins.ats as ats
monkeypatch.setattr(ats, "fetch_board", lambda target: ([], True))
monkeypatch.setattr(ats, "board_size", lambda target, **kw: 0)
agent = store.save_agent("A")
store.add_company("Meta", key="meta", hand_reason="blocked")
store.set_affinity(agent.id, "meta", 1)