Size a board once with board_size, before a pull reads it with no size, and name what a site's ids leave out

set_board sizes a board from page 1's total, or reads to the end on a host that states none, instead of reading it whole up to page 100. A pull no longer rewrites ats:size; a board with none is handed the agents' reach marked unsized and is sized in its batch first, or by its whole read on a host that cannot narrow the reach. reach.sizes drops the open-role fallback. The Companies line names countries a Workday or Oracle site does not list and a missing remote option.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 13:43:52 +02:00
1 parent a19c528e3f
commit a0882e9659
8 files changed
+579 -67

No files matched your search

+113 -19
View File
@@ -21,8 +21,13 @@ on. Computed once per pull (`plan`) and handed to each board's batch as the
it skips the `any` gates; it does not widen what is fetched.
- **Board override**: a board whose subscription `params` name a country, a
place or remote gets no reach here; `ask_for` reads the params instead.
- **Small boards**: a board that listed under `SMALL` roles when last read
whole gets no filter; filtering it saves little and risks closing roles.
- **Small boards**: a board of under `SMALL` roles gets no filter;
filtering it saves little and risks closing roles. A board's size is set
once, when it is added or rechecked (`plugins/ats.py`, `board_size`), and
never by a pull. A board with no size yet is handed the reach it would
have as a big one, with `unsized` and `small`, and its batch sizes it
before it reads it, so a big new board is not read whole first; a board
whose reader cannot narrow that reach is sized by its whole read instead.
A reach of more countries than a `one`-filter host should be asked one by
one (the wide reach's Europe on SmartRecruiters) is dropped by `ask_for`,
@@ -43,14 +48,15 @@ board sends the roles that match one of them rather than every role:
changes nothing on the narrow reach today, which takes no terms.
- Only a reader that declares `query="one"` takes them, one series a term;
`ask_for` has the rule for terms times the reach's series.
- A read with terms is never the whole board: it sets no `ats:size`, and
the pull closes nothing against it (`app/jobs/pull.py`).
- A read with terms is never the whole board: the pull closes nothing
against it (`app/jobs/pull.py`).
- The search providers take none. Every saved search has a query of its
own, which terms would replace; a search is a query already.
"""
from __future__ import annotations
import json
from collections.abc import Iterable, Mapping
from dataclasses import dataclass, field
from typing import Any
@@ -65,9 +71,15 @@ from plugins.readers import _runner
#: whole.
SMALL = 300
#: `state` key of a board's size, `ats:size:<target>`: the roles its last
#: unfiltered read listed, written by `plugins/ats.py`.
#: `state` keys of a board's size, `ats:size:<target>`, the roles it listed
#: when it was sized, and of the day it was sized (ISO), both written by
#: `plugins/ats.py` (`keep_size`).
SIZE = "ats:size:"
SIZE_CHECKED = "ats:sizechecked:"
#: `state` key of the host ids a board's site states (`{"checked",
#: "facets"}`), kept by the runner (`_runner.facets_for`).
FACETS = "ats:facets:"
#: The params that make a board's own reach (see `_runner._PARAM_DIMENSION`).
OVERRIDE_KEYS = frozenset(("countries", "country", "places", "place", "remote"))
@@ -239,7 +251,9 @@ class Plan:
"""The reach of one pull: `base` for every board, `wide` for the boards
of the companies in `all_countries`, each board's size by target, the
fetch `terms` of the wide reads, which skip the companies in
`all_roles`, and `wide_names`, what the agents call the wide reach."""
`all_roles`, `wide_names`, what the agents call the wide reach, and
`facets`, the host ids the runner keeps for a board on a host that
filters by them, for the Companies line."""
base: Reach
wide: Reach = EVERYWHERE
@@ -248,6 +262,7 @@ class Plan:
terms: tuple[str, ...] = ()
all_roles: frozenset[str] = frozenset()
wide_names: str = ""
facets: dict[str, dict[str, dict[str, str]]] = field(default_factory=dict)
def why(self, sub: Subscription) -> tuple[Reach, str]:
"""The reach `sub` is read with, and why: `override`, `small`,
@@ -260,6 +275,10 @@ class Plan:
return EVERYWHERE, "unknown"
if size < SMALL:
return EVERYWHERE, "small"
return self._big(sub)
def _big(self, sub: Subscription) -> tuple[Reach, str]:
"""The reach of `sub` as a board of `SMALL` roles or more."""
if company_key(sub.company or sub.label) in self.all_countries:
return self.wide, "countries"
return self.base, "agents"
@@ -274,30 +293,51 @@ class Plan:
fetch terms, and unless its company is set to All roles, a board on
the wide reach carries them and `wide`, and a board on the agents'
reach carries them for `ask_for` to use if it drops the countries as
too many (see the module docstring)."""
too many (see the module docstring). A board with no size carries
`unsized` and `small`, and the reach of a big board unless its params
set their own, for its batch to size it first."""
found, why = self.why(sub)
if why == "unknown":
found, why = self._big(sub)
out = found.params()
if self.terms and why in ("countries", "agents") and self.narrows_roles(sub):
out = (out or {}) | {"terms": list(self.terms)}
if why == "countries":
out["wide"] = True
if sub.target not in self.sizes:
out = (out or {}) | {"unsized": True, "small": SMALL}
return out
def sizes(store: Store, subs: Iterable[Subscription]) -> dict[str, int]:
"""Each board's size: the roles its last unfiltered read listed
(`ats:size:<target>`), else, for a board read before that was kept and
never filtered, its open roles. A board never read has none."""
"""Each board's size as kept (`ats:size:<target>`); a board never sized
has none."""
state = store.state
counts = store.open_countries_by_subscription()
out: dict[str, int] = {}
for sub in subs:
kept = state.get(f"{SIZE}{sub.target}", "")
if kept.isdigit():
out[sub.target] = int(kept)
elif (state.get(f"ats:ok:{sub.target}") and not state.get(f"ats:filtered:{sub.target}")
and not state.get(f"ats:narrowed:{sub.target}")):
out[sub.target] = sum(counts.get(sub.id, {}).values())
return out
def site_facets(store: Store, subs: Iterable[Subscription]) -> dict[str, dict[str, dict[str, str]]]:
"""The host ids the runner keeps for each board on a host whose country
or remote filter is `ids` (`ats:facets:<target>`); a board with none kept
is left out."""
out: dict[str, dict[str, dict[str, str]]] = {}
for sub in subs:
module = readers.HOSTS.get(sub.target.partition(":")[0])
if module is None or not _runner.declared(module) \
or "ids" not in (module.READS.filters.country, module.READS.filters.remote):
continue
try:
kept = json.loads(store.state.get(f"{FACETS}{sub.target}") or "{}")
except (TypeError, ValueError):
continue
found = kept.get("facets") if isinstance(kept, dict) else None
if isinstance(found, dict):
out[sub.target] = found
return out
@@ -321,7 +361,8 @@ def plan(store: Store, subs: Iterable[Subscription] | None = None) -> Plan:
all_countries=frozenset(c.key for c in rows if c.all_countries),
terms=fetch_terms(agents),
all_roles=frozenset(c.key for c in rows if c.all_roles),
wide_names=wide_names(agents, gaz) if agents else "everywhere")
wide_names=wide_names(agents, gaz) if agents else "everywhere",
facets=site_facets(store, subs))
def reaches(store: Store, subs: Iterable[Subscription]) -> Mapping[str, dict[str, Any]]:
@@ -338,14 +379,58 @@ _WHOLE = {"small": f"under {SMALL} roles", "unknown": "size not known yet",
"agents": "an agent takes any place"}
def _codes(codes: Iterable[str]) -> list[str]:
"""Country codes as the line names them: up to four, else a count."""
found = sorted(codes)
return found if len(found) <= 4 else [f"{len(found)} countries"]
def _places(ask: readers.Ask) -> str:
words = sorted(ask.countries) if len(ask.countries) <= 4 \
else [f"{len(ask.countries)} countries"]
words = _codes(ask.countries)
if ask.remote:
words.append("remote")
return ", ".join(words)
def _site(module: Any, ask: readers.Ask, facets: Mapping[str, Mapping[str, str]] | None,
lead: str, roles: str) -> str:
"""The line of a board whose site does not know every value `ask` wants
on an `ids` filter, from the host ids the runner keeps (`facets`, None
before any are kept), as `_runner.series` drops them: what is asked,
then what the site does not know ("DK · SE not on this site · no remote
option"), after `lead` and before `roles`; or, with nothing left to
ask, "site lists none of DK, SE: nothing asked". "" when the site knows
every value, or `lead` is "" and something is still asked."""
if facets is None:
return ""
reads = module.READS.filters
known, missing = list(ask.countries), []
if reads.country == "ids" and ask.countries and "country" in facets:
listed = facets.get("country") or {}
known = [c for c in ask.countries if c in listed]
missing = [c for c in ask.countries if c not in listed]
no_remote = bool(ask.remote) and reads.remote == "ids" and not facets.get("remote")
if not (missing or no_remote):
return ""
if no_remote and not (ask.countries or ask.places) and "remote" not in facets:
# Remote alone, on a site that states no remote facet, is read whole.
return f"all places, no remote option on this site · {roles}" if lead else ""
remote = bool(ask.remote) and not no_remote
if not (known or ask.places or remote):
if not missing:
return "site has no remote option: nothing asked"
named = _codes(missing) + (["remote"] if no_remote else [])
return f"site lists none of {', '.join(named)}: nothing asked"
if not lead:
return ""
words = [", ".join(_codes(known) + list(ask.places) + (["remote"] if remote else []))]
if missing:
words.append(f"{', '.join(_codes(missing))} not on this site")
if no_remote:
words.append("no remote option")
return f"{lead}: {' · '.join(w for w in words if w)} · {roles}"
def _roles(ask: readers.Ask) -> str:
"""The role-type half of the line: the agents' terms the board is
asked for, or all roles."""
@@ -363,14 +448,23 @@ def line(found: Plan, sub: Subscription) -> str:
· all roles", "set here: DE · all roles", "all places, too many countries
to ask · follows agents' terms: engineer", or, for a board read whole,
why: "under 300 roles: read whole", "Greenhouse cannot filter: read
whole"."""
whole". A board with no size yet says it is sized on the next pull. On
a host that filters by its own ids, the line names what the site does
not know once the runner has asked it (`_site`): "follows agents: DK ·
SE not on this site · no remote option · all roles", or "site lists
none of DK, SE: nothing asked"."""
module = readers.HOSTS.get(sub.target.partition(":")[0])
if module is None:
return ""
given, why = found.why(sub)
if why == "unknown":
return f"{_WHOLE[why]}: sized on the next pull"
ask = _runner.ask_for(module, sub.params, found.for_subscription(sub))
roles = _roles(ask)
if _runner.filtered(ask):
lead = {"agents": "follows agents", "override": "set here"}.get(why, "")
if site := _site(module, ask, found.facets.get(sub.target), lead, roles):
return site
if why == "countries":
named = found.wide_names or f"{len(found.wide.countries)} countries"
return f"all countries ({named}) · {roles}"
+29 -10
View File
@@ -46,10 +46,11 @@ def set_board(store: Store, key: str, board: str, *, name: str = "",
check: bool = True, **fields: str) -> tuple[Subscription, int | None, int]:
"""Read company `key` from `board` (`host:slug`): attach it to a company
that has none, replace the one it has, or create the company from `name`
when there is no row. The board is fetched first and refused if its host
is unknown or it does not answer; a board listing zero roles is kept, since
a company between openings is still one to follow. `check=False` skips the
fetch, for the seed, whose boards were read once when it was written.
when there is no row. The board is sized first (`board_size`) and refused
if its host is unknown or it does not answer; a board listing zero roles
is kept, since a company between openings is still one to follow.
`check=False` skips the request, for the seed, whose boards were read
once when it was written; such a board is sized by its first pull.
The row's `ats` and `ats_slug` become the board's host and slug, its
`careers_url` the board's page where it was empty or the old board's, and
@@ -59,7 +60,7 @@ def set_board(store: Store, key: str, board: str, *, name: str = "",
change in one transaction (`Store.set_company_board`), and replacing a
board closes the old one's open roles. Returns the subscription, how many
roles the board lists (None unchecked) and how many roles closed."""
from plugins.ats import careers_url, fetch_board, is_known_target
from plugins.ats import board_size, careers_url, is_known_target, keep_size
board, name = board.strip(), name.strip()
key = key.strip() or company_key(name)
@@ -72,10 +73,10 @@ def set_board(store: Store, key: str, board: str, *, name: str = "",
raise ValueError(f"no company {key!r}; a new company needs a name")
if company_key(name) != key:
raise ValueError(f"{name!r} is not the company {key!r}")
rows: list[Any] | None = None
size: int | None = None
if check:
try:
rows, _complete = fetch_board(board)
size = board_size(board)
except Exception as exc:
raise ValueError(f"{board} did not answer: {exc}") from exc
host, _, slug = board.partition(":")
@@ -95,7 +96,23 @@ def set_board(store: Store, key: str, board: str, *, name: str = "",
assert sub is not None
if before is not None and before.target != board:
_forget_failing(store, before.target)
return sub, None if rows is None else len(rows), closed
if size is not None:
keep_size(store.state, board, size)
return sub, size, closed
def size_board(store: Store, board: str) -> int:
"""Size `board` again (`plugins.ats.board_size`) and keep it with the
day: how a board's size changes once it was added (a Recheck, `app
companies size`). Raises ValueError when it does not answer."""
from plugins.ats import board_size, keep_size
try:
size = board_size(board)
except Exception as exc:
raise ValueError(f"{board} did not answer: {exc}") from exc
keep_size(store.state, board, size)
return size
def detach_board(store: Store, key: str) -> int:
@@ -118,9 +135,11 @@ def _forget_failing(store: Store, target: str) -> None:
record of `target` failing would otherwise outlive the board, and count
in `app companies check` for good. The `ats` plugin's robots.txt verdict
and its lists of roles read and failed in detail go with it, since they
are about that board alone and would only grow stale."""
are about that board alone and would only grow stale, as would its size,
which a board attached again gets anew."""
store.state.pop(FAILING + target, None)
for key in ("ats:robots:", "ats:detailed:", "ats:detailfailed:"):
for key in ("ats:robots:", "ats:detailed:", "ats:detailfailed:", "ats:size:",
"ats:sizechecked:"):
store.state.pop(key + target, None)
+72 -13
View File
@@ -20,10 +20,18 @@ recorded as failing under `ats:failing:<target>`, and skipped. A board's
filter is the `reach` option the pull hands it (the agents', see
`app/jobs/reach.py`), over which its subscription `params` (`{"countries":
["DK"], "remote": true}`) win, narrowed to what its reader declares
(`_runner.ask_for`). An unfiltered read records the board's size under
`ats:size:<target>`, which the reach reads to leave small boards whole. A
read narrowed by the agents' fetch terms is never the whole board: it
records no size and prunes no per-role state.
(`_runner.ask_for`). A read narrowed by the agents' fetch terms is never
the whole board: it prunes no per-role state.
A board's size (`board_size`) is set once, when the board is added
(`app.jobs.sources.set_board`) or rechecked, and kept under
`ats:size:<target>` with the day it was checked under
`ats:sizechecked:<target>` (`keep_size`); the reach reads it to leave small
boards whole. A pull does not rewrite it. A board with no size yet is sized
in its batch before it is read: its reach then says `unsized` and the size
under which a board is read whole (`small`, `app.jobs.reach.SMALL`). A board
whose reader cannot narrow that reach is read whole anyway, and sized by
that read once it is complete, rather than by a second request.
A pull reads listings only. A host that states a role's text on the role's
own page (`text="detail"`) is read there by `app/jobs/enrich.py` after
@@ -307,12 +315,46 @@ def _read_details(client: httpx.Client, target: str, rows: list[Row], complete:
def fetch_board(target: str) -> tuple[list[Row], bool]:
"""Fetch and parse one board directly: no `Source`, no state, just "does
this board answer, what does it list, and is that all of it". For `app
companies add|check`. The listing only, unfiltered, no detail reads.
companies check`. The listing only, unfiltered, no detail reads.
Raises the same way `_fetch_rows` does."""
with httpx.Client() as client:
return _fetch_rows(client, target)
#: Pages read at most to size a board whose host states no total on page 1:
#: 5,000 roles on Teamtailor, 10,000 on HR Manager, far past any board seen.
SIZE_PAGES = 100
def board_size(target: str, *, client: httpx.Client | None = None) -> int:
"""How many roles board `target` lists, unfiltered and without terms: a
whole host's listing, the total a `complete="stated"` host gives on page
1, else the listing read to the end, at most `SIZE_PAGES` pages. Raises
as `_fetch_rows` does, so a board that does not answer is refused."""
if not is_known_target(target):
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
if client is None:
with httpx.Client() as own:
return board_size(target, client=own)
host, _, slug = target.partition(":")
module = readers.HOSTS[host]
get = _getter(client, host, slug)
if (_runner.declared(module) and module.READS.listing == "paged"
and module.READS.complete == "stated"):
_data, _results, total, _cursor = module.page(get, slug, None, readers.Ask())
if isinstance(total, int):
return total
limits = _runner.Limits(max_pages=SIZE_PAGES, page_delay=PAGE_DELAY)
return len(_runner.read_listing(module, slug, get, limits=limits).ids)
def keep_size(state: Any, target: str, size: int, *, today: str = "") -> None:
"""Keep board `target`'s size and the day it was checked (ISO,
`today`, else today in UTC)."""
state[f"ats:size:{target}"] = str(size)
state[f"ats:sizechecked:{target}"] = today or datetime.now(UTC).date().isoformat()
failure_reason = _runner.failure_reason
@@ -358,8 +400,7 @@ class ATSSource(Source):
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
module = readers.HOSTS[host]
get = _getter(client, host, slug)
ask = _runner.ask_for(module, self.params_for(target), self.options.get("reach"),
keys=self.options.get("keys"))
ask, unsized = self._ask(client, target, module)
listing = _runner.read(self.state, "ats", target, module, slug, get, ask,
limits=_limits())
except Exception as exc:
@@ -397,18 +438,36 @@ class ATSSource(Source):
# as the whole of it.
_runner.stamp(self.state, "ats", target, run_marker, listing,
[compound(i) for i in listing.ids])
if not (listing.filtered or listing.narrowed):
size = len(listing.ids)
if not listing.complete:
kept = self.state.get(f"ats:size:{target}", "")
size = max(size, int(kept) if kept.isdigit() else 0)
self.state[f"ats:size:{target}"] = str(size)
if unsized and listing.complete and not (listing.filtered or listing.narrowed):
keep_size(self.state, target, len(listing.ids))
if detail_sets is not None:
# A complete listing drops the ids it no longer lists.
for key, ids in zip(detail_keys(target), detail_sets, strict=False):
if key in self.state:
self.state[key] = json.dumps(sorted(ids))
def _ask(self, client: httpx.Client, target: str,
module: Any) -> tuple[readers.Ask, bool]:
"""What `target` is asked for, from the reach its batch handed it and
its params, and whether it still needs a size. A board with no size
yet (`unsized` in its reach) whose reach would narrow the read is
sized first (`board_size`, kept by `keep_size`), and read whole under
`small` roles. One whose read is whole anyway is sized by that read,
once it is complete."""
given = self.options.get("reach")
params, keys = self.params_for(target), self.options.get("keys")
if not isinstance(given, dict) or not given.get("unsized"):
return _runner.ask_for(module, params, given, keys=keys), False
reach = {k: v for k, v in given.items() if k not in ("unsized", "small")}
ask = _runner.ask_for(module, params, reach or None, keys=keys)
if not (_runner.filtered(ask) or _runner.narrowed(module, ask)):
return ask, True
size = board_size(target, client=client)
keep_size(self.state, target, size)
if size < int(given.get("small") or 0):
ask = _runner.ask_for(module, params, None, keys=keys)
return ask, False
# ------------------------------------------------------ the per-role reads
+343
View File
@@ -0,0 +1,343 @@
"""A board's size, set once: `ats.board_size` per kind of host, `set_board`
that keeps it, the pull that sizes a board
with none before it reads it, and the Companies line that names what a
site's own ids leave out. No network: `ats._get` or the runner's listing is
stubbed."""
from __future__ import annotations
import json
import sys
from datetime import UTC, datetime
from pathlib import Path
from typing import Any, ClassVar
from urllib.parse import parse_qs, urlsplit
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from pipeline import Tally # noqa: E402
from app.jobs import pull, reach, sources # noqa: E402
from app.models import Gate # noqa: E402
from app.storage import Store # noqa: E402
from plugins import ats # noqa: E402
from plugins.readers import Ask, _runner, greenhouse, workday # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
BOSCH = "smartrecruiters:BoschGroup"
TODAY = datetime.now(UTC).date().isoformat()
class FakeConfig:
sources: ClassVar[dict[str, object]] = {}
searches: ClassVar[list[object]] = []
@pytest.fixture
def store(tmp_path: Path) -> Store:
return Store(tmp_path / "t.db")
@pytest.fixture(autouse=True)
def _no_pause(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
def _query(url: str) -> dict[str, list[str]]:
return parse_qs(urlsplit(url).query)
def _posting(job_id: str, country: str = "de") -> dict[str, Any]:
return {"id": job_id, "name": f"Engineer {job_id}",
"location": {"city": "Berlin", "country": country}}
class SmartRecruiters:
"""A SmartRecruiters board of `total` roles that answers each request
with one page of three and the stated total, or, per country, the roles
there. Remembers every URL asked."""
def __init__(self, total: int | None, *, down: bool = False) -> None:
self.total, self.down = total, down
self.asked: list[str] = []
def __call__(self, client: Any, url: str, host: str, slug: str, **kwargs: Any) -> bytes:
self.asked.append(url)
if self.down:
raise OSError("connection refused")
query = _query(url)
if int(query.get("offset", ["0"])[0]):
return json.dumps({"content": []}).encode()
country = query.get("country", [""])[0]
content = [_posting(f"{country}{n}", country or "de") for n in range(3)]
answer: dict[str, Any] = {"content": content}
if self.total is not None:
answer["totalFound"] = 3 if country else self.total
return json.dumps(answer).encode()
def _teamtailor(count: int) -> Any:
"""A Teamtailor site of `count` roles, served 50 a page by offset."""
asked: list[str] = []
def get(client: Any, url: str, host: str, slug: str, **kwargs: Any) -> bytes:
asked.append(url)
offset = int(_query(url)["offset"][0])
items = "".join(
f"<item><title>Role {n}</title><link>https://acme.teamtailor.com/jobs/{n}</link>"
f"<guid>{n}</guid></item>" for n in range(offset, min(offset + 50, count)))
return f"<rss><channel>{items}</channel></rss>".encode()
get.asked = asked # type: ignore[attr-defined]
return get
def _wolf(store: Store, countries: tuple[str, ...] = ("DK", "SE"), remote: bool = True) -> None:
gates = [Gate(field="country", values=list(countries), mode="any")]
if remote:
gates.append(Gate(field="remote", values=["remote"], mode="any"))
store.save_agent("Wolf", gates=gates)
def _board(store: Store, target: str, company: str, size: int | None = None,
params: dict[str, object] | None = None) -> None:
store.add_subscription("ats", target, label=company, company=company,
board=target.partition(":")[0], params=params or {})
store.add_company(company)
if size is not None:
store.state[f"ats:size:{target}"] = str(size)
def _sub(store: Store, target: str) -> Any:
return next(s for s in store.subscriptions(source="ats") if s.target == target)
def _pull(store: Store) -> list[dict[str, Any]]:
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
results = [pull.run_batch(FakeConfig(), store, **batch)
for batch in batches]
pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type]
return batches
# ------------------------------------------------------------- board_size
def test_a_host_that_states_a_total_is_sized_by_page_one(
monkeypatch: pytest.MonkeyPatch) -> None:
board = SmartRecruiters(999)
monkeypatch.setattr(ats, "_get", board)
assert ats.board_size(BOSCH) == 999
# One unfiltered request: no country, no terms.
[url] = board.asked
assert "country" not in _query(url) and "q" not in _query(url)
def test_a_host_that_states_no_total_is_read_to_the_end(
monkeypatch: pytest.MonkeyPatch) -> None:
get = _teamtailor(120)
monkeypatch.setattr(ats, "_get", get)
assert ats.board_size("teamtailor:acme") == 120
assert [_query(u)["offset"][0] for u in get.asked] == ["0", "50", "100", "120"]
def test_a_stated_host_that_leaves_out_its_total_is_read_to_the_end(
monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(ats, "_get", SmartRecruiters(None))
assert ats.board_size(BOSCH) == 3
def test_a_whole_host_is_sized_by_its_one_answer(monkeypatch: pytest.MonkeyPatch) -> None:
body = (FIXTURES / "greenhouse.json").read_bytes()
monkeypatch.setattr(ats, "_get", lambda *args, **kwargs: body)
assert ats.board_size("greenhouse:acme") == len(greenhouse.parse(body, "acme"))
def test_board_size_refuses_an_unknown_board_and_raises_on_no_answer(
monkeypatch: pytest.MonkeyPatch) -> None:
with pytest.raises(ValueError, match="host:slug"):
ats.board_size("taleo:acme")
monkeypatch.setattr(ats, "_get", SmartRecruiters(5, down=True))
with pytest.raises(OSError):
ats.board_size(BOSCH)
# -------------------------------------------------------------- set_board
def test_set_board_keeps_the_size_and_the_day_without_reading_the_board_whole(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
board = SmartRecruiters(4830)
monkeypatch.setattr(ats, "_get", board)
sub, found, _closed = sources.set_board(store, "bosch", BOSCH, name="Bosch")
assert found == 4830 and len(board.asked) == 1
assert store.state[f"ats:size:{BOSCH}"] == "4830"
assert store.state[f"ats:sizechecked:{BOSCH}"] == TODAY
assert reach.plan(store).why(sub)[1] == "agents"
def test_the_seed_keeps_no_size_and_a_detached_board_forgets_it(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(ats, "_get", SmartRecruiters(4830, down=True))
sources.set_board(store, "bosch", BOSCH, name="Bosch", check=False)
assert f"ats:size:{BOSCH}" not in store.state
ats.keep_size(store.state, BOSCH, 4830)
sources.detach_board(store, "bosch")
assert f"ats:size:{BOSCH}" not in store.state
assert f"ats:sizechecked:{BOSCH}" not in store.state
# ------------------------------------------------------ the unsized reach
def test_a_board_with_no_size_is_handed_the_agents_reach_marked_unsized(store: Store) -> None:
_wolf(store)
_board(store, BOSCH, "Bosch")
_board(store, "workday:acme.wd3/Jobs", "Acme", params={"countries": ["DE"]})
_board(store, "smartrecruiters:tiny", "Tiny", size=12)
plan = reach.plan(store)
assert plan.for_subscription(_sub(store, BOSCH)) == {
"countries": ["DK", "SE"], "remote": True, "unsized": True, "small": reach.SMALL}
# A board with its own reach carries only the marks; a sized one none.
assert plan.for_subscription(_sub(store, "workday:acme.wd3/Jobs")) == {
"unsized": True, "small": reach.SMALL}
assert plan.for_subscription(_sub(store, "smartrecruiters:tiny")) is None
assert reach.line(plan, _sub(store, BOSCH)) == "size not known yet: sized on the next pull"
def test_a_big_board_with_no_size_is_sized_then_read_with_the_reach(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_wolf(store)
_board(store, BOSCH, "Bosch")
board = SmartRecruiters(4830)
monkeypatch.setattr(ats, "_get", board)
_pull(store)
assert store.state[f"ats:size:{BOSCH}"] == "4830"
assert store.state[f"ats:sizechecked:{BOSCH}"] == TODAY
# The first request sizes the board; the read then asks for DK and SE.
assert "country" not in _query(board.asked[0])
assert sorted(_query(u).get("country", [""])[0] for u in board.asked[1:]) == ["dk", "se"]
assert store.state[f"ats:filtered:{BOSCH}"] == "1"
# The next pull finds the size and does not ask for it again.
board.asked.clear()
_pull(store)
assert all("country" in _query(u) for u in board.asked)
def test_a_small_board_with_no_size_is_sized_then_read_whole(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_wolf(store)
_board(store, BOSCH, "Bosch")
board = SmartRecruiters(3)
monkeypatch.setattr(ats, "_get", board)
_pull(store)
assert store.state[f"ats:size:{BOSCH}"] == "3"
assert all("country" not in _query(u) for u in board.asked)
assert store.state[f"ats:filtered:{BOSCH}"] == ""
def test_a_board_that_does_not_answer_its_sizing_fails_as_any_read(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_wolf(store)
_board(store, BOSCH, "Bosch")
monkeypatch.setattr(ats, "_get", SmartRecruiters(4830, down=True))
_pull(store)
assert f"ats:size:{BOSCH}" not in store.state
assert "connection refused" in json.loads(store.state[f"ats:failing:{BOSCH}"])["reason"]
def test_a_board_read_whole_anyway_is_sized_by_that_read_once_it_is_complete(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_wolf(store)
_board(store, "greenhouse:acme", "Acme")
rows = [{"source_id": str(n), "title": f"Role {n}", "url": f"https://x.example/{n}",
"places": [], "description": ""} for n in range(4)]
answer = {"complete": False}
asked: list[Ask] = []
def fake(module: object, slug: str, get: object, ask: Ask, **kwargs: object
) -> _runner.Listing:
asked.append(ask)
return _runner.Listing(rows, answer["complete"], [r["source_id"] for r in rows]) # type: ignore[misc]
monkeypatch.setattr(_runner, "read_listing", fake)
monkeypatch.setattr(ats, "board_size", lambda *a, **k: pytest.fail("sized apart"))
_pull(store)
# Greenhouse cannot filter: one read, and an incomplete one sizes nothing.
assert asked == [Ask()]
assert "ats:size:greenhouse:acme" not in store.state
answer["complete"] = True
_pull(store)
assert store.state["ats:size:greenhouse:acme"] == "4"
assert store.state["ats:sizechecked:greenhouse:acme"] == TODAY
def test_a_pull_leaves_a_kept_size_alone(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_wolf(store)
_board(store, "greenhouse:acme", "Acme", size=250)
def fake(module: object, slug: str, get: object, ask: Ask, **kwargs: object
) -> _runner.Listing:
return _runner.Listing([], True, [])
monkeypatch.setattr(_runner, "read_listing", fake)
_pull(store)
assert store.state["ats:size:greenhouse:acme"] == "250"
# ---------------------------------------------------- the line and facets
WORKDAY = "workday:acme.wd3/Jobs"
@pytest.mark.parametrize("facets, line", [
(None, "follows agents: DK, SE, remote · all roles"),
({"country": {"DK": "c=dk", "SE": "c=se"}, "remote": {"remote": "r=1"}},
"follows agents: DK, SE, remote · all roles"),
({"country": {"DK": "c=dk"}},
"follows agents: DK · SE not on this site · no remote option · all roles"),
({"country": {"DK": "c=dk"}, "remote": {"remote": "r=1"}},
"follows agents: DK, remote · SE not on this site · all roles"),
({"country": {"DE": "c=de"}, "remote": {"remote": "r=1"}},
"follows agents: remote · DK, SE not on this site · all roles"),
({"country": {"DE": "c=de"}}, "site lists none of DK, SE, remote: nothing asked"),
], ids=["none kept", "all known", "one country, no remote", "one country and remote",
"remote only", "none"])
def test_the_line_names_what_a_site_does_not_know(
store: Store, facets: dict[str, dict[str, str]] | None, line: str) -> None:
_wolf(store)
_board(store, WORKDAY, "Acme", size=1000)
if facets is not None:
store.state[f"ats:facets:{WORKDAY}"] = json.dumps(
{"checked": "2026-09-30T06:00:00+00:00", "facets": facets})
plan = reach.plan(store)
sub = _sub(store, WORKDAY)
assert reach.line(plan, sub) == line
# What the line says is left to ask is what the runner asks.
ask = _runner.ask_for(workday, sub.params, plan.for_subscription(sub))
assert (_runner.series(workday, ask, facets) == []) == line.endswith("nothing asked")
def test_the_line_says_nothing_is_asked_in_the_plans_words(store: Store) -> None:
_wolf(store, remote=False)
_board(store, WORKDAY, "Acme", size=1000)
store.state[f"ats:facets:{WORKDAY}"] = json.dumps(
{"checked": "2026-09-30T06:00:00+00:00", "facets": {"country": {"DE": "c=de"}}})
assert reach.line(reach.plan(store), _sub(store, WORKDAY)) == \
"site lists none of DK, SE: nothing asked"
def test_a_boards_own_reach_is_named_the_same_and_remote_alone_is_read_whole(
store: Store) -> None:
_wolf(store)
_board(store, WORKDAY, "Acme", size=1000, params={"countries": ["DK", "NO"]})
_board(store, "workday:other.wd3/Jobs", "Other", size=1000, params={"remote": True})
for target in (WORKDAY, "workday:other.wd3/Jobs"):
store.state[f"ats:facets:{target}"] = json.dumps(
{"checked": "2026-09-30T06:00:00+00:00", "facets": {"country": {"DK": "c=dk"}}})
plan = reach.plan(store)
assert reach.line(plan, _sub(store, WORKDAY)) == \
"set here: DK · NO not on this site · all roles"
assert reach.line(plan, _sub(store, "workday:other.wd3/Jobs")) == \
"all places, no remote option on this site · all roles"
+4 -7
View File
@@ -127,8 +127,8 @@ def test_the_narrow_reach_takes_no_terms(store: Store) -> None:
@pytest.mark.parametrize("size, params", [
(299, {}), (None, {}), (5000, {"countries": ["DE"]}),
], ids=["small", "never read whole", "own reach"])
(299, {}), (5000, {"countries": ["DE"]}),
], ids=["small", "own reach"])
def test_small_unknown_overridden_and_switched_off_boards_take_no_terms(
store: Store, size: int | None, params: dict[str, object]) -> None:
_wolf(store, TERMS)
@@ -394,10 +394,10 @@ def test_a_term_read_writes_no_size_and_closes_nothing(
Tally(results=tuple(results)))
# No terms: Europe is too many countries for SmartRecruiters, so whole.
# A whole read writes no size either: the board keeps the one it has.
run()
assert [_query(u).get("q") for u in asked] == [None]
assert store.state["ats:size:smartrecruiters:bfl"] == "3"
store.state["ats:size:smartrecruiters:bfl"] = "1000"
assert store.state["ats:size:smartrecruiters:bfl"] == "1000"
open_before = {r.source_id for r in store.roles(limit=50)}
assert open_before == {f"smartrecruiters:bfl:{n}" for n in ("1", "2", "9")}
@@ -413,9 +413,6 @@ def test_a_term_read_writes_no_size_and_closes_nothing(
assert {r.source_id for r in store.roles(limit=50)} == open_before
assert settled["closed"] == 0
assert store.state["ats:size:smartrecruiters:bfl"] == "1000"
# A board sized by its open roles is never sized by a term read.
del store.state["ats:size:smartrecruiters:bfl"]
assert reach.sizes(store, store.subscriptions(source="ats")) == {}
# ------------------------------------------------------- storage and the API
+6 -6
View File
@@ -185,14 +185,14 @@ def test_small_unknown_and_overridden_boards_get_no_reach(
assert (found.params() is None) == (why != "agents")
def test_a_board_read_before_sizes_were_kept_is_sized_by_its_open_roles(store: Store) -> None:
def test_a_board_has_only_the_size_it_was_given(store: Store) -> None:
_board(store, "workday:acme.wd3/Jobs", "Acme", size=None)
[sub] = store.subscriptions(source="ats")
# A board read whole before sizes were kept is not sized by its open roles.
store.state["ats:ok:workday:acme.wd3/Jobs"] = "run-0"
assert reach.sizes(store, [sub]) == {"workday:acme.wd3/Jobs": 0}
# A filtered board's open roles are not its size.
store.state["ats:filtered:workday:acme.wd3/Jobs"] = "1"
assert reach.sizes(store, [sub]) == {}
store.state["ats:size:workday:acme.wd3/Jobs"] = "12"
assert reach.sizes(store, [sub]) == {"workday:acme.wd3/Jobs": 12}
def test_describe_is_short() -> None:
@@ -237,8 +237,8 @@ def test_the_pull_hands_each_board_its_reach_and_the_runner_narrows_it(
assert asked["BoschGroup"] == Ask(countries=("DK", "SE"))
assert asked["tiny"] == Ask()
assert store.state["ats:filtered:smartrecruiters:BoschGroup"] == "1"
# The small board, read whole, keeps its size; the filtered one keeps its last.
assert store.state["ats:size:smartrecruiters:tiny"] == "1"
# A pull rewrites no size: the small board read whole keeps its 12.
assert store.state["ats:size:smartrecruiters:tiny"] == "12"
assert store.state["ats:size:smartrecruiters:BoschGroup"] == "1000"
+5 -5
View File
@@ -29,24 +29,24 @@ def store(tmp_path: Path) -> Store:
class Boards:
"""A fake `fetch_board`: `listed` maps a target to how many roles it
lists; any other target does not answer. Remembers what was fetched."""
"""A fake `board_size`: `listed` maps a target to how many roles it
lists; any other target does not answer. Remembers what was sized."""
def __init__(self, **listed: int) -> None:
self.listed = {k.replace("_", ":", 1): v for k, v in listed.items()}
self.fetched: list[str] = []
def __call__(self, target: str) -> tuple[list[dict[str, Any]], bool]:
def __call__(self, target: str, **kwargs: Any) -> int:
self.fetched.append(target)
if target not in self.listed:
raise OSError("connection refused")
return [{"source_id": str(i)} for i in range(self.listed[target])], True
return self.listed[target]
@pytest.fixture
def boards(monkeypatch: pytest.MonkeyPatch) -> Boards:
fake = Boards(greenhouse_acme=3, ashby_acme=2, lever_quiet=0, lever_acme=1)
monkeypatch.setattr(ats, "fetch_board", fake)
monkeypatch.setattr(ats, "board_size", fake)
return fake
+7 -7
View File
@@ -603,7 +603,7 @@ def test_companies_lists_board_and_open_role_count(
"careers_opened_at": "", "failing_since": "", "failing_reason": "",
"from_search": False, "hand_reason": "", "by_hand": False,
"all_countries": False, "all_roles": False,
"reach_params": {}, "reach": "size not known yet: read whole"}
"reach_params": {}, "reach": "size not known yet: sized on the next pull"}
[other] = [c for c in res.json() if c["key"] == "other"]
assert (other["board"], other["subscription_id"], other["on"], other["ats"]) == (
"", None, None, "")
@@ -1371,7 +1371,7 @@ def test_a_board_is_followed_only_if_it_answers(
client: TestClient, store: Store, queue: Any, monkeypatch: pytest.MonkeyPatch) -> None:
import plugins.ats as ats
monkeypatch.setattr(ats, "fetch_board", lambda target: ([{"source_id": "1"}], True))
monkeypatch.setattr(ats, "board_size", lambda target, **kw: 1)
res = client.post("/api/boards", json={"board": "ashby:acme", "name": "Acme"})
assert res.status_code == 201
assert res.json()["found"] == 1 and res.json()["source"]["target"] == "ashby:acme"
@@ -1389,10 +1389,10 @@ def test_a_board_is_followed_only_if_it_answers(
bad = client.post("/api/boards", json={"board": "acme", "name": "Other"})
assert bad.status_code == 422 and "host:slug" in bad.json()["detail"]
def down(target: str) -> list[dict[str, str]]:
def down(target: str, **kw: object) -> int:
raise OSError("connection refused")
monkeypatch.setattr(ats, "fetch_board", down)
monkeypatch.setattr(ats, "board_size", down)
dead = client.post("/api/boards", json={"board": "ashby:other", "name": "Other"})
assert dead.status_code == 422 and "did not answer" in dead.json()["detail"]
assert store.company("other") is None
@@ -1402,7 +1402,7 @@ def test_a_companys_board_is_set_replaced_and_detached(
client: TestClient, store: Store, queue: Any, monkeypatch: pytest.MonkeyPatch) -> None:
import plugins.ats as ats
monkeypatch.setattr(ats, "fetch_board", lambda target: ([], True)) # between openings
monkeypatch.setattr(ats, "board_size", lambda target, **kw: 0) # between openings
store.add_company("Acme", key="acme", ats="teamtailor", ats_slug="acme")
res = client.post("/api/companies/acme/board", json={"board": "greenhouse:acme"})
@@ -1522,7 +1522,7 @@ def test_a_company_with_a_board_keeps_its_name_and_takes_the_other_edits(
client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
import plugins.ats as ats
monkeypatch.setattr(ats, "fetch_board", lambda target: ([], True))
monkeypatch.setattr(ats, "board_size", lambda target, **kw: 0)
store.add_company("Acme", key="acme")
assert client.post("/api/companies/acme/board", json={"board": "greenhouse:acme"}
).status_code == 200
@@ -1563,7 +1563,7 @@ def test_a_company_is_deleted_unless_it_is_read(
client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
import plugins.ats as ats
monkeypatch.setattr(ats, "fetch_board", lambda target: ([], True))
monkeypatch.setattr(ats, "board_size", lambda target, **kw: 0)
agent = store.save_agent("A")
store.add_company("Meta", key="meta", hand_reason="blocked")
store.set_affinity(agent.id, "meta", 1)