diff --git a/app/jobs/reach.py b/app/jobs/reach.py index 63f9be0..c79c8fb 100644 --- a/app/jobs/reach.py +++ b/app/jobs/reach.py @@ -21,8 +21,13 @@ on. Computed once per pull (`plan`) and handed to each board's batch as the it skips the `any` gates; it does not widen what is fetched. - **Board override**: a board whose subscription `params` name a country, a place or remote gets no reach here; `ask_for` reads the params instead. -- **Small boards**: a board that listed under `SMALL` roles when last read - whole gets no filter; filtering it saves little and risks closing roles. +- **Small boards**: a board of under `SMALL` roles gets no filter; + filtering it saves little and risks closing roles. A board's size is set + once, when it is added or rechecked (`plugins/ats.py`, `board_size`), and + never by a pull. A board with no size yet is handed the reach it would + have as a big one, with `unsized` and `small`, and its batch sizes it + before it reads it, so a big new board is not read whole first; a board + whose reader cannot narrow that reach is sized by its whole read instead. A reach of more countries than a `one`-filter host should be asked one by one (the wide reach's Europe on SmartRecruiters) is dropped by `ask_for`, @@ -43,14 +48,15 @@ board sends the roles that match one of them rather than every role: changes nothing on the narrow reach today, which takes no terms. - Only a reader that declares `query="one"` takes them, one series a term; `ask_for` has the rule for terms times the reach's series. -- A read with terms is never the whole board: it sets no `ats:size`, and - the pull closes nothing against it (`app/jobs/pull.py`). +- A read with terms is never the whole board: the pull closes nothing + against it (`app/jobs/pull.py`). - The search providers take none. Every saved search has a query of its own, which terms would replace; a search is a query already. """ from __future__ import annotations +import json from collections.abc import Iterable, Mapping from dataclasses import dataclass, field from typing import Any @@ -65,9 +71,15 @@ from plugins.readers import _runner #: whole. SMALL = 300 -#: `state` key of a board's size, `ats:size:`: the roles its last -#: unfiltered read listed, written by `plugins/ats.py`. +#: `state` keys of a board's size, `ats:size:`, the roles it listed +#: when it was sized, and of the day it was sized (ISO), both written by +#: `plugins/ats.py` (`keep_size`). SIZE = "ats:size:" +SIZE_CHECKED = "ats:sizechecked:" + +#: `state` key of the host ids a board's site states (`{"checked", +#: "facets"}`), kept by the runner (`_runner.facets_for`). +FACETS = "ats:facets:" #: The params that make a board's own reach (see `_runner._PARAM_DIMENSION`). OVERRIDE_KEYS = frozenset(("countries", "country", "places", "place", "remote")) @@ -239,7 +251,9 @@ class Plan: """The reach of one pull: `base` for every board, `wide` for the boards of the companies in `all_countries`, each board's size by target, the fetch `terms` of the wide reads, which skip the companies in - `all_roles`, and `wide_names`, what the agents call the wide reach.""" + `all_roles`, `wide_names`, what the agents call the wide reach, and + `facets`, the host ids the runner keeps for a board on a host that + filters by them, for the Companies line.""" base: Reach wide: Reach = EVERYWHERE @@ -248,6 +262,7 @@ class Plan: terms: tuple[str, ...] = () all_roles: frozenset[str] = frozenset() wide_names: str = "" + facets: dict[str, dict[str, dict[str, str]]] = field(default_factory=dict) def why(self, sub: Subscription) -> tuple[Reach, str]: """The reach `sub` is read with, and why: `override`, `small`, @@ -260,6 +275,10 @@ class Plan: return EVERYWHERE, "unknown" if size < SMALL: return EVERYWHERE, "small" + return self._big(sub) + + def _big(self, sub: Subscription) -> tuple[Reach, str]: + """The reach of `sub` as a board of `SMALL` roles or more.""" if company_key(sub.company or sub.label) in self.all_countries: return self.wide, "countries" return self.base, "agents" @@ -274,30 +293,51 @@ class Plan: fetch terms, and unless its company is set to All roles, a board on the wide reach carries them and `wide`, and a board on the agents' reach carries them for `ask_for` to use if it drops the countries as - too many (see the module docstring).""" + too many (see the module docstring). A board with no size carries + `unsized` and `small`, and the reach of a big board unless its params + set their own, for its batch to size it first.""" found, why = self.why(sub) + if why == "unknown": + found, why = self._big(sub) out = found.params() if self.terms and why in ("countries", "agents") and self.narrows_roles(sub): out = (out or {}) | {"terms": list(self.terms)} if why == "countries": out["wide"] = True + if sub.target not in self.sizes: + out = (out or {}) | {"unsized": True, "small": SMALL} return out def sizes(store: Store, subs: Iterable[Subscription]) -> dict[str, int]: - """Each board's size: the roles its last unfiltered read listed - (`ats:size:`), else, for a board read before that was kept and - never filtered, its open roles. A board never read has none.""" + """Each board's size as kept (`ats:size:`); a board never sized + has none.""" state = store.state - counts = store.open_countries_by_subscription() out: dict[str, int] = {} for sub in subs: kept = state.get(f"{SIZE}{sub.target}", "") if kept.isdigit(): out[sub.target] = int(kept) - elif (state.get(f"ats:ok:{sub.target}") and not state.get(f"ats:filtered:{sub.target}") - and not state.get(f"ats:narrowed:{sub.target}")): - out[sub.target] = sum(counts.get(sub.id, {}).values()) + return out + + +def site_facets(store: Store, subs: Iterable[Subscription]) -> dict[str, dict[str, dict[str, str]]]: + """The host ids the runner keeps for each board on a host whose country + or remote filter is `ids` (`ats:facets:`); a board with none kept + is left out.""" + out: dict[str, dict[str, dict[str, str]]] = {} + for sub in subs: + module = readers.HOSTS.get(sub.target.partition(":")[0]) + if module is None or not _runner.declared(module) \ + or "ids" not in (module.READS.filters.country, module.READS.filters.remote): + continue + try: + kept = json.loads(store.state.get(f"{FACETS}{sub.target}") or "{}") + except (TypeError, ValueError): + continue + found = kept.get("facets") if isinstance(kept, dict) else None + if isinstance(found, dict): + out[sub.target] = found return out @@ -321,7 +361,8 @@ def plan(store: Store, subs: Iterable[Subscription] | None = None) -> Plan: all_countries=frozenset(c.key for c in rows if c.all_countries), terms=fetch_terms(agents), all_roles=frozenset(c.key for c in rows if c.all_roles), - wide_names=wide_names(agents, gaz) if agents else "everywhere") + wide_names=wide_names(agents, gaz) if agents else "everywhere", + facets=site_facets(store, subs)) def reaches(store: Store, subs: Iterable[Subscription]) -> Mapping[str, dict[str, Any]]: @@ -338,14 +379,58 @@ _WHOLE = {"small": f"under {SMALL} roles", "unknown": "size not known yet", "agents": "an agent takes any place"} +def _codes(codes: Iterable[str]) -> list[str]: + """Country codes as the line names them: up to four, else a count.""" + found = sorted(codes) + return found if len(found) <= 4 else [f"{len(found)} countries"] + + def _places(ask: readers.Ask) -> str: - words = sorted(ask.countries) if len(ask.countries) <= 4 \ - else [f"{len(ask.countries)} countries"] + words = _codes(ask.countries) if ask.remote: words.append("remote") return ", ".join(words) +def _site(module: Any, ask: readers.Ask, facets: Mapping[str, Mapping[str, str]] | None, + lead: str, roles: str) -> str: + """The line of a board whose site does not know every value `ask` wants + on an `ids` filter, from the host ids the runner keeps (`facets`, None + before any are kept), as `_runner.series` drops them: what is asked, + then what the site does not know ("DK · SE not on this site · no remote + option"), after `lead` and before `roles`; or, with nothing left to + ask, "site lists none of DK, SE: nothing asked". "" when the site knows + every value, or `lead` is "" and something is still asked.""" + if facets is None: + return "" + reads = module.READS.filters + known, missing = list(ask.countries), [] + if reads.country == "ids" and ask.countries and "country" in facets: + listed = facets.get("country") or {} + known = [c for c in ask.countries if c in listed] + missing = [c for c in ask.countries if c not in listed] + no_remote = bool(ask.remote) and reads.remote == "ids" and not facets.get("remote") + if not (missing or no_remote): + return "" + if no_remote and not (ask.countries or ask.places) and "remote" not in facets: + # Remote alone, on a site that states no remote facet, is read whole. + return f"all places, no remote option on this site · {roles}" if lead else "" + remote = bool(ask.remote) and not no_remote + if not (known or ask.places or remote): + if not missing: + return "site has no remote option: nothing asked" + named = _codes(missing) + (["remote"] if no_remote else []) + return f"site lists none of {', '.join(named)}: nothing asked" + if not lead: + return "" + words = [", ".join(_codes(known) + list(ask.places) + (["remote"] if remote else []))] + if missing: + words.append(f"{', '.join(_codes(missing))} not on this site") + if no_remote: + words.append("no remote option") + return f"{lead}: {' · '.join(w for w in words if w)} · {roles}" + + def _roles(ask: readers.Ask) -> str: """The role-type half of the line: the agents' terms the board is asked for, or all roles.""" @@ -363,14 +448,23 @@ def line(found: Plan, sub: Subscription) -> str: · all roles", "set here: DE · all roles", "all places, too many countries to ask · follows agents' terms: engineer", or, for a board read whole, why: "under 300 roles: read whole", "Greenhouse cannot filter: read - whole".""" + whole". A board with no size yet says it is sized on the next pull. On + a host that filters by its own ids, the line names what the site does + not know once the runner has asked it (`_site`): "follows agents: DK · + SE not on this site · no remote option · all roles", or "site lists + none of DK, SE: nothing asked".""" module = readers.HOSTS.get(sub.target.partition(":")[0]) if module is None: return "" given, why = found.why(sub) + if why == "unknown": + return f"{_WHOLE[why]}: sized on the next pull" ask = _runner.ask_for(module, sub.params, found.for_subscription(sub)) roles = _roles(ask) if _runner.filtered(ask): + lead = {"agents": "follows agents", "override": "set here"}.get(why, "") + if site := _site(module, ask, found.facets.get(sub.target), lead, roles): + return site if why == "countries": named = found.wide_names or f"{len(found.wide.countries)} countries" return f"all countries ({named}) · {roles}" diff --git a/app/jobs/sources.py b/app/jobs/sources.py index 7a5a132..add9a2f 100644 --- a/app/jobs/sources.py +++ b/app/jobs/sources.py @@ -46,10 +46,11 @@ def set_board(store: Store, key: str, board: str, *, name: str = "", check: bool = True, **fields: str) -> tuple[Subscription, int | None, int]: """Read company `key` from `board` (`host:slug`): attach it to a company that has none, replace the one it has, or create the company from `name` - when there is no row. The board is fetched first and refused if its host - is unknown or it does not answer; a board listing zero roles is kept, since - a company between openings is still one to follow. `check=False` skips the - fetch, for the seed, whose boards were read once when it was written. + when there is no row. The board is sized first (`board_size`) and refused + if its host is unknown or it does not answer; a board listing zero roles + is kept, since a company between openings is still one to follow. + `check=False` skips the request, for the seed, whose boards were read + once when it was written; such a board is sized by its first pull. The row's `ats` and `ats_slug` become the board's host and slug, its `careers_url` the board's page where it was empty or the old board's, and @@ -59,7 +60,7 @@ def set_board(store: Store, key: str, board: str, *, name: str = "", change in one transaction (`Store.set_company_board`), and replacing a board closes the old one's open roles. Returns the subscription, how many roles the board lists (None unchecked) and how many roles closed.""" - from plugins.ats import careers_url, fetch_board, is_known_target + from plugins.ats import board_size, careers_url, is_known_target, keep_size board, name = board.strip(), name.strip() key = key.strip() or company_key(name) @@ -72,10 +73,10 @@ def set_board(store: Store, key: str, board: str, *, name: str = "", raise ValueError(f"no company {key!r}; a new company needs a name") if company_key(name) != key: raise ValueError(f"{name!r} is not the company {key!r}") - rows: list[Any] | None = None + size: int | None = None if check: try: - rows, _complete = fetch_board(board) + size = board_size(board) except Exception as exc: raise ValueError(f"{board} did not answer: {exc}") from exc host, _, slug = board.partition(":") @@ -95,7 +96,23 @@ def set_board(store: Store, key: str, board: str, *, name: str = "", assert sub is not None if before is not None and before.target != board: _forget_failing(store, before.target) - return sub, None if rows is None else len(rows), closed + if size is not None: + keep_size(store.state, board, size) + return sub, size, closed + + +def size_board(store: Store, board: str) -> int: + """Size `board` again (`plugins.ats.board_size`) and keep it with the + day: how a board's size changes once it was added (a Recheck, `app + companies size`). Raises ValueError when it does not answer.""" + from plugins.ats import board_size, keep_size + + try: + size = board_size(board) + except Exception as exc: + raise ValueError(f"{board} did not answer: {exc}") from exc + keep_size(store.state, board, size) + return size def detach_board(store: Store, key: str) -> int: @@ -118,9 +135,11 @@ def _forget_failing(store: Store, target: str) -> None: record of `target` failing would otherwise outlive the board, and count in `app companies check` for good. The `ats` plugin's robots.txt verdict and its lists of roles read and failed in detail go with it, since they - are about that board alone and would only grow stale.""" + are about that board alone and would only grow stale, as would its size, + which a board attached again gets anew.""" store.state.pop(FAILING + target, None) - for key in ("ats:robots:", "ats:detailed:", "ats:detailfailed:"): + for key in ("ats:robots:", "ats:detailed:", "ats:detailfailed:", "ats:size:", + "ats:sizechecked:"): store.state.pop(key + target, None) diff --git a/plugins/ats.py b/plugins/ats.py index 7aa8ab7..0533775 100644 --- a/plugins/ats.py +++ b/plugins/ats.py @@ -20,10 +20,18 @@ recorded as failing under `ats:failing:`, and skipped. A board's filter is the `reach` option the pull hands it (the agents', see `app/jobs/reach.py`), over which its subscription `params` (`{"countries": ["DK"], "remote": true}`) win, narrowed to what its reader declares -(`_runner.ask_for`). An unfiltered read records the board's size under -`ats:size:`, which the reach reads to leave small boards whole. A -read narrowed by the agents' fetch terms is never the whole board: it -records no size and prunes no per-role state. +(`_runner.ask_for`). A read narrowed by the agents' fetch terms is never +the whole board: it prunes no per-role state. + +A board's size (`board_size`) is set once, when the board is added +(`app.jobs.sources.set_board`) or rechecked, and kept under +`ats:size:` with the day it was checked under +`ats:sizechecked:` (`keep_size`); the reach reads it to leave small +boards whole. A pull does not rewrite it. A board with no size yet is sized +in its batch before it is read: its reach then says `unsized` and the size +under which a board is read whole (`small`, `app.jobs.reach.SMALL`). A board +whose reader cannot narrow that reach is read whole anyway, and sized by +that read once it is complete, rather than by a second request. A pull reads listings only. A host that states a role's text on the role's own page (`text="detail"`) is read there by `app/jobs/enrich.py` after @@ -307,12 +315,46 @@ def _read_details(client: httpx.Client, target: str, rows: list[Row], complete: def fetch_board(target: str) -> tuple[list[Row], bool]: """Fetch and parse one board directly: no `Source`, no state, just "does this board answer, what does it list, and is that all of it". For `app - companies add|check`. The listing only, unfiltered, no detail reads. + companies check`. The listing only, unfiltered, no detail reads. Raises the same way `_fetch_rows` does.""" with httpx.Client() as client: return _fetch_rows(client, target) +#: Pages read at most to size a board whose host states no total on page 1: +#: 5,000 roles on Teamtailor, 10,000 on HR Manager, far past any board seen. +SIZE_PAGES = 100 + + +def board_size(target: str, *, client: httpx.Client | None = None) -> int: + """How many roles board `target` lists, unfiltered and without terms: a + whole host's listing, the total a `complete="stated"` host gives on page + 1, else the listing read to the end, at most `SIZE_PAGES` pages. Raises + as `_fetch_rows` does, so a board that does not answer is refused.""" + if not is_known_target(target): + raise ValueError(f"{target!r} is not host:slug for a known ATS host") + if client is None: + with httpx.Client() as own: + return board_size(target, client=own) + host, _, slug = target.partition(":") + module = readers.HOSTS[host] + get = _getter(client, host, slug) + if (_runner.declared(module) and module.READS.listing == "paged" + and module.READS.complete == "stated"): + _data, _results, total, _cursor = module.page(get, slug, None, readers.Ask()) + if isinstance(total, int): + return total + limits = _runner.Limits(max_pages=SIZE_PAGES, page_delay=PAGE_DELAY) + return len(_runner.read_listing(module, slug, get, limits=limits).ids) + + +def keep_size(state: Any, target: str, size: int, *, today: str = "") -> None: + """Keep board `target`'s size and the day it was checked (ISO, + `today`, else today in UTC).""" + state[f"ats:size:{target}"] = str(size) + state[f"ats:sizechecked:{target}"] = today or datetime.now(UTC).date().isoformat() + + failure_reason = _runner.failure_reason @@ -358,8 +400,7 @@ class ATSSource(Source): raise ValueError(f"{target!r} is not host:slug for a known ATS host") module = readers.HOSTS[host] get = _getter(client, host, slug) - ask = _runner.ask_for(module, self.params_for(target), self.options.get("reach"), - keys=self.options.get("keys")) + ask, unsized = self._ask(client, target, module) listing = _runner.read(self.state, "ats", target, module, slug, get, ask, limits=_limits()) except Exception as exc: @@ -397,18 +438,36 @@ class ATSSource(Source): # as the whole of it. _runner.stamp(self.state, "ats", target, run_marker, listing, [compound(i) for i in listing.ids]) - if not (listing.filtered or listing.narrowed): - size = len(listing.ids) - if not listing.complete: - kept = self.state.get(f"ats:size:{target}", "") - size = max(size, int(kept) if kept.isdigit() else 0) - self.state[f"ats:size:{target}"] = str(size) + if unsized and listing.complete and not (listing.filtered or listing.narrowed): + keep_size(self.state, target, len(listing.ids)) if detail_sets is not None: # A complete listing drops the ids it no longer lists. for key, ids in zip(detail_keys(target), detail_sets, strict=False): if key in self.state: self.state[key] = json.dumps(sorted(ids)) + def _ask(self, client: httpx.Client, target: str, + module: Any) -> tuple[readers.Ask, bool]: + """What `target` is asked for, from the reach its batch handed it and + its params, and whether it still needs a size. A board with no size + yet (`unsized` in its reach) whose reach would narrow the read is + sized first (`board_size`, kept by `keep_size`), and read whole under + `small` roles. One whose read is whole anyway is sized by that read, + once it is complete.""" + given = self.options.get("reach") + params, keys = self.params_for(target), self.options.get("keys") + if not isinstance(given, dict) or not given.get("unsized"): + return _runner.ask_for(module, params, given, keys=keys), False + reach = {k: v for k, v in given.items() if k not in ("unsized", "small")} + ask = _runner.ask_for(module, params, reach or None, keys=keys) + if not (_runner.filtered(ask) or _runner.narrowed(module, ask)): + return ask, True + size = board_size(target, client=client) + keep_size(self.state, target, size) + if size < int(given.get("small") or 0): + ask = _runner.ask_for(module, params, None, keys=keys) + return ask, False + # ------------------------------------------------------ the per-role reads diff --git a/tests/test_board_size.py b/tests/test_board_size.py new file mode 100644 index 0000000..329b46c --- /dev/null +++ b/tests/test_board_size.py @@ -0,0 +1,343 @@ +"""A board's size, set once: `ats.board_size` per kind of host, `set_board` +that keeps it, the pull that sizes a board +with none before it reads it, and the Companies line that names what a +site's own ids leave out. No network: `ats._get` or the runner's listing is +stubbed.""" + +from __future__ import annotations + +import json +import sys +from datetime import UTC, datetime +from pathlib import Path +from typing import Any, ClassVar +from urllib.parse import parse_qs, urlsplit + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from pipeline import Tally # noqa: E402 +from app.jobs import pull, reach, sources # noqa: E402 +from app.models import Gate # noqa: E402 +from app.storage import Store # noqa: E402 +from plugins import ats # noqa: E402 +from plugins.readers import Ask, _runner, greenhouse, workday # noqa: E402 + +FIXTURES = ROOT / "tests" / "fixtures" / "ats" +BOSCH = "smartrecruiters:BoschGroup" +TODAY = datetime.now(UTC).date().isoformat() + + +class FakeConfig: + sources: ClassVar[dict[str, object]] = {} + searches: ClassVar[list[object]] = [] + + +@pytest.fixture +def store(tmp_path: Path) -> Store: + return Store(tmp_path / "t.db") + + +@pytest.fixture(autouse=True) +def _no_pause(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(ats, "PAGE_DELAY", 0) + + +def _query(url: str) -> dict[str, list[str]]: + return parse_qs(urlsplit(url).query) + + +def _posting(job_id: str, country: str = "de") -> dict[str, Any]: + return {"id": job_id, "name": f"Engineer {job_id}", + "location": {"city": "Berlin", "country": country}} + + +class SmartRecruiters: + """A SmartRecruiters board of `total` roles that answers each request + with one page of three and the stated total, or, per country, the roles + there. Remembers every URL asked.""" + + def __init__(self, total: int | None, *, down: bool = False) -> None: + self.total, self.down = total, down + self.asked: list[str] = [] + + def __call__(self, client: Any, url: str, host: str, slug: str, **kwargs: Any) -> bytes: + self.asked.append(url) + if self.down: + raise OSError("connection refused") + query = _query(url) + if int(query.get("offset", ["0"])[0]): + return json.dumps({"content": []}).encode() + country = query.get("country", [""])[0] + content = [_posting(f"{country}{n}", country or "de") for n in range(3)] + answer: dict[str, Any] = {"content": content} + if self.total is not None: + answer["totalFound"] = 3 if country else self.total + return json.dumps(answer).encode() + + +def _teamtailor(count: int) -> Any: + """A Teamtailor site of `count` roles, served 50 a page by offset.""" + asked: list[str] = [] + + def get(client: Any, url: str, host: str, slug: str, **kwargs: Any) -> bytes: + asked.append(url) + offset = int(_query(url)["offset"][0]) + items = "".join( + f"Role {n}https://acme.teamtailor.com/jobs/{n}" + f"{n}" for n in range(offset, min(offset + 50, count))) + return f"{items}".encode() + + get.asked = asked # type: ignore[attr-defined] + return get + + +def _wolf(store: Store, countries: tuple[str, ...] = ("DK", "SE"), remote: bool = True) -> None: + gates = [Gate(field="country", values=list(countries), mode="any")] + if remote: + gates.append(Gate(field="remote", values=["remote"], mode="any")) + store.save_agent("Wolf", gates=gates) + + +def _board(store: Store, target: str, company: str, size: int | None = None, + params: dict[str, object] | None = None) -> None: + store.add_subscription("ats", target, label=company, company=company, + board=target.partition(":")[0], params=params or {}) + store.add_company(company) + if size is not None: + store.state[f"ats:size:{target}"] = str(size) + + +def _sub(store: Store, target: str) -> Any: + return next(s for s in store.subscriptions(source="ats") if s.target == target) + + +def _pull(store: Store) -> list[dict[str, Any]]: + batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type] + results = [pull.run_batch(FakeConfig(), store, **batch) + for batch in batches] + pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type] + return batches + + +# ------------------------------------------------------------- board_size + +def test_a_host_that_states_a_total_is_sized_by_page_one( + monkeypatch: pytest.MonkeyPatch) -> None: + board = SmartRecruiters(999) + monkeypatch.setattr(ats, "_get", board) + assert ats.board_size(BOSCH) == 999 + # One unfiltered request: no country, no terms. + [url] = board.asked + assert "country" not in _query(url) and "q" not in _query(url) + + +def test_a_host_that_states_no_total_is_read_to_the_end( + monkeypatch: pytest.MonkeyPatch) -> None: + get = _teamtailor(120) + monkeypatch.setattr(ats, "_get", get) + assert ats.board_size("teamtailor:acme") == 120 + assert [_query(u)["offset"][0] for u in get.asked] == ["0", "50", "100", "120"] + + +def test_a_stated_host_that_leaves_out_its_total_is_read_to_the_end( + monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(ats, "_get", SmartRecruiters(None)) + assert ats.board_size(BOSCH) == 3 + + +def test_a_whole_host_is_sized_by_its_one_answer(monkeypatch: pytest.MonkeyPatch) -> None: + body = (FIXTURES / "greenhouse.json").read_bytes() + monkeypatch.setattr(ats, "_get", lambda *args, **kwargs: body) + assert ats.board_size("greenhouse:acme") == len(greenhouse.parse(body, "acme")) + + +def test_board_size_refuses_an_unknown_board_and_raises_on_no_answer( + monkeypatch: pytest.MonkeyPatch) -> None: + with pytest.raises(ValueError, match="host:slug"): + ats.board_size("taleo:acme") + monkeypatch.setattr(ats, "_get", SmartRecruiters(5, down=True)) + with pytest.raises(OSError): + ats.board_size(BOSCH) + + +# -------------------------------------------------------------- set_board + +def test_set_board_keeps_the_size_and_the_day_without_reading_the_board_whole( + store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + board = SmartRecruiters(4830) + monkeypatch.setattr(ats, "_get", board) + sub, found, _closed = sources.set_board(store, "bosch", BOSCH, name="Bosch") + assert found == 4830 and len(board.asked) == 1 + assert store.state[f"ats:size:{BOSCH}"] == "4830" + assert store.state[f"ats:sizechecked:{BOSCH}"] == TODAY + assert reach.plan(store).why(sub)[1] == "agents" + + +def test_the_seed_keeps_no_size_and_a_detached_board_forgets_it( + store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(ats, "_get", SmartRecruiters(4830, down=True)) + sources.set_board(store, "bosch", BOSCH, name="Bosch", check=False) + assert f"ats:size:{BOSCH}" not in store.state + ats.keep_size(store.state, BOSCH, 4830) + sources.detach_board(store, "bosch") + assert f"ats:size:{BOSCH}" not in store.state + assert f"ats:sizechecked:{BOSCH}" not in store.state + + +# ------------------------------------------------------ the unsized reach + +def test_a_board_with_no_size_is_handed_the_agents_reach_marked_unsized(store: Store) -> None: + _wolf(store) + _board(store, BOSCH, "Bosch") + _board(store, "workday:acme.wd3/Jobs", "Acme", params={"countries": ["DE"]}) + _board(store, "smartrecruiters:tiny", "Tiny", size=12) + plan = reach.plan(store) + assert plan.for_subscription(_sub(store, BOSCH)) == { + "countries": ["DK", "SE"], "remote": True, "unsized": True, "small": reach.SMALL} + # A board with its own reach carries only the marks; a sized one none. + assert plan.for_subscription(_sub(store, "workday:acme.wd3/Jobs")) == { + "unsized": True, "small": reach.SMALL} + assert plan.for_subscription(_sub(store, "smartrecruiters:tiny")) is None + assert reach.line(plan, _sub(store, BOSCH)) == "size not known yet: sized on the next pull" + + +def test_a_big_board_with_no_size_is_sized_then_read_with_the_reach( + store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + _wolf(store) + _board(store, BOSCH, "Bosch") + board = SmartRecruiters(4830) + monkeypatch.setattr(ats, "_get", board) + _pull(store) + assert store.state[f"ats:size:{BOSCH}"] == "4830" + assert store.state[f"ats:sizechecked:{BOSCH}"] == TODAY + # The first request sizes the board; the read then asks for DK and SE. + assert "country" not in _query(board.asked[0]) + assert sorted(_query(u).get("country", [""])[0] for u in board.asked[1:]) == ["dk", "se"] + assert store.state[f"ats:filtered:{BOSCH}"] == "1" + # The next pull finds the size and does not ask for it again. + board.asked.clear() + _pull(store) + assert all("country" in _query(u) for u in board.asked) + + +def test_a_small_board_with_no_size_is_sized_then_read_whole( + store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + _wolf(store) + _board(store, BOSCH, "Bosch") + board = SmartRecruiters(3) + monkeypatch.setattr(ats, "_get", board) + _pull(store) + assert store.state[f"ats:size:{BOSCH}"] == "3" + assert all("country" not in _query(u) for u in board.asked) + assert store.state[f"ats:filtered:{BOSCH}"] == "" + + +def test_a_board_that_does_not_answer_its_sizing_fails_as_any_read( + store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + _wolf(store) + _board(store, BOSCH, "Bosch") + monkeypatch.setattr(ats, "_get", SmartRecruiters(4830, down=True)) + _pull(store) + assert f"ats:size:{BOSCH}" not in store.state + assert "connection refused" in json.loads(store.state[f"ats:failing:{BOSCH}"])["reason"] + + +def test_a_board_read_whole_anyway_is_sized_by_that_read_once_it_is_complete( + store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + _wolf(store) + _board(store, "greenhouse:acme", "Acme") + rows = [{"source_id": str(n), "title": f"Role {n}", "url": f"https://x.example/{n}", + "places": [], "description": ""} for n in range(4)] + answer = {"complete": False} + asked: list[Ask] = [] + + def fake(module: object, slug: str, get: object, ask: Ask, **kwargs: object + ) -> _runner.Listing: + asked.append(ask) + return _runner.Listing(rows, answer["complete"], [r["source_id"] for r in rows]) # type: ignore[misc] + + monkeypatch.setattr(_runner, "read_listing", fake) + monkeypatch.setattr(ats, "board_size", lambda *a, **k: pytest.fail("sized apart")) + _pull(store) + # Greenhouse cannot filter: one read, and an incomplete one sizes nothing. + assert asked == [Ask()] + assert "ats:size:greenhouse:acme" not in store.state + answer["complete"] = True + _pull(store) + assert store.state["ats:size:greenhouse:acme"] == "4" + assert store.state["ats:sizechecked:greenhouse:acme"] == TODAY + + +def test_a_pull_leaves_a_kept_size_alone( + store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + _wolf(store) + _board(store, "greenhouse:acme", "Acme", size=250) + + def fake(module: object, slug: str, get: object, ask: Ask, **kwargs: object + ) -> _runner.Listing: + return _runner.Listing([], True, []) + + monkeypatch.setattr(_runner, "read_listing", fake) + _pull(store) + assert store.state["ats:size:greenhouse:acme"] == "250" + + +# ---------------------------------------------------- the line and facets + +WORKDAY = "workday:acme.wd3/Jobs" + + +@pytest.mark.parametrize("facets, line", [ + (None, "follows agents: DK, SE, remote · all roles"), + ({"country": {"DK": "c=dk", "SE": "c=se"}, "remote": {"remote": "r=1"}}, + "follows agents: DK, SE, remote · all roles"), + ({"country": {"DK": "c=dk"}}, + "follows agents: DK · SE not on this site · no remote option · all roles"), + ({"country": {"DK": "c=dk"}, "remote": {"remote": "r=1"}}, + "follows agents: DK, remote · SE not on this site · all roles"), + ({"country": {"DE": "c=de"}, "remote": {"remote": "r=1"}}, + "follows agents: remote · DK, SE not on this site · all roles"), + ({"country": {"DE": "c=de"}}, "site lists none of DK, SE, remote: nothing asked"), +], ids=["none kept", "all known", "one country, no remote", "one country and remote", + "remote only", "none"]) +def test_the_line_names_what_a_site_does_not_know( + store: Store, facets: dict[str, dict[str, str]] | None, line: str) -> None: + _wolf(store) + _board(store, WORKDAY, "Acme", size=1000) + if facets is not None: + store.state[f"ats:facets:{WORKDAY}"] = json.dumps( + {"checked": "2026-09-30T06:00:00+00:00", "facets": facets}) + plan = reach.plan(store) + sub = _sub(store, WORKDAY) + assert reach.line(plan, sub) == line + # What the line says is left to ask is what the runner asks. + ask = _runner.ask_for(workday, sub.params, plan.for_subscription(sub)) + assert (_runner.series(workday, ask, facets) == []) == line.endswith("nothing asked") + + +def test_the_line_says_nothing_is_asked_in_the_plans_words(store: Store) -> None: + _wolf(store, remote=False) + _board(store, WORKDAY, "Acme", size=1000) + store.state[f"ats:facets:{WORKDAY}"] = json.dumps( + {"checked": "2026-09-30T06:00:00+00:00", "facets": {"country": {"DE": "c=de"}}}) + assert reach.line(reach.plan(store), _sub(store, WORKDAY)) == \ + "site lists none of DK, SE: nothing asked" + + +def test_a_boards_own_reach_is_named_the_same_and_remote_alone_is_read_whole( + store: Store) -> None: + _wolf(store) + _board(store, WORKDAY, "Acme", size=1000, params={"countries": ["DK", "NO"]}) + _board(store, "workday:other.wd3/Jobs", "Other", size=1000, params={"remote": True}) + for target in (WORKDAY, "workday:other.wd3/Jobs"): + store.state[f"ats:facets:{target}"] = json.dumps( + {"checked": "2026-09-30T06:00:00+00:00", "facets": {"country": {"DK": "c=dk"}}}) + plan = reach.plan(store) + assert reach.line(plan, _sub(store, WORKDAY)) == \ + "set here: DK · NO not on this site · all roles" + assert reach.line(plan, _sub(store, "workday:other.wd3/Jobs")) == \ + "all places, no remote option on this site · all roles" diff --git a/tests/test_fetch_terms.py b/tests/test_fetch_terms.py index 100c533..ab109dc 100644 --- a/tests/test_fetch_terms.py +++ b/tests/test_fetch_terms.py @@ -127,8 +127,8 @@ def test_the_narrow_reach_takes_no_terms(store: Store) -> None: @pytest.mark.parametrize("size, params", [ - (299, {}), (None, {}), (5000, {"countries": ["DE"]}), -], ids=["small", "never read whole", "own reach"]) + (299, {}), (5000, {"countries": ["DE"]}), +], ids=["small", "own reach"]) def test_small_unknown_overridden_and_switched_off_boards_take_no_terms( store: Store, size: int | None, params: dict[str, object]) -> None: _wolf(store, TERMS) @@ -394,10 +394,10 @@ def test_a_term_read_writes_no_size_and_closes_nothing( Tally(results=tuple(results))) # No terms: Europe is too many countries for SmartRecruiters, so whole. + # A whole read writes no size either: the board keeps the one it has. run() assert [_query(u).get("q") for u in asked] == [None] - assert store.state["ats:size:smartrecruiters:bfl"] == "3" - store.state["ats:size:smartrecruiters:bfl"] = "1000" + assert store.state["ats:size:smartrecruiters:bfl"] == "1000" open_before = {r.source_id for r in store.roles(limit=50)} assert open_before == {f"smartrecruiters:bfl:{n}" for n in ("1", "2", "9")} @@ -413,9 +413,6 @@ def test_a_term_read_writes_no_size_and_closes_nothing( assert {r.source_id for r in store.roles(limit=50)} == open_before assert settled["closed"] == 0 assert store.state["ats:size:smartrecruiters:bfl"] == "1000" - # A board sized by its open roles is never sized by a term read. - del store.state["ats:size:smartrecruiters:bfl"] - assert reach.sizes(store, store.subscriptions(source="ats")) == {} # ------------------------------------------------------- storage and the API diff --git a/tests/test_reach.py b/tests/test_reach.py index ba7a63a..3d29918 100644 --- a/tests/test_reach.py +++ b/tests/test_reach.py @@ -185,14 +185,14 @@ def test_small_unknown_and_overridden_boards_get_no_reach( assert (found.params() is None) == (why != "agents") -def test_a_board_read_before_sizes_were_kept_is_sized_by_its_open_roles(store: Store) -> None: +def test_a_board_has_only_the_size_it_was_given(store: Store) -> None: _board(store, "workday:acme.wd3/Jobs", "Acme", size=None) [sub] = store.subscriptions(source="ats") + # A board read whole before sizes were kept is not sized by its open roles. store.state["ats:ok:workday:acme.wd3/Jobs"] = "run-0" - assert reach.sizes(store, [sub]) == {"workday:acme.wd3/Jobs": 0} - # A filtered board's open roles are not its size. - store.state["ats:filtered:workday:acme.wd3/Jobs"] = "1" assert reach.sizes(store, [sub]) == {} + store.state["ats:size:workday:acme.wd3/Jobs"] = "12" + assert reach.sizes(store, [sub]) == {"workday:acme.wd3/Jobs": 12} def test_describe_is_short() -> None: @@ -237,8 +237,8 @@ def test_the_pull_hands_each_board_its_reach_and_the_runner_narrows_it( assert asked["BoschGroup"] == Ask(countries=("DK", "SE")) assert asked["tiny"] == Ask() assert store.state["ats:filtered:smartrecruiters:BoschGroup"] == "1" - # The small board, read whole, keeps its size; the filtered one keeps its last. - assert store.state["ats:size:smartrecruiters:tiny"] == "1" + # A pull rewrites no size: the small board read whole keeps its 12. + assert store.state["ats:size:smartrecruiters:tiny"] == "12" assert store.state["ats:size:smartrecruiters:BoschGroup"] == "1000" diff --git a/tests/test_sources.py b/tests/test_sources.py index 26e01cc..0dbcafc 100644 --- a/tests/test_sources.py +++ b/tests/test_sources.py @@ -29,24 +29,24 @@ def store(tmp_path: Path) -> Store: class Boards: - """A fake `fetch_board`: `listed` maps a target to how many roles it - lists; any other target does not answer. Remembers what was fetched.""" + """A fake `board_size`: `listed` maps a target to how many roles it + lists; any other target does not answer. Remembers what was sized.""" def __init__(self, **listed: int) -> None: self.listed = {k.replace("_", ":", 1): v for k, v in listed.items()} self.fetched: list[str] = [] - def __call__(self, target: str) -> tuple[list[dict[str, Any]], bool]: + def __call__(self, target: str, **kwargs: Any) -> int: self.fetched.append(target) if target not in self.listed: raise OSError("connection refused") - return [{"source_id": str(i)} for i in range(self.listed[target])], True + return self.listed[target] @pytest.fixture def boards(monkeypatch: pytest.MonkeyPatch) -> Boards: fake = Boards(greenhouse_acme=3, ashby_acme=2, lever_quiet=0, lever_acme=1) - monkeypatch.setattr(ats, "fetch_board", fake) + monkeypatch.setattr(ats, "board_size", fake) return fake diff --git a/tests/test_web.py b/tests/test_web.py index 0ba1b6e..0d7d0b3 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -603,7 +603,7 @@ def test_companies_lists_board_and_open_role_count( "careers_opened_at": "", "failing_since": "", "failing_reason": "", "from_search": False, "hand_reason": "", "by_hand": False, "all_countries": False, "all_roles": False, - "reach_params": {}, "reach": "size not known yet: read whole"} + "reach_params": {}, "reach": "size not known yet: sized on the next pull"} [other] = [c for c in res.json() if c["key"] == "other"] assert (other["board"], other["subscription_id"], other["on"], other["ats"]) == ( "", None, None, "") @@ -1371,7 +1371,7 @@ def test_a_board_is_followed_only_if_it_answers( client: TestClient, store: Store, queue: Any, monkeypatch: pytest.MonkeyPatch) -> None: import plugins.ats as ats - monkeypatch.setattr(ats, "fetch_board", lambda target: ([{"source_id": "1"}], True)) + monkeypatch.setattr(ats, "board_size", lambda target, **kw: 1) res = client.post("/api/boards", json={"board": "ashby:acme", "name": "Acme"}) assert res.status_code == 201 assert res.json()["found"] == 1 and res.json()["source"]["target"] == "ashby:acme" @@ -1389,10 +1389,10 @@ def test_a_board_is_followed_only_if_it_answers( bad = client.post("/api/boards", json={"board": "acme", "name": "Other"}) assert bad.status_code == 422 and "host:slug" in bad.json()["detail"] - def down(target: str) -> list[dict[str, str]]: + def down(target: str, **kw: object) -> int: raise OSError("connection refused") - monkeypatch.setattr(ats, "fetch_board", down) + monkeypatch.setattr(ats, "board_size", down) dead = client.post("/api/boards", json={"board": "ashby:other", "name": "Other"}) assert dead.status_code == 422 and "did not answer" in dead.json()["detail"] assert store.company("other") is None @@ -1402,7 +1402,7 @@ def test_a_companys_board_is_set_replaced_and_detached( client: TestClient, store: Store, queue: Any, monkeypatch: pytest.MonkeyPatch) -> None: import plugins.ats as ats - monkeypatch.setattr(ats, "fetch_board", lambda target: ([], True)) # between openings + monkeypatch.setattr(ats, "board_size", lambda target, **kw: 0) # between openings store.add_company("Acme", key="acme", ats="teamtailor", ats_slug="acme") res = client.post("/api/companies/acme/board", json={"board": "greenhouse:acme"}) @@ -1522,7 +1522,7 @@ def test_a_company_with_a_board_keeps_its_name_and_takes_the_other_edits( client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None: import plugins.ats as ats - monkeypatch.setattr(ats, "fetch_board", lambda target: ([], True)) + monkeypatch.setattr(ats, "board_size", lambda target, **kw: 0) store.add_company("Acme", key="acme") assert client.post("/api/companies/acme/board", json={"board": "greenhouse:acme"} ).status_code == 200 @@ -1563,7 +1563,7 @@ def test_a_company_is_deleted_unless_it_is_read( client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None: import plugins.ats as ats - monkeypatch.setattr(ats, "fetch_board", lambda target: ([], True)) + monkeypatch.setattr(ats, "board_size", lambda target, **kw: 0) agent = store.save_agent("A") store.add_company("Meta", key="meta", hand_reason="blocked") store.set_affinity(agent.id, "meta", 1)