Size a board once with board_size, before a pull reads it with no size, and name what a site's ids leave out

set_board sizes a board from page 1's total, or reads to the end on a host that states none, instead of reading it whole up to page 100. A pull no longer rewrites ats:size; a board with none is handed the agents' reach marked unsized and is sized in its batch first, or by its whole read on a host that cannot narrow the reach. reach.sizes drops the open-role fallback. The Companies line names countries a Workday or Oracle site does not list and a missing remote option.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 13:43:52 +02:00
1 parent a19c528e3f
commit a0882e9659
8 files changed
+579 -67

No files matched your search

+113 -19
View File
@@ -21,8 +21,13 @@ on. Computed once per pull (`plan`) and handed to each board's batch as the
it skips the `any` gates; it does not widen what is fetched.
- **Board override**: a board whose subscription `params` name a country, a
place or remote gets no reach here; `ask_for` reads the params instead.
- **Small boards**: a board that listed under `SMALL` roles when last read
whole gets no filter; filtering it saves little and risks closing roles.
- **Small boards**: a board of under `SMALL` roles gets no filter;
filtering it saves little and risks closing roles. A board's size is set
once, when it is added or rechecked (`plugins/ats.py`, `board_size`), and
never by a pull. A board with no size yet is handed the reach it would
have as a big one, with `unsized` and `small`, and its batch sizes it
before it reads it, so a big new board is not read whole first; a board
whose reader cannot narrow that reach is sized by its whole read instead.
A reach of more countries than a `one`-filter host should be asked one by
one (the wide reach's Europe on SmartRecruiters) is dropped by `ask_for`,
@@ -43,14 +48,15 @@ board sends the roles that match one of them rather than every role:
changes nothing on the narrow reach today, which takes no terms.
- Only a reader that declares `query="one"` takes them, one series a term;
`ask_for` has the rule for terms times the reach's series.
- A read with terms is never the whole board: it sets no `ats:size`, and
the pull closes nothing against it (`app/jobs/pull.py`).
- A read with terms is never the whole board: the pull closes nothing
against it (`app/jobs/pull.py`).
- The search providers take none. Every saved search has a query of its
own, which terms would replace; a search is a query already.
"""
from __future__ import annotations
import json
from collections.abc import Iterable, Mapping
from dataclasses import dataclass, field
from typing import Any
@@ -65,9 +71,15 @@ from plugins.readers import _runner
#: whole.
SMALL = 300
#: `state` key of a board's size, `ats:size:<target>`: the roles its last
#: unfiltered read listed, written by `plugins/ats.py`.
#: `state` keys of a board's size, `ats:size:<target>`, the roles it listed
#: when it was sized, and of the day it was sized (ISO), both written by
#: `plugins/ats.py` (`keep_size`).
SIZE = "ats:size:"
SIZE_CHECKED = "ats:sizechecked:"
#: `state` key of the host ids a board's site states (`{"checked",
#: "facets"}`), kept by the runner (`_runner.facets_for`).
FACETS = "ats:facets:"
#: The params that make a board's own reach (see `_runner._PARAM_DIMENSION`).
OVERRIDE_KEYS = frozenset(("countries", "country", "places", "place", "remote"))
@@ -239,7 +251,9 @@ class Plan:
"""The reach of one pull: `base` for every board, `wide` for the boards
of the companies in `all_countries`, each board's size by target, the
fetch `terms` of the wide reads, which skip the companies in
`all_roles`, and `wide_names`, what the agents call the wide reach."""
`all_roles`, `wide_names`, what the agents call the wide reach, and
`facets`, the host ids the runner keeps for a board on a host that
filters by them, for the Companies line."""
base: Reach
wide: Reach = EVERYWHERE
@@ -248,6 +262,7 @@ class Plan:
terms: tuple[str, ...] = ()
all_roles: frozenset[str] = frozenset()
wide_names: str = ""
facets: dict[str, dict[str, dict[str, str]]] = field(default_factory=dict)
def why(self, sub: Subscription) -> tuple[Reach, str]:
"""The reach `sub` is read with, and why: `override`, `small`,
@@ -260,6 +275,10 @@ class Plan:
return EVERYWHERE, "unknown"
if size < SMALL:
return EVERYWHERE, "small"
return self._big(sub)
def _big(self, sub: Subscription) -> tuple[Reach, str]:
"""The reach of `sub` as a board of `SMALL` roles or more."""
if company_key(sub.company or sub.label) in self.all_countries:
return self.wide, "countries"
return self.base, "agents"
@@ -274,30 +293,51 @@ class Plan:
fetch terms, and unless its company is set to All roles, a board on
the wide reach carries them and `wide`, and a board on the agents'
reach carries them for `ask_for` to use if it drops the countries as
too many (see the module docstring)."""
too many (see the module docstring). A board with no size carries
`unsized` and `small`, and the reach of a big board unless its params
set their own, for its batch to size it first."""
found, why = self.why(sub)
if why == "unknown":
found, why = self._big(sub)
out = found.params()
if self.terms and why in ("countries", "agents") and self.narrows_roles(sub):
out = (out or {}) | {"terms": list(self.terms)}
if why == "countries":
out["wide"] = True
if sub.target not in self.sizes:
out = (out or {}) | {"unsized": True, "small": SMALL}
return out
def sizes(store: Store, subs: Iterable[Subscription]) -> dict[str, int]:
"""Each board's size: the roles its last unfiltered read listed
(`ats:size:<target>`), else, for a board read before that was kept and
never filtered, its open roles. A board never read has none."""
"""Each board's size as kept (`ats:size:<target>`); a board never sized
has none."""
state = store.state
counts = store.open_countries_by_subscription()
out: dict[str, int] = {}
for sub in subs:
kept = state.get(f"{SIZE}{sub.target}", "")
if kept.isdigit():
out[sub.target] = int(kept)
elif (state.get(f"ats:ok:{sub.target}") and not state.get(f"ats:filtered:{sub.target}")
and not state.get(f"ats:narrowed:{sub.target}")):
out[sub.target] = sum(counts.get(sub.id, {}).values())
return out
def site_facets(store: Store, subs: Iterable[Subscription]) -> dict[str, dict[str, dict[str, str]]]:
"""The host ids the runner keeps for each board on a host whose country
or remote filter is `ids` (`ats:facets:<target>`); a board with none kept
is left out."""
out: dict[str, dict[str, dict[str, str]]] = {}
for sub in subs:
module = readers.HOSTS.get(sub.target.partition(":")[0])
if module is None or not _runner.declared(module) \
or "ids" not in (module.READS.filters.country, module.READS.filters.remote):
continue
try:
kept = json.loads(store.state.get(f"{FACETS}{sub.target}") or "{}")
except (TypeError, ValueError):
continue
found = kept.get("facets") if isinstance(kept, dict) else None
if isinstance(found, dict):
out[sub.target] = found
return out
@@ -321,7 +361,8 @@ def plan(store: Store, subs: Iterable[Subscription] | None = None) -> Plan:
all_countries=frozenset(c.key for c in rows if c.all_countries),
terms=fetch_terms(agents),
all_roles=frozenset(c.key for c in rows if c.all_roles),
wide_names=wide_names(agents, gaz) if agents else "everywhere")
wide_names=wide_names(agents, gaz) if agents else "everywhere",
facets=site_facets(store, subs))
def reaches(store: Store, subs: Iterable[Subscription]) -> Mapping[str, dict[str, Any]]:
@@ -338,14 +379,58 @@ _WHOLE = {"small": f"under {SMALL} roles", "unknown": "size not known yet",
"agents": "an agent takes any place"}
def _codes(codes: Iterable[str]) -> list[str]:
"""Country codes as the line names them: up to four, else a count."""
found = sorted(codes)
return found if len(found) <= 4 else [f"{len(found)} countries"]
def _places(ask: readers.Ask) -> str:
words = sorted(ask.countries) if len(ask.countries) <= 4 \
else [f"{len(ask.countries)} countries"]
words = _codes(ask.countries)
if ask.remote:
words.append("remote")
return ", ".join(words)
def _site(module: Any, ask: readers.Ask, facets: Mapping[str, Mapping[str, str]] | None,
lead: str, roles: str) -> str:
"""The line of a board whose site does not know every value `ask` wants
on an `ids` filter, from the host ids the runner keeps (`facets`, None
before any are kept), as `_runner.series` drops them: what is asked,
then what the site does not know ("DK · SE not on this site · no remote
option"), after `lead` and before `roles`; or, with nothing left to
ask, "site lists none of DK, SE: nothing asked". "" when the site knows
every value, or `lead` is "" and something is still asked."""
if facets is None:
return ""
reads = module.READS.filters
known, missing = list(ask.countries), []
if reads.country == "ids" and ask.countries and "country" in facets:
listed = facets.get("country") or {}
known = [c for c in ask.countries if c in listed]
missing = [c for c in ask.countries if c not in listed]
no_remote = bool(ask.remote) and reads.remote == "ids" and not facets.get("remote")
if not (missing or no_remote):
return ""
if no_remote and not (ask.countries or ask.places) and "remote" not in facets:
# Remote alone, on a site that states no remote facet, is read whole.
return f"all places, no remote option on this site · {roles}" if lead else ""
remote = bool(ask.remote) and not no_remote
if not (known or ask.places or remote):
if not missing:
return "site has no remote option: nothing asked"
named = _codes(missing) + (["remote"] if no_remote else [])
return f"site lists none of {', '.join(named)}: nothing asked"
if not lead:
return ""
words = [", ".join(_codes(known) + list(ask.places) + (["remote"] if remote else []))]
if missing:
words.append(f"{', '.join(_codes(missing))} not on this site")
if no_remote:
words.append("no remote option")
return f"{lead}: {' · '.join(w for w in words if w)} · {roles}"
def _roles(ask: readers.Ask) -> str:
"""The role-type half of the line: the agents' terms the board is
asked for, or all roles."""
@@ -363,14 +448,23 @@ def line(found: Plan, sub: Subscription) -> str:
· all roles", "set here: DE · all roles", "all places, too many countries
to ask · follows agents' terms: engineer", or, for a board read whole,
why: "under 300 roles: read whole", "Greenhouse cannot filter: read
whole"."""
whole". A board with no size yet says it is sized on the next pull. On
a host that filters by its own ids, the line names what the site does
not know once the runner has asked it (`_site`): "follows agents: DK ·
SE not on this site · no remote option · all roles", or "site lists
none of DK, SE: nothing asked"."""
module = readers.HOSTS.get(sub.target.partition(":")[0])
if module is None:
return ""
given, why = found.why(sub)
if why == "unknown":
return f"{_WHOLE[why]}: sized on the next pull"
ask = _runner.ask_for(module, sub.params, found.for_subscription(sub))
roles = _roles(ask)
if _runner.filtered(ask):
lead = {"agents": "follows agents", "override": "set here"}.get(why, "")
if site := _site(module, ask, found.facets.get(sub.target), lead, roles):
return site
if why == "countries":
named = found.wide_names or f"{len(found.wide.countries)} countries"
return f"all countries ({named}) · {roles}"
+29 -10
View File
@@ -46,10 +46,11 @@ def set_board(store: Store, key: str, board: str, *, name: str = "",
check: bool = True, **fields: str) -> tuple[Subscription, int | None, int]:
"""Read company `key` from `board` (`host:slug`): attach it to a company
that has none, replace the one it has, or create the company from `name`
when there is no row. The board is fetched first and refused if its host
is unknown or it does not answer; a board listing zero roles is kept, since
a company between openings is still one to follow. `check=False` skips the
fetch, for the seed, whose boards were read once when it was written.
when there is no row. The board is sized first (`board_size`) and refused
if its host is unknown or it does not answer; a board listing zero roles
is kept, since a company between openings is still one to follow.
`check=False` skips the request, for the seed, whose boards were read
once when it was written; such a board is sized by its first pull.
The row's `ats` and `ats_slug` become the board's host and slug, its
`careers_url` the board's page where it was empty or the old board's, and
@@ -59,7 +60,7 @@ def set_board(store: Store, key: str, board: str, *, name: str = "",
change in one transaction (`Store.set_company_board`), and replacing a
board closes the old one's open roles. Returns the subscription, how many
roles the board lists (None unchecked) and how many roles closed."""
from plugins.ats import careers_url, fetch_board, is_known_target
from plugins.ats import board_size, careers_url, is_known_target, keep_size
board, name = board.strip(), name.strip()
key = key.strip() or company_key(name)
@@ -72,10 +73,10 @@ def set_board(store: Store, key: str, board: str, *, name: str = "",
raise ValueError(f"no company {key!r}; a new company needs a name")
if company_key(name) != key:
raise ValueError(f"{name!r} is not the company {key!r}")
rows: list[Any] | None = None
size: int | None = None
if check:
try:
rows, _complete = fetch_board(board)
size = board_size(board)
except Exception as exc:
raise ValueError(f"{board} did not answer: {exc}") from exc
host, _, slug = board.partition(":")
@@ -95,7 +96,23 @@ def set_board(store: Store, key: str, board: str, *, name: str = "",
assert sub is not None
if before is not None and before.target != board:
_forget_failing(store, before.target)
return sub, None if rows is None else len(rows), closed
if size is not None:
keep_size(store.state, board, size)
return sub, size, closed
def size_board(store: Store, board: str) -> int:
"""Size `board` again (`plugins.ats.board_size`) and keep it with the
day: how a board's size changes once it was added (a Recheck, `app
companies size`). Raises ValueError when it does not answer."""
from plugins.ats import board_size, keep_size
try:
size = board_size(board)
except Exception as exc:
raise ValueError(f"{board} did not answer: {exc}") from exc
keep_size(store.state, board, size)
return size
def detach_board(store: Store, key: str) -> int:
@@ -118,9 +135,11 @@ def _forget_failing(store: Store, target: str) -> None:
record of `target` failing would otherwise outlive the board, and count
in `app companies check` for good. The `ats` plugin's robots.txt verdict
and its lists of roles read and failed in detail go with it, since they
are about that board alone and would only grow stale."""
are about that board alone and would only grow stale, as would its size,
which a board attached again gets anew."""
store.state.pop(FAILING + target, None)
for key in ("ats:robots:", "ats:detailed:", "ats:detailfailed:"):
for key in ("ats:robots:", "ats:detailed:", "ats:detailfailed:", "ats:size:",
"ats:sizechecked:"):
store.state.pop(key + target, None)