Fetch terms narrow what big boards read with the wide reach send (WP F2)

An agent has fetch_terms (migration 23, API, a chip field in the agent
editor). The terms of a pull are the union over the enabled agents, none
when one of them has none. They go only to wide reads: a company tagged All
roles, or a reach ask_for drops as too many countries for a one host; a
board's {"terms": false} switches them off. SmartRecruiters q,
SuccessFactors q, Workday searchText, Phenom keywords, Eightfold query and
Oracle keyword declare query="one"; the runner makes one series per term
times the country series, drops the places first over MAX_SERIES, then the
terms. A term read is narrowed: it writes no ats:size, closes nothing and
prunes no per-role state, and the 10% guard holds it to its own
fingerprint. The Companies reach line counts the terms.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 11:59:27 +02:00
1 parent 3917b31209
commit 0fd45b333a
28 files changed
+839 -113

No files matched your search

+100 -25
View File
@@ -124,7 +124,9 @@ class Listing:
`source_id`s the read listed: this pull's, or the whole sweep's once it
reached the end. `complete` says `ids` are the whole list. `sweep` is the
state to keep for the next pull, None when no sweep is under way.
`filtered` says the read asked the host for a part of the list."""
`filtered` says the read asked the host for the roles in some places,
`narrowed` for the roles matching words (`narrowed()`): the ids of
neither are the whole board."""
rows: list[Row]
complete: bool
@@ -132,6 +134,7 @@ class Listing:
filtered: bool = False
sweep: dict[str, Any] | None = None
asks: str = ""
narrowed: bool = False
# --------------------------------------------------------------- modules
@@ -289,11 +292,39 @@ def resolve_key(key: Key, keys: Mapping[str, Any] | None) -> dict[str, str]:
return out
#: The most series a reach may cost on a `one` filter: a reach of more
#: countries (the wide reach, Europe) reads such a host unfiltered instead.
#: The most series a reach may cost: a reach of more countries on a `one`
#: filter (the wide reach, Europe) reads such a host unfiltered instead, and
#: fetch terms times the reach's series may not cost more either.
MAX_SERIES = 8
def _terms(value: Any) -> tuple[str, ...]:
"""Fetch terms as given in a reach: stripped, empty ones dropped, one of
each regardless of case, in the order given."""
seen: set[str] = set()
out: list[str] = []
for term in _values(value):
if term.casefold() not in seen:
seen.add(term.casefold())
out.append(term)
return tuple(out)
def _cost(reads: Reads, wanted: Mapping[str, Any]) -> int:
"""How many series the location part of `wanted` takes on this reader:
one per value of a `one` dimension (the product of two), one for a
`many` or `ids` dimension, and one more for remote beside places, which
the runner asks as a series of its own (see `series`)."""
count = 1
for dimension in ("country", "place"):
values = wanted.get(dimension, ())
if values and getattr(reads.filters, dimension) == "one":
count *= len(values)
if wanted.get("remote") and (wanted.get("country") or wanted.get("place")):
count += 1
return count
def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None = None,
reach: Mapping[str, Any] | None = None, *,
keys: Mapping[str, Any] | None = None) -> Ask:
@@ -306,30 +337,63 @@ def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None =
dropped. A `remote` of False asks nothing. A search's params go whole into
`Ask.params`; a reader with `READS.key` gets the resolved key under
`Ask.params["key"]` (see `resolve_key`, which raises when it is missing).
A module without `READS` gets `Ask()`."""
A module without `READS` gets `Ask()`.
Fetch terms (`reach["terms"]`) go only to a board reader that declares
`query="one"`, and only on a wide read: `reach["wide"]` (a company
tagged All roles), or a reach this rule just dropped as too many
countries. A board whose params say `"terms": false` or name a query of
their own takes none. Terms times the reach's series (`_cost`) may cost
at most `MAX_SERIES`: over it, the place part of the reach goes first,
whole, since remote without its countries would read only the remote
roles; over it with terms alone, the terms go too and the board is read
unfiltered."""
if not declared(module):
return Ask()
reads: Reads = module.READS
params = dict(subscription_params or {})
reached = _dimensions(module, reach or {})
given = reach or {}
reached = _dimensions(module, given)
wide = bool(given.get("wide"))
if reads.filters.country == "one" and len(reached.get("country", ())) > MAX_SERIES:
reached = {}
wide = True
wanted = reached | _dimensions(module, params)
if wanted.get("remote") is False:
del wanted["remote"]
fields: dict[str, Any] = {_ASK_FIELD[name]: value for name, value in wanted.items()
if name in reads.filters.declared()}
declared_now = reads.filters.declared()
wanted = {name: value for name, value in wanted.items() if name in declared_now}
terms = _terms(given.get("terms"))
if (not wide or reads.scope != "board" or reads.filters.query != "one"
or params.get("terms") is False or "query" in wanted):
terms = ()
if terms and len(terms) * _cost(reads, wanted) > MAX_SERIES:
wanted = {name: value for name, value in wanted.items()
if name not in ("country", "place", "remote")}
if len(terms) > MAX_SERIES:
terms = ()
fields: dict[str, Any] = {_ASK_FIELD[name]: value for name, value in wanted.items()}
extra: dict[str, Any] = dict(params) if reads.scope == "search" else {}
if reads.key is not None:
extra["key"] = resolve_key(reads.key, keys)
return Ask(**fields, params=extra)
return Ask(**fields, terms=terms, params=extra)
def filtered(ask: Ask) -> bool:
"""Whether `ask` reads a part of a board: a country, a place or remote."""
"""Whether `ask` reads a part of a board by place: a country, a place or
remote. Terms are `narrowed`."""
return bool(ask.countries or ask.places or ask.remote is not None)
def narrowed(module: ModuleType, ask: Ask) -> bool:
"""Whether `ask` reads a board by words: fetch terms, or a board's own
query. Such a read is never the whole board, nor the board within its
places, so it sets no size and closes nothing. A search's query is what
the search is, and does not count."""
board = not declared(module) or module.READS.scope == "board"
return bool(ask.terms) or (board and bool(ask.query))
def series(module: ModuleType, ask: Ask,
facets: Mapping[str, Mapping[str, str]] | None = None) -> list[Ask]:
"""The series of requests `ask` takes on this reader. Remote with a
@@ -340,15 +404,22 @@ def series(module: ModuleType, ask: Ask,
values `facets` does not know are dropped, and a part with none left is
no series, since the host has nothing of it; a dimension `facets` does
not state at all is read unfiltered, and so is remote alone, while the
remote part beside places is then left out. `[ask]` for a module
without `READS`."""
remote part beside places is then left out. Each of those is then asked
once per fetch term (`ask.terms`), with the term as `query`: the host
intersects a keyword with its facets, as the survey found on every
host, so the union by id is "one of the terms, in these places". `[ask]`
for a module without `READS`."""
if not declared(module):
return [ask]
if ask.remote and (ask.countries or ask.places):
return [*_series(module, ask.model_copy(update={"remote": None}), facets),
*_series(module, ask.model_copy(update={"countries": (), "places": ()}),
facets, beside=True)]
return _series(module, ask, facets)
parts = [*_series(module, ask.model_copy(update={"remote": None}), facets),
*_series(module, ask.model_copy(update={"countries": (), "places": ()}),
facets, beside=True)]
else:
parts = _series(module, ask, facets)
if not ask.terms:
return parts
return [part.model_copy(update={"query": term}) for term in ask.terms for part in parts]
def _series(module: ModuleType, ask: Ask, facets: Mapping[str, Mapping[str, str]] | None,
@@ -455,12 +526,13 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
ask = ask or Ask()
asks = series(module, ask, facets)
is_filtered = filtered(ask)
by_words = narrowed(module, ask)
key = fingerprint(asks)
if not asks:
return Listing([], True, [], is_filtered, None, key)
return Listing([], True, [], is_filtered, None, key, by_words)
if not hasattr(module, "page"):
rows = _whole(module, slug, get, asks, limits)
return Listing(list(rows.values()), True, list(rows), is_filtered, None, key)
return Listing(list(rows.values()), True, list(rows), is_filtered, None, key, by_words)
reads: Reads | None = module.READS if declared(module) else None
never = reads is not None and reads.complete == "never"
@@ -518,15 +590,15 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
listed = list(dict.fromkeys(earlier + list(rows)))
if not stop:
return Listing(list(rows.values()), whole, listed, is_filtered, None, key)
return Listing(list(rows.values()), whole, listed, is_filtered, None, key, by_words)
log.warning("readers: %s:%s: stopped after %d pages (%s); not complete",
module.NAME, slug, pages, stop)
at_ceiling = ceiling is not None and len(seen) >= ceiling
if not can_sweep or never or at_ceiling or not _storable(cursor):
return Listing(list(rows.values()), False, list(rows), is_filtered, None, key)
return Listing(list(rows.values()), False, list(rows), is_filtered, None, key, by_words)
keep = {"asks": key, "series": index, "cursor": cursor, "seen": sorted(seen),
"rows": listed, "whole": whole}
return Listing(list(rows.values()), False, list(rows), is_filtered, keep, key)
return Listing(list(rows.values()), False, list(rows), is_filtered, keep, key, by_words)
def _storable(cursor: Any) -> bool:
@@ -577,9 +649,10 @@ def read(state: MutableMapping[str, str], prefix: str, target: str, module: Modu
limits: Limits | None = None, can_sweep: bool = True) -> Listing:
"""`read_listing` for one target with its state under `<prefix>:...:<target>`:
the facets it needs, the sweep it continues, and the 10% guard. A read
that is filtered or whose reader declares `empty="suspect"`, and that
listed under `SUSPECT_SHARE` of what the same read listed last time,
raises `Suspect`. Nothing is written here but the facets; `stamp` keeps
that is filtered, narrowed by terms, or whose reader declares
`empty="suspect"`, and that listed under `SUSPECT_SHARE` of what the same
read listed last time (the same `fingerprint`, so a term read is only
held to a term read with the same terms and places), raises `Suspect`. Nothing is written here but the facets; `stamp` keeps
the rest once the rows are stored."""
ask = ask or Ask()
facets = facets_for(module, slug, get, ask, state, f"{prefix}:facets:{target}")
@@ -588,7 +661,7 @@ def read(state: MutableMapping[str, str], prefix: str, target: str, module: Modu
suspect = declared(module) and module.READS.empty == "suspect"
last = _load(state, f"{prefix}:count:{target}")
count = int(last.get("count") or 0) if last.get("asks") == listing.asks else 0
if ((listing.filtered or suspect) and listing.sweep is None and count
if ((listing.filtered or listing.narrowed or suspect) and listing.sweep is None and count
and len(listing.ids) < SUSPECT_SHARE * count):
raise Suspect(f"listed {len(listing.ids)} roles, under "
f"{SUSPECT_SHARE:.0%} of the {count} it listed last time")
@@ -599,12 +672,14 @@ def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str,
listing: Listing, ids: list[str]) -> None:
"""Record a read whose rows are stored: `ok` (the run's marker),
`complete`, `ids` (the compound ids, what the pull closes against),
`filtered`, the sweep to continue or none, and the count for the 10%
`filtered`, `narrowed` (a read by terms, which the pull never closes
against), the sweep to continue or none, and the count for the 10%
guard once a read is whole."""
state[f"{prefix}:ok:{target}"] = run
state[f"{prefix}:complete:{target}"] = "1" if listing.complete else ""
state[f"{prefix}:ids:{target}"] = json.dumps(ids)
state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else ""
state[f"{prefix}:narrowed:{target}"] = "1" if listing.narrowed else ""
if listing.sweep is not None:
state[f"{prefix}:sweep:{target}"] = json.dumps(listing.sweep)
else: