Fetch terms narrow what big boards read with the wide reach send (WP F2)
An agent has fetch_terms (migration 23, API, a chip field in the agent
editor). The terms of a pull are the union over the enabled agents, none
when one of them has none. They go only to wide reads: a company tagged All
roles, or a reach ask_for drops as too many countries for a one host; a
board's {"terms": false} switches them off. SmartRecruiters q,
SuccessFactors q, Workday searchText, Phenom keywords, Eightfold query and
Oracle keyword declare query="one"; the runner makes one series per term
times the country series, drops the places first over MAX_SERIES, then the
terms. A term read is narrowed: it writes no ats:size, closes nothing and
prunes no per-role state, and the 10% guard holds it to its own
fingerprint. The Companies reach line counts the terms.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3917b31209
commit
0fd45b333a
28 files changed
+839
-113
No files matched your search
+100
-25
@@ -124,7 +124,9 @@ class Listing:
|
||||
`source_id`s the read listed: this pull's, or the whole sweep's once it
|
||||
reached the end. `complete` says `ids` are the whole list. `sweep` is the
|
||||
state to keep for the next pull, None when no sweep is under way.
|
||||
`filtered` says the read asked the host for a part of the list."""
|
||||
`filtered` says the read asked the host for the roles in some places,
|
||||
`narrowed` for the roles matching words (`narrowed()`): the ids of
|
||||
neither are the whole board."""
|
||||
|
||||
rows: list[Row]
|
||||
complete: bool
|
||||
@@ -132,6 +134,7 @@ class Listing:
|
||||
filtered: bool = False
|
||||
sweep: dict[str, Any] | None = None
|
||||
asks: str = ""
|
||||
narrowed: bool = False
|
||||
|
||||
|
||||
# --------------------------------------------------------------- modules
|
||||
@@ -289,11 +292,39 @@ def resolve_key(key: Key, keys: Mapping[str, Any] | None) -> dict[str, str]:
|
||||
return out
|
||||
|
||||
|
||||
#: The most series a reach may cost on a `one` filter: a reach of more
|
||||
#: countries (the wide reach, Europe) reads such a host unfiltered instead.
|
||||
#: The most series a reach may cost: a reach of more countries on a `one`
|
||||
#: filter (the wide reach, Europe) reads such a host unfiltered instead, and
|
||||
#: fetch terms times the reach's series may not cost more either.
|
||||
MAX_SERIES = 8
|
||||
|
||||
|
||||
def _terms(value: Any) -> tuple[str, ...]:
|
||||
"""Fetch terms as given in a reach: stripped, empty ones dropped, one of
|
||||
each regardless of case, in the order given."""
|
||||
seen: set[str] = set()
|
||||
out: list[str] = []
|
||||
for term in _values(value):
|
||||
if term.casefold() not in seen:
|
||||
seen.add(term.casefold())
|
||||
out.append(term)
|
||||
return tuple(out)
|
||||
|
||||
|
||||
def _cost(reads: Reads, wanted: Mapping[str, Any]) -> int:
|
||||
"""How many series the location part of `wanted` takes on this reader:
|
||||
one per value of a `one` dimension (the product of two), one for a
|
||||
`many` or `ids` dimension, and one more for remote beside places, which
|
||||
the runner asks as a series of its own (see `series`)."""
|
||||
count = 1
|
||||
for dimension in ("country", "place"):
|
||||
values = wanted.get(dimension, ())
|
||||
if values and getattr(reads.filters, dimension) == "one":
|
||||
count *= len(values)
|
||||
if wanted.get("remote") and (wanted.get("country") or wanted.get("place")):
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None = None,
|
||||
reach: Mapping[str, Any] | None = None, *,
|
||||
keys: Mapping[str, Any] | None = None) -> Ask:
|
||||
@@ -306,30 +337,63 @@ def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None =
|
||||
dropped. A `remote` of False asks nothing. A search's params go whole into
|
||||
`Ask.params`; a reader with `READS.key` gets the resolved key under
|
||||
`Ask.params["key"]` (see `resolve_key`, which raises when it is missing).
|
||||
A module without `READS` gets `Ask()`."""
|
||||
A module without `READS` gets `Ask()`.
|
||||
|
||||
Fetch terms (`reach["terms"]`) go only to a board reader that declares
|
||||
`query="one"`, and only on a wide read: `reach["wide"]` (a company
|
||||
tagged All roles), or a reach this rule just dropped as too many
|
||||
countries. A board whose params say `"terms": false` or name a query of
|
||||
their own takes none. Terms times the reach's series (`_cost`) may cost
|
||||
at most `MAX_SERIES`: over it, the place part of the reach goes first,
|
||||
whole, since remote without its countries would read only the remote
|
||||
roles; over it with terms alone, the terms go too and the board is read
|
||||
unfiltered."""
|
||||
if not declared(module):
|
||||
return Ask()
|
||||
reads: Reads = module.READS
|
||||
params = dict(subscription_params or {})
|
||||
reached = _dimensions(module, reach or {})
|
||||
given = reach or {}
|
||||
reached = _dimensions(module, given)
|
||||
wide = bool(given.get("wide"))
|
||||
if reads.filters.country == "one" and len(reached.get("country", ())) > MAX_SERIES:
|
||||
reached = {}
|
||||
wide = True
|
||||
wanted = reached | _dimensions(module, params)
|
||||
if wanted.get("remote") is False:
|
||||
del wanted["remote"]
|
||||
fields: dict[str, Any] = {_ASK_FIELD[name]: value for name, value in wanted.items()
|
||||
if name in reads.filters.declared()}
|
||||
declared_now = reads.filters.declared()
|
||||
wanted = {name: value for name, value in wanted.items() if name in declared_now}
|
||||
terms = _terms(given.get("terms"))
|
||||
if (not wide or reads.scope != "board" or reads.filters.query != "one"
|
||||
or params.get("terms") is False or "query" in wanted):
|
||||
terms = ()
|
||||
if terms and len(terms) * _cost(reads, wanted) > MAX_SERIES:
|
||||
wanted = {name: value for name, value in wanted.items()
|
||||
if name not in ("country", "place", "remote")}
|
||||
if len(terms) > MAX_SERIES:
|
||||
terms = ()
|
||||
fields: dict[str, Any] = {_ASK_FIELD[name]: value for name, value in wanted.items()}
|
||||
extra: dict[str, Any] = dict(params) if reads.scope == "search" else {}
|
||||
if reads.key is not None:
|
||||
extra["key"] = resolve_key(reads.key, keys)
|
||||
return Ask(**fields, params=extra)
|
||||
return Ask(**fields, terms=terms, params=extra)
|
||||
|
||||
|
||||
def filtered(ask: Ask) -> bool:
|
||||
"""Whether `ask` reads a part of a board: a country, a place or remote."""
|
||||
"""Whether `ask` reads a part of a board by place: a country, a place or
|
||||
remote. Terms are `narrowed`."""
|
||||
return bool(ask.countries or ask.places or ask.remote is not None)
|
||||
|
||||
|
||||
def narrowed(module: ModuleType, ask: Ask) -> bool:
|
||||
"""Whether `ask` reads a board by words: fetch terms, or a board's own
|
||||
query. Such a read is never the whole board, nor the board within its
|
||||
places, so it sets no size and closes nothing. A search's query is what
|
||||
the search is, and does not count."""
|
||||
board = not declared(module) or module.READS.scope == "board"
|
||||
return bool(ask.terms) or (board and bool(ask.query))
|
||||
|
||||
|
||||
def series(module: ModuleType, ask: Ask,
|
||||
facets: Mapping[str, Mapping[str, str]] | None = None) -> list[Ask]:
|
||||
"""The series of requests `ask` takes on this reader. Remote with a
|
||||
@@ -340,15 +404,22 @@ def series(module: ModuleType, ask: Ask,
|
||||
values `facets` does not know are dropped, and a part with none left is
|
||||
no series, since the host has nothing of it; a dimension `facets` does
|
||||
not state at all is read unfiltered, and so is remote alone, while the
|
||||
remote part beside places is then left out. `[ask]` for a module
|
||||
without `READS`."""
|
||||
remote part beside places is then left out. Each of those is then asked
|
||||
once per fetch term (`ask.terms`), with the term as `query`: the host
|
||||
intersects a keyword with its facets, as the survey found on every
|
||||
host, so the union by id is "one of the terms, in these places". `[ask]`
|
||||
for a module without `READS`."""
|
||||
if not declared(module):
|
||||
return [ask]
|
||||
if ask.remote and (ask.countries or ask.places):
|
||||
return [*_series(module, ask.model_copy(update={"remote": None}), facets),
|
||||
*_series(module, ask.model_copy(update={"countries": (), "places": ()}),
|
||||
facets, beside=True)]
|
||||
return _series(module, ask, facets)
|
||||
parts = [*_series(module, ask.model_copy(update={"remote": None}), facets),
|
||||
*_series(module, ask.model_copy(update={"countries": (), "places": ()}),
|
||||
facets, beside=True)]
|
||||
else:
|
||||
parts = _series(module, ask, facets)
|
||||
if not ask.terms:
|
||||
return parts
|
||||
return [part.model_copy(update={"query": term}) for term in ask.terms for part in parts]
|
||||
|
||||
|
||||
def _series(module: ModuleType, ask: Ask, facets: Mapping[str, Mapping[str, str]] | None,
|
||||
@@ -455,12 +526,13 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
||||
ask = ask or Ask()
|
||||
asks = series(module, ask, facets)
|
||||
is_filtered = filtered(ask)
|
||||
by_words = narrowed(module, ask)
|
||||
key = fingerprint(asks)
|
||||
if not asks:
|
||||
return Listing([], True, [], is_filtered, None, key)
|
||||
return Listing([], True, [], is_filtered, None, key, by_words)
|
||||
if not hasattr(module, "page"):
|
||||
rows = _whole(module, slug, get, asks, limits)
|
||||
return Listing(list(rows.values()), True, list(rows), is_filtered, None, key)
|
||||
return Listing(list(rows.values()), True, list(rows), is_filtered, None, key, by_words)
|
||||
|
||||
reads: Reads | None = module.READS if declared(module) else None
|
||||
never = reads is not None and reads.complete == "never"
|
||||
@@ -518,15 +590,15 @@ def read_listing(module: ModuleType, slug: str, get: Get, ask: Ask | None = None
|
||||
|
||||
listed = list(dict.fromkeys(earlier + list(rows)))
|
||||
if not stop:
|
||||
return Listing(list(rows.values()), whole, listed, is_filtered, None, key)
|
||||
return Listing(list(rows.values()), whole, listed, is_filtered, None, key, by_words)
|
||||
log.warning("readers: %s:%s: stopped after %d pages (%s); not complete",
|
||||
module.NAME, slug, pages, stop)
|
||||
at_ceiling = ceiling is not None and len(seen) >= ceiling
|
||||
if not can_sweep or never or at_ceiling or not _storable(cursor):
|
||||
return Listing(list(rows.values()), False, list(rows), is_filtered, None, key)
|
||||
return Listing(list(rows.values()), False, list(rows), is_filtered, None, key, by_words)
|
||||
keep = {"asks": key, "series": index, "cursor": cursor, "seen": sorted(seen),
|
||||
"rows": listed, "whole": whole}
|
||||
return Listing(list(rows.values()), False, list(rows), is_filtered, keep, key)
|
||||
return Listing(list(rows.values()), False, list(rows), is_filtered, keep, key, by_words)
|
||||
|
||||
|
||||
def _storable(cursor: Any) -> bool:
|
||||
@@ -577,9 +649,10 @@ def read(state: MutableMapping[str, str], prefix: str, target: str, module: Modu
|
||||
limits: Limits | None = None, can_sweep: bool = True) -> Listing:
|
||||
"""`read_listing` for one target with its state under `<prefix>:...:<target>`:
|
||||
the facets it needs, the sweep it continues, and the 10% guard. A read
|
||||
that is filtered or whose reader declares `empty="suspect"`, and that
|
||||
listed under `SUSPECT_SHARE` of what the same read listed last time,
|
||||
raises `Suspect`. Nothing is written here but the facets; `stamp` keeps
|
||||
that is filtered, narrowed by terms, or whose reader declares
|
||||
`empty="suspect"`, and that listed under `SUSPECT_SHARE` of what the same
|
||||
read listed last time (the same `fingerprint`, so a term read is only
|
||||
held to a term read with the same terms and places), raises `Suspect`. Nothing is written here but the facets; `stamp` keeps
|
||||
the rest once the rows are stored."""
|
||||
ask = ask or Ask()
|
||||
facets = facets_for(module, slug, get, ask, state, f"{prefix}:facets:{target}")
|
||||
@@ -588,7 +661,7 @@ def read(state: MutableMapping[str, str], prefix: str, target: str, module: Modu
|
||||
suspect = declared(module) and module.READS.empty == "suspect"
|
||||
last = _load(state, f"{prefix}:count:{target}")
|
||||
count = int(last.get("count") or 0) if last.get("asks") == listing.asks else 0
|
||||
if ((listing.filtered or suspect) and listing.sweep is None and count
|
||||
if ((listing.filtered or listing.narrowed or suspect) and listing.sweep is None and count
|
||||
and len(listing.ids) < SUSPECT_SHARE * count):
|
||||
raise Suspect(f"listed {len(listing.ids)} roles, under "
|
||||
f"{SUSPECT_SHARE:.0%} of the {count} it listed last time")
|
||||
@@ -599,12 +672,14 @@ def stamp(state: MutableMapping[str, str], prefix: str, target: str, run: str,
|
||||
listing: Listing, ids: list[str]) -> None:
|
||||
"""Record a read whose rows are stored: `ok` (the run's marker),
|
||||
`complete`, `ids` (the compound ids, what the pull closes against),
|
||||
`filtered`, the sweep to continue or none, and the count for the 10%
|
||||
`filtered`, `narrowed` (a read by terms, which the pull never closes
|
||||
against), the sweep to continue or none, and the count for the 10%
|
||||
guard once a read is whole."""
|
||||
state[f"{prefix}:ok:{target}"] = run
|
||||
state[f"{prefix}:complete:{target}"] = "1" if listing.complete else ""
|
||||
state[f"{prefix}:ids:{target}"] = json.dumps(ids)
|
||||
state[f"{prefix}:filtered:{target}"] = "1" if listing.filtered else ""
|
||||
state[f"{prefix}:narrowed:{target}"] = "1" if listing.narrowed else ""
|
||||
if listing.sweep is not None:
|
||||
state[f"{prefix}:sweep:{target}"] = json.dumps(listing.sweep)
|
||||
else:
|
||||
|
||||
Reference in new issue
Block a user