Fetch terms narrow what big boards read with the wide reach send (WP F2)

An agent has fetch_terms (migration 23, API, a chip field in the agent
editor). The terms of a pull are the union over the enabled agents, none
when one of them has none. They go only to wide reads: a company tagged All
roles, or a reach ask_for drops as too many countries for a one host; a
board's {"terms": false} switches them off. SmartRecruiters q,
SuccessFactors q, Workday searchText, Phenom keywords, Eightfold query and
Oracle keyword declare query="one"; the runner makes one series per term
times the country series, drops the places first over MAX_SERIES, then the
terms. A term read is narrowed: it writes no ats:size, closes nothing and
prunes no per-role state, and the 10% guard holds it to its own
fingerprint. The Companies reach line counts the terms.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 11:59:27 +02:00
1 parent 3917b31209
commit 0fd45b333a
28 files changed
+839 -113

No files matched your search

+7
View File
@@ -24,6 +24,11 @@ filtered read, or one of a reader that declares `empty="suspect"`, that lists
under a tenth of what the same read listed last time is recorded by the
runner as a failure, not as a list to close against.
A read narrowed by the agents' fetch terms (`ats:narrowed:<target>`) lists
the roles matching a word, which says nothing of the roles it did not list,
so it closes nothing; only an unfiltered or a place-filtered complete read
closes.
A search closes the same way when it listed everything that matches (see
`plugins/search.py` for how each provider knows). One that did not closes its
roles only once no search has seen them for `STALE_DAYS`. A board
@@ -274,6 +279,8 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
if source == "ats":
incomplete += 1
continue
if store.state.get(f"{source}:narrowed:{sub.target}", ""):
continue
seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]"))
if store.state.get(f"{source}:filtered:{sub.target}", ""):
closed += _close_unlisted(store, source, seen_ids, sub.id)
+59 -10
View File
@@ -27,6 +27,24 @@ on. Computed once per pull (`plan`) and handed to each board's batch as the
A reach of more countries than a `one`-filter host should be asked one by
one (the wide reach's Europe on SmartRecruiters) is dropped by `ask_for`,
which knows the reader (`MAX_SERIES`), so such a board is read whole.
**Fetch terms** (`Agent.fetch_terms`) narrow a wide read by words, so a big
board sends the roles that match one of them rather than every role:
- The terms of a pull are the union over the enabled agents. An enabled
agent with none wants every role, so then there are none.
- They go only with the wide reach: a company tagged All roles (`wide` in
the reach), and a board whose reach `ask_for` drops as too many countries
for a `one` host. The narrow reach (DK, SE) already leaves at most about
150 roles on any big board, and small, unknown and overridden boards get
no reach at all, so none of those take terms.
- A board's params switch them off with `{"terms": false}`.
- Only a reader that declares `query="one"` takes them, one series a term;
`ask_for` has the rule for terms times the reach's series.
- A read with terms is never the whole board: it sets no `ats:size`, and
the pull closes nothing against it (`app/jobs/pull.py`).
- The search providers take none. Every saved search has a query of its
own, which terms would replace; a search is a query already.
"""
from __future__ import annotations
@@ -162,16 +180,33 @@ def require_reach(agent: Agent, gaz: places.Gazetteer) -> Reach:
return Reach(remote=True) if remote and kept is None else EVERYWHERE
def fetch_terms(agents: Iterable[Agent]) -> tuple[str, ...]:
"""The fetch terms of a pull: the union of the agents' terms, one of each
regardless of case, sorted; none when an agent has none, or with no
agent."""
found: dict[str, str] = {}
listed = False
for agent in agents:
listed = True
terms = [t.strip() for t in agent.fetch_terms if t.strip()]
if not terms:
return ()
for term in terms:
found.setdefault(term.casefold(), term)
return tuple(found[k] for k in sorted(found)) if listed else ()
@dataclass
class Plan:
"""The reach of one pull: `base` for every board, `wide` for the boards
of the companies in `tagged` (tagged All roles), and each board's size
by target."""
of the companies in `tagged` (tagged All roles), each board's size by
target, and the fetch `terms` of the wide reads."""
base: Reach
wide: Reach = EVERYWHERE
sizes: dict[str, int] = field(default_factory=dict)
tagged: frozenset[str] = frozenset()
terms: tuple[str, ...] = ()
def why(self, sub: Subscription) -> tuple[Reach, str]:
"""The reach `sub` is read with, and why: `override`, `small`,
@@ -189,8 +224,17 @@ class Plan:
return self.base, "agents"
def for_subscription(self, sub: Subscription) -> dict[str, Any] | None:
"""The `reach` a board's batch hands `ask_for`; None for none."""
return self.why(sub)[0].params()
"""The `reach` a board's batch hands `ask_for`; None for none. With
fetch terms, a tagged board's carries them and `wide`, and a board on
the agents' reach carries them for `ask_for` to use if it drops the
countries as too many (see the module docstring)."""
found, why = self.why(sub)
out = found.params()
if self.terms and why in ("tag", "agents") and (sub.params or {}).get("terms") is not False:
out = (out or {}) | {"terms": list(self.terms)}
if why == "tag":
out["wide"] = True
return out
def sizes(store: Store, subs: Iterable[Subscription]) -> dict[str, int]:
@@ -204,7 +248,8 @@ def sizes(store: Store, subs: Iterable[Subscription]) -> dict[str, int]:
kept = state.get(f"{SIZE}{sub.target}", "")
if kept.isdigit():
out[sub.target] = int(kept)
elif state.get(f"ats:ok:{sub.target}") and not state.get(f"ats:filtered:{sub.target}"):
elif (state.get(f"ats:ok:{sub.target}") and not state.get(f"ats:filtered:{sub.target}")
and not state.get(f"ats:narrowed:{sub.target}")):
out[sub.target] = sum(counts.get(sub.id, {}).values())
return out
@@ -225,7 +270,7 @@ def plan(store: Store, subs: Iterable[Subscription] | None = None) -> Plan:
tagged = frozenset(c.key for c in store.companies() if c.all_roles)
# A tag never reads less than the board would untagged: the base's
# remote, which a `require` of Europe does not name, is kept.
return Plan(base, wide | base, sizes(store, subs), tagged)
return Plan(base, wide | base, sizes(store, subs), tagged, fetch_terms(agents))
def reaches(store: Store, subs: Iterable[Subscription]) -> Mapping[str, dict[str, Any]]:
@@ -248,19 +293,23 @@ _SAID = {"override": "set here", "tag": "company tag"}
def line(found: Plan, sub: Subscription) -> str:
"""What a pull asks `sub`'s board for, once its reader has narrowed the
reach, in a few words: "DK, SE, remote · agents", "33 countries, remote ·
company tag", "all roles · under 300 roles", "all roles · Greenhouse
cannot filter"."""
company tag", "57 countries · company tag · 3 terms", "all places ·
company tag · 3 terms", "all roles · under 300 roles", "all roles ·
Greenhouse cannot filter"."""
module = readers.HOSTS.get(sub.target.partition(":")[0])
if module is None:
return ""
given, why = found.why(sub)
ask = _runner.ask_for(module, sub.params, given.params())
ask = _runner.ask_for(module, sub.params, found.for_subscription(sub))
terms = f" · {len(ask.terms)} term{'s' if len(ask.terms) != 1 else ''}" if ask.terms else ""
if _runner.filtered(ask):
words = sorted(ask.countries) if len(ask.countries) <= 4 \
else [f"{len(ask.countries)} countries"]
if ask.remote:
words.append("remote")
return f"{', '.join(words)} · {_SAID.get(why, why)}"
return f"{', '.join(words)} · {_SAID.get(why, why)}{terms}"
if terms:
return f"all places · {_SAID.get(why, why)}{terms}"
if why in ("agents", "tag") and given.everywhere:
return f"all roles · {'company tag' if why == 'tag' else _WHOLE[why]}"
if why in ("agents", "tag") and not module.READS.filters.declared():