Read big boards within the agents' reach: countries and remote from the enabled agents, Europe for favourites, filtered at the source by six hosts

app/jobs/reach.py derives each board's reach once per pull and the pull hands it to the runner through the batch payload. SmartRecruiters, Eightfold and SuccessFactors filter one country a request, Phenom a list, Workday and Oracle by host ids from facets. Remote beside countries is a series of its own, since two facets intersect.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 09:55:18 +02:00
1 parent 3adafd8950
commit 3462a4abe7
17 files changed
+1324 -116

No files matched your search

+19 -8
View File
@@ -17,8 +17,8 @@ cap) stores what it read and closes nothing; a paged board over the page cap
is swept across pulls and closes once the sweep reached its end (see
`plugins/readers/_runner.py`).
A board read with a filter (`ats:filtered:<target>`, from its subscription's
`params`) lists only the roles in reach, so its close keeps open the roles a
A board read with a filter (`ats:filtered:<target>`, from the agents' reach,
`app.jobs.reach`, or its subscription's `params`) lists only the roles in reach, so its close keeps open the roles a
person acted on: those with a vote or an application (`_close_unlisted`). A
filtered read, or one of a reader that declares `empty="suspect"`, that lists
under a tenth of what the same read listed last time is recorded by the
@@ -51,6 +51,7 @@ from typing import Any
from pipeline import NULL, RunContext, Tally, Task, job
from app.config import Config
from app.jobs.reach import reaches
from app.models import company_key
from app.storage import Store
@@ -131,8 +132,8 @@ def _boards(config: Config, store: Store, run: str = "") -> list[dict[str, Any]]
its own, and queues nothing when another run is in flight. A run with
nothing to read is settled here, since no settle will see a batch of it."""
seed_searches(config, store)
targets = [(source, [s.target for s in store.subscriptions(source=source, fetchable=True)])
for source in SOURCES]
subs = {source: store.subscriptions(source=source, fetchable=True) for source in SOURCES}
targets = [(source, [s.target for s in subs[source]]) for source in SOURCES]
if not any(found for _, found in targets):
if run:
release_run(store, run)
@@ -142,13 +143,20 @@ def _boards(config: Config, store: Store, run: str = "") -> list[dict[str, Any]]
log.warning("pull: %s since %s; this one reads nothing", ALREADY_RUNNING,
running(store))
return []
return [{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run}
for source, found in targets for i in range(0, len(found), BATCH_SIZE)]
# The agents' reach, once per run, each board's in its batch (`app.jobs.reach`).
reach_of = reaches(store, subs["ats"])
batches = [{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run}
for source, found in targets for i in range(0, len(found), BATCH_SIZE)]
for batch in batches:
if batch["source"] == "ats":
batch["reach"] = {t: reach_of[t] for t in batch["targets"] if t in reach_of}
return batches
@job("pull_batch", "Reading boards", resource="http")
def run_batch(config: Config, store: Store, *, targets: list[str], source: str = "ats",
run: str = "", context: RunContext = NULL) -> dict[str, Any]:
run: str = "", reach: dict[str, Any] | None = None,
context: RunContext = NULL) -> dict[str, Any]:
"""Fetch this batch's boards or searches and `upsert_role` every item
they list.
@@ -163,7 +171,9 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
`run` is the marker of the run that queued this batch, handed to the
plugin as its `run` option, so its stamps belong to that run whatever the
stored marker says by the time the batch runs.
stored marker says by the time the batch runs. `reach` is `target ->` the
reach the run computed for a board (`app.jobs.reach`), handed to the
plugin as its `reach` option.
"""
from plugins.ats import ATSSource
from plugins.search import SearchSource
@@ -189,6 +199,7 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
options["boards"] = [{"url": target, "label": sub.company or sub.label,
"params": sub.params}]
options["keys"] = dict(config.sources.get("search") or {})
options["reach"] = (reach or {}).get(target)
plugin = ATSSource(options)
plugin.state = store.state
for item in plugin.fetch():
+235
View File
@@ -0,0 +1,235 @@
"""The reach of a pull: which countries, and whether remote roles, each
company board is asked for, derived from the search agents that are switched
on. Computed once per pull (`plan`) and handed to each board's batch as the
`reach` the runner narrows to its reader's filters
(`plugins/readers/_runner.py`, `ask_for`).
- **Base reach**: the union of the enabled agents' `any` gates on country,
place (a town or region, mapped to its country through the gazetteer) and
remote. An agent whose `any` group is empty, or holds any other member (a
function, a remote value other than `remote`, a place the gazetteer does
not know), can take a role from anywhere, so the base reach is then
everywhere.
- **Favourite companies** (affinity 2, which skips the `any` gates in
scoring) are read with the `require` reach of the agent that starred them:
the intersection of its `require` gates on country and place, Europe
today, expanded to its countries through the gazetteer. A company is
starred per agent, so its reach is the union over the enabled agents of
the `require` reach where it is a favourite and the `any` reach where not.
- **Board override**: a board whose subscription `params` name a country, a
place, remote or `{"reach": "all"}` gets no reach here; `ask_for` reads the
params instead.
- **Small boards**: a board that listed under `SMALL` roles when last read
whole gets no filter; filtering it saves little and risks closing roles.
A reach of more countries than a `one`-filter host should be asked one by
one (a favourite's Europe on SmartRecruiters) is dropped by `ask_for`, which
knows the reader (`MAX_SERIES`).
"""
from __future__ import annotations
from collections.abc import Iterable, Mapping
from dataclasses import dataclass, field
from typing import Any
from app import places
from app.models import Agent, Gate, Subscription, company_key
from app.storage import Store
#: A board that listed fewer roles than this when last read whole is read
#: whole.
SMALL = 300
#: `state` key of a board's size, `ats:size:<target>`: the roles its last
#: unfiltered read listed, written by `plugins/ats.py`.
SIZE = "ats:size:"
#: The params that make a board's own reach (see `_runner._PARAM_DIMENSION`).
OVERRIDE_KEYS = frozenset(("countries", "country", "places", "place", "remote", "reach"))
@dataclass(frozen=True)
class Reach:
"""Countries (ISO codes) and remote, or everywhere."""
countries: frozenset[str] = frozenset()
remote: bool = False
everywhere: bool = False
def __or__(self, other: Reach) -> Reach:
if self.everywhere or other.everywhere:
return EVERYWHERE
return Reach(self.countries | other.countries, self.remote or other.remote)
def params(self) -> dict[str, Any] | None:
"""What `ask_for` takes as `reach`; None for everywhere."""
if self.everywhere:
return None
out: dict[str, Any] = {"countries": sorted(self.countries)}
if self.remote:
out["remote"] = True
return out
def describe(self) -> str:
""""DK, SE, remote", "31 countries", "everywhere"."""
if self.everywhere:
return "everywhere"
codes = sorted(self.countries)
words = [", ".join(codes)] if len(codes) <= 4 else [f"{len(codes)} countries"]
if self.remote:
words.append("remote")
return ", ".join(w for w in words if w) or "nothing"
EVERYWHERE = Reach(everywhere=True)
NOWHERE = Reach()
def _countries_of(ids: Iterable[int], gaz: places.Gazetteer) -> set[str]:
"""The ISO codes of the countries the places lie in, a continent's every
country."""
out: set[str] = set()
for place_id in ids:
place = gaz.places.get(place_id)
if place is None:
continue
if place.kind == "continent":
out |= {gaz.places[p].key for p in gaz.under((place_id,))
if gaz.places[p].kind == "country"}
elif code := gaz.country(place_id):
out.add(code)
return out
def gate_reach(gate: Gate, gaz: places.Gazetteer) -> Reach:
"""Where a role that meets `gate` can be: its countries, remote, or
everywhere for a gate that says nothing of place or names one the
gazetteer does not know."""
values = [str(v).strip() for v in gate.values if str(v).strip()]
if gate.field == "remote":
return Reach(remote=True) if values and all(v.lower() == "remote" for v in values) \
else EVERYWHERE
if gate.field not in ("country", "place") or not values:
return EVERYWHERE
found: set[str] = set()
for value in values:
named = gaz.lookup(f"country:{value}" if gate.field == "country" else value)
if named is None:
if gate.field == "country" and len(value) == 2 and value.isalpha():
found.add(value.upper())
continue
return EVERYWHERE
found |= _countries_of(named, gaz)
return Reach(frozenset(found)) if found else EVERYWHERE
def any_reach(agent: Agent, gaz: places.Gazetteer) -> Reach:
"""The union of the agent's `any` gates; everywhere with none."""
group = [g for g in agent.gates if g.mode == "any"]
if not group:
return EVERYWHERE
out = NOWHERE
for gate in group:
out = out | gate_reach(gate, gaz)
return out
def require_reach(agent: Agent, gaz: places.Gazetteer) -> Reach:
"""The intersection of the agent's `require` gates on country and place,
or remote alone for a `require` of remote only; everywhere with neither,
or when the intersection is empty."""
kept: frozenset[str] | None = None
remote = False
for gate in agent.gates:
if gate.mode != "require":
continue
reach = gate_reach(gate, gaz)
if reach.everywhere:
continue
if reach.countries:
kept = reach.countries if kept is None else kept & reach.countries
elif reach.remote:
remote = True
if kept:
return Reach(kept)
return Reach(remote=True) if remote and kept is None else EVERYWHERE
@dataclass
class Plan:
"""The reach of one pull: `base` for every board, `favourites` for the
companies some agent starred, and each board's size by target."""
base: Reach
favourites: dict[str, Reach] = field(default_factory=dict)
sizes: dict[str, int] = field(default_factory=dict)
def why(self, sub: Subscription) -> tuple[Reach, str]:
"""The reach `sub` is read with, and why: `override`, `small`,
`unknown` (a board never read whole), `favourite` or `agents`."""
if OVERRIDE_KEYS & set(sub.params or {}):
return EVERYWHERE, "override"
size = self.sizes.get(sub.target)
if size is None:
return EVERYWHERE, "unknown"
if size < SMALL:
return EVERYWHERE, "small"
key = company_key(sub.company or sub.label)
if key in self.favourites:
return self.favourites[key], "favourite"
return self.base, "agents"
def for_subscription(self, sub: Subscription) -> dict[str, Any] | None:
"""The `reach` a board's batch hands `ask_for`; None for none."""
return self.why(sub)[0].params()
def sizes(store: Store, subs: Iterable[Subscription]) -> dict[str, int]:
"""Each board's size: the roles its last unfiltered read listed
(`ats:size:<target>`), else, for a board read before that was kept and
never filtered, its open roles. A board never read has none."""
state = store.state
counts = store.open_countries_by_subscription()
out: dict[str, int] = {}
for sub in subs:
kept = state.get(f"{SIZE}{sub.target}", "")
if kept.isdigit():
out[sub.target] = int(kept)
elif state.get(f"ats:ok:{sub.target}") and not state.get(f"ats:filtered:{sub.target}"):
out[sub.target] = sum(counts.get(sub.id, {}).values())
return out
def plan(store: Store, subs: Iterable[Subscription] | None = None) -> Plan:
"""The reach of a pull, from the enabled agents and their favourites, and
the sizes of `subs` (every `ats` subscription when None)."""
gaz = store.gazetteer()
agents = store.agents(enabled_only=True)
reaches = {a.id: (any_reach(a, gaz), require_reach(a, gaz)) for a in agents}
base = NOWHERE
for any_of, _ in reaches.values():
base = base | any_of
if not agents:
base = EVERYWHERE
starred: dict[str, set[int]] = {}
for agent in agents:
for company in store.companies(agent.id):
if company.affinity == 2:
starred.setdefault(company.key, set()).add(agent.id)
favourites: dict[str, Reach] = {}
for key, by in starred.items():
reach = NOWHERE
for agent_id, (any_of, required) in reaches.items():
reach = reach | (required if agent_id in by else any_of)
favourites[key] = reach
subs = list(store.subscriptions(source="ats") if subs is None else subs)
return Plan(base, favourites, sizes(store, subs))
def reaches(store: Store, subs: Iterable[Subscription]) -> Mapping[str, dict[str, Any]]:
"""`target -> reach` for the boards that get one, what a pull puts in
its batches' payloads."""
subs = list(subs)
found = plan(store, subs)
return {s.target: r for s in subs if (r := found.for_subscription(s)) is not None}