Read big boards within the agents' reach: countries and remote from the enabled agents, Europe for favourites, filtered at the source by six hosts
app/jobs/reach.py derives each board's reach once per pull and the pull hands it to the runner through the batch payload. SmartRecruiters, Eightfold and SuccessFactors filter one country a request, Phenom a list, Workday and Oracle by host ids from facets. Remote beside countries is a series of its own, since two facets intersect. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
3adafd8950
commit
3462a4abe7
17 files changed
+1324
-116
No files matched your search
+19
-8
@@ -17,8 +17,8 @@ cap) stores what it read and closes nothing; a paged board over the page cap
|
||||
is swept across pulls and closes once the sweep reached its end (see
|
||||
`plugins/readers/_runner.py`).
|
||||
|
||||
A board read with a filter (`ats:filtered:<target>`, from its subscription's
|
||||
`params`) lists only the roles in reach, so its close keeps open the roles a
|
||||
A board read with a filter (`ats:filtered:<target>`, from the agents' reach,
|
||||
`app.jobs.reach`, or its subscription's `params`) lists only the roles in reach, so its close keeps open the roles a
|
||||
person acted on: those with a vote or an application (`_close_unlisted`). A
|
||||
filtered read, or one of a reader that declares `empty="suspect"`, that lists
|
||||
under a tenth of what the same read listed last time is recorded by the
|
||||
@@ -51,6 +51,7 @@ from typing import Any
|
||||
from pipeline import NULL, RunContext, Tally, Task, job
|
||||
|
||||
from app.config import Config
|
||||
from app.jobs.reach import reaches
|
||||
from app.models import company_key
|
||||
from app.storage import Store
|
||||
|
||||
@@ -131,8 +132,8 @@ def _boards(config: Config, store: Store, run: str = "") -> list[dict[str, Any]]
|
||||
its own, and queues nothing when another run is in flight. A run with
|
||||
nothing to read is settled here, since no settle will see a batch of it."""
|
||||
seed_searches(config, store)
|
||||
targets = [(source, [s.target for s in store.subscriptions(source=source, fetchable=True)])
|
||||
for source in SOURCES]
|
||||
subs = {source: store.subscriptions(source=source, fetchable=True) for source in SOURCES}
|
||||
targets = [(source, [s.target for s in subs[source]]) for source in SOURCES]
|
||||
if not any(found for _, found in targets):
|
||||
if run:
|
||||
release_run(store, run)
|
||||
@@ -142,13 +143,20 @@ def _boards(config: Config, store: Store, run: str = "") -> list[dict[str, Any]]
|
||||
log.warning("pull: %s since %s; this one reads nothing", ALREADY_RUNNING,
|
||||
running(store))
|
||||
return []
|
||||
return [{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run}
|
||||
for source, found in targets for i in range(0, len(found), BATCH_SIZE)]
|
||||
# The agents' reach, once per run, each board's in its batch (`app.jobs.reach`).
|
||||
reach_of = reaches(store, subs["ats"])
|
||||
batches = [{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run}
|
||||
for source, found in targets for i in range(0, len(found), BATCH_SIZE)]
|
||||
for batch in batches:
|
||||
if batch["source"] == "ats":
|
||||
batch["reach"] = {t: reach_of[t] for t in batch["targets"] if t in reach_of}
|
||||
return batches
|
||||
|
||||
|
||||
@job("pull_batch", "Reading boards", resource="http")
|
||||
def run_batch(config: Config, store: Store, *, targets: list[str], source: str = "ats",
|
||||
run: str = "", context: RunContext = NULL) -> dict[str, Any]:
|
||||
run: str = "", reach: dict[str, Any] | None = None,
|
||||
context: RunContext = NULL) -> dict[str, Any]:
|
||||
"""Fetch this batch's boards or searches and `upsert_role` every item
|
||||
they list.
|
||||
|
||||
@@ -163,7 +171,9 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
|
||||
|
||||
`run` is the marker of the run that queued this batch, handed to the
|
||||
plugin as its `run` option, so its stamps belong to that run whatever the
|
||||
stored marker says by the time the batch runs.
|
||||
stored marker says by the time the batch runs. `reach` is `target ->` the
|
||||
reach the run computed for a board (`app.jobs.reach`), handed to the
|
||||
plugin as its `reach` option.
|
||||
"""
|
||||
from plugins.ats import ATSSource
|
||||
from plugins.search import SearchSource
|
||||
@@ -189,6 +199,7 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
|
||||
options["boards"] = [{"url": target, "label": sub.company or sub.label,
|
||||
"params": sub.params}]
|
||||
options["keys"] = dict(config.sources.get("search") or {})
|
||||
options["reach"] = (reach or {}).get(target)
|
||||
plugin = ATSSource(options)
|
||||
plugin.state = store.state
|
||||
for item in plugin.fetch():
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
"""The reach of a pull: which countries, and whether remote roles, each
|
||||
company board is asked for, derived from the search agents that are switched
|
||||
on. Computed once per pull (`plan`) and handed to each board's batch as the
|
||||
`reach` the runner narrows to its reader's filters
|
||||
(`plugins/readers/_runner.py`, `ask_for`).
|
||||
|
||||
- **Base reach**: the union of the enabled agents' `any` gates on country,
|
||||
place (a town or region, mapped to its country through the gazetteer) and
|
||||
remote. An agent whose `any` group is empty, or holds any other member (a
|
||||
function, a remote value other than `remote`, a place the gazetteer does
|
||||
not know), can take a role from anywhere, so the base reach is then
|
||||
everywhere.
|
||||
- **Favourite companies** (affinity 2, which skips the `any` gates in
|
||||
scoring) are read with the `require` reach of the agent that starred them:
|
||||
the intersection of its `require` gates on country and place, Europe
|
||||
today, expanded to its countries through the gazetteer. A company is
|
||||
starred per agent, so its reach is the union over the enabled agents of
|
||||
the `require` reach where it is a favourite and the `any` reach where not.
|
||||
- **Board override**: a board whose subscription `params` name a country, a
|
||||
place, remote or `{"reach": "all"}` gets no reach here; `ask_for` reads the
|
||||
params instead.
|
||||
- **Small boards**: a board that listed under `SMALL` roles when last read
|
||||
whole gets no filter; filtering it saves little and risks closing roles.
|
||||
|
||||
A reach of more countries than a `one`-filter host should be asked one by
|
||||
one (a favourite's Europe on SmartRecruiters) is dropped by `ask_for`, which
|
||||
knows the reader (`MAX_SERIES`).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
from app import places
|
||||
from app.models import Agent, Gate, Subscription, company_key
|
||||
from app.storage import Store
|
||||
|
||||
#: A board that listed fewer roles than this when last read whole is read
|
||||
#: whole.
|
||||
SMALL = 300
|
||||
|
||||
#: `state` key of a board's size, `ats:size:<target>`: the roles its last
|
||||
#: unfiltered read listed, written by `plugins/ats.py`.
|
||||
SIZE = "ats:size:"
|
||||
|
||||
#: The params that make a board's own reach (see `_runner._PARAM_DIMENSION`).
|
||||
OVERRIDE_KEYS = frozenset(("countries", "country", "places", "place", "remote", "reach"))
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Reach:
|
||||
"""Countries (ISO codes) and remote, or everywhere."""
|
||||
|
||||
countries: frozenset[str] = frozenset()
|
||||
remote: bool = False
|
||||
everywhere: bool = False
|
||||
|
||||
def __or__(self, other: Reach) -> Reach:
|
||||
if self.everywhere or other.everywhere:
|
||||
return EVERYWHERE
|
||||
return Reach(self.countries | other.countries, self.remote or other.remote)
|
||||
|
||||
def params(self) -> dict[str, Any] | None:
|
||||
"""What `ask_for` takes as `reach`; None for everywhere."""
|
||||
if self.everywhere:
|
||||
return None
|
||||
out: dict[str, Any] = {"countries": sorted(self.countries)}
|
||||
if self.remote:
|
||||
out["remote"] = True
|
||||
return out
|
||||
|
||||
def describe(self) -> str:
|
||||
""""DK, SE, remote", "31 countries", "everywhere"."""
|
||||
if self.everywhere:
|
||||
return "everywhere"
|
||||
codes = sorted(self.countries)
|
||||
words = [", ".join(codes)] if len(codes) <= 4 else [f"{len(codes)} countries"]
|
||||
if self.remote:
|
||||
words.append("remote")
|
||||
return ", ".join(w for w in words if w) or "nothing"
|
||||
|
||||
|
||||
EVERYWHERE = Reach(everywhere=True)
|
||||
NOWHERE = Reach()
|
||||
|
||||
|
||||
def _countries_of(ids: Iterable[int], gaz: places.Gazetteer) -> set[str]:
|
||||
"""The ISO codes of the countries the places lie in, a continent's every
|
||||
country."""
|
||||
out: set[str] = set()
|
||||
for place_id in ids:
|
||||
place = gaz.places.get(place_id)
|
||||
if place is None:
|
||||
continue
|
||||
if place.kind == "continent":
|
||||
out |= {gaz.places[p].key for p in gaz.under((place_id,))
|
||||
if gaz.places[p].kind == "country"}
|
||||
elif code := gaz.country(place_id):
|
||||
out.add(code)
|
||||
return out
|
||||
|
||||
|
||||
def gate_reach(gate: Gate, gaz: places.Gazetteer) -> Reach:
|
||||
"""Where a role that meets `gate` can be: its countries, remote, or
|
||||
everywhere for a gate that says nothing of place or names one the
|
||||
gazetteer does not know."""
|
||||
values = [str(v).strip() for v in gate.values if str(v).strip()]
|
||||
if gate.field == "remote":
|
||||
return Reach(remote=True) if values and all(v.lower() == "remote" for v in values) \
|
||||
else EVERYWHERE
|
||||
if gate.field not in ("country", "place") or not values:
|
||||
return EVERYWHERE
|
||||
found: set[str] = set()
|
||||
for value in values:
|
||||
named = gaz.lookup(f"country:{value}" if gate.field == "country" else value)
|
||||
if named is None:
|
||||
if gate.field == "country" and len(value) == 2 and value.isalpha():
|
||||
found.add(value.upper())
|
||||
continue
|
||||
return EVERYWHERE
|
||||
found |= _countries_of(named, gaz)
|
||||
return Reach(frozenset(found)) if found else EVERYWHERE
|
||||
|
||||
|
||||
def any_reach(agent: Agent, gaz: places.Gazetteer) -> Reach:
|
||||
"""The union of the agent's `any` gates; everywhere with none."""
|
||||
group = [g for g in agent.gates if g.mode == "any"]
|
||||
if not group:
|
||||
return EVERYWHERE
|
||||
out = NOWHERE
|
||||
for gate in group:
|
||||
out = out | gate_reach(gate, gaz)
|
||||
return out
|
||||
|
||||
|
||||
def require_reach(agent: Agent, gaz: places.Gazetteer) -> Reach:
|
||||
"""The intersection of the agent's `require` gates on country and place,
|
||||
or remote alone for a `require` of remote only; everywhere with neither,
|
||||
or when the intersection is empty."""
|
||||
kept: frozenset[str] | None = None
|
||||
remote = False
|
||||
for gate in agent.gates:
|
||||
if gate.mode != "require":
|
||||
continue
|
||||
reach = gate_reach(gate, gaz)
|
||||
if reach.everywhere:
|
||||
continue
|
||||
if reach.countries:
|
||||
kept = reach.countries if kept is None else kept & reach.countries
|
||||
elif reach.remote:
|
||||
remote = True
|
||||
if kept:
|
||||
return Reach(kept)
|
||||
return Reach(remote=True) if remote and kept is None else EVERYWHERE
|
||||
|
||||
|
||||
@dataclass
|
||||
class Plan:
|
||||
"""The reach of one pull: `base` for every board, `favourites` for the
|
||||
companies some agent starred, and each board's size by target."""
|
||||
|
||||
base: Reach
|
||||
favourites: dict[str, Reach] = field(default_factory=dict)
|
||||
sizes: dict[str, int] = field(default_factory=dict)
|
||||
|
||||
def why(self, sub: Subscription) -> tuple[Reach, str]:
|
||||
"""The reach `sub` is read with, and why: `override`, `small`,
|
||||
`unknown` (a board never read whole), `favourite` or `agents`."""
|
||||
if OVERRIDE_KEYS & set(sub.params or {}):
|
||||
return EVERYWHERE, "override"
|
||||
size = self.sizes.get(sub.target)
|
||||
if size is None:
|
||||
return EVERYWHERE, "unknown"
|
||||
if size < SMALL:
|
||||
return EVERYWHERE, "small"
|
||||
key = company_key(sub.company or sub.label)
|
||||
if key in self.favourites:
|
||||
return self.favourites[key], "favourite"
|
||||
return self.base, "agents"
|
||||
|
||||
def for_subscription(self, sub: Subscription) -> dict[str, Any] | None:
|
||||
"""The `reach` a board's batch hands `ask_for`; None for none."""
|
||||
return self.why(sub)[0].params()
|
||||
|
||||
|
||||
def sizes(store: Store, subs: Iterable[Subscription]) -> dict[str, int]:
|
||||
"""Each board's size: the roles its last unfiltered read listed
|
||||
(`ats:size:<target>`), else, for a board read before that was kept and
|
||||
never filtered, its open roles. A board never read has none."""
|
||||
state = store.state
|
||||
counts = store.open_countries_by_subscription()
|
||||
out: dict[str, int] = {}
|
||||
for sub in subs:
|
||||
kept = state.get(f"{SIZE}{sub.target}", "")
|
||||
if kept.isdigit():
|
||||
out[sub.target] = int(kept)
|
||||
elif state.get(f"ats:ok:{sub.target}") and not state.get(f"ats:filtered:{sub.target}"):
|
||||
out[sub.target] = sum(counts.get(sub.id, {}).values())
|
||||
return out
|
||||
|
||||
|
||||
def plan(store: Store, subs: Iterable[Subscription] | None = None) -> Plan:
|
||||
"""The reach of a pull, from the enabled agents and their favourites, and
|
||||
the sizes of `subs` (every `ats` subscription when None)."""
|
||||
gaz = store.gazetteer()
|
||||
agents = store.agents(enabled_only=True)
|
||||
reaches = {a.id: (any_reach(a, gaz), require_reach(a, gaz)) for a in agents}
|
||||
base = NOWHERE
|
||||
for any_of, _ in reaches.values():
|
||||
base = base | any_of
|
||||
if not agents:
|
||||
base = EVERYWHERE
|
||||
starred: dict[str, set[int]] = {}
|
||||
for agent in agents:
|
||||
for company in store.companies(agent.id):
|
||||
if company.affinity == 2:
|
||||
starred.setdefault(company.key, set()).add(agent.id)
|
||||
favourites: dict[str, Reach] = {}
|
||||
for key, by in starred.items():
|
||||
reach = NOWHERE
|
||||
for agent_id, (any_of, required) in reaches.items():
|
||||
reach = reach | (required if agent_id in by else any_of)
|
||||
favourites[key] = reach
|
||||
subs = list(store.subscriptions(source="ats") if subs is None else subs)
|
||||
return Plan(base, favourites, sizes(store, subs))
|
||||
|
||||
|
||||
def reaches(store: Store, subs: Iterable[Subscription]) -> Mapping[str, dict[str, Any]]:
|
||||
"""`target -> reach` for the boards that get one, what a pull puts in
|
||||
its batches' payloads."""
|
||||
subs = list(subs)
|
||||
found = plan(store, subs)
|
||||
return {s.target: r for s in subs if (r := found.for_subscription(s)) is not None}
|
||||
Reference in new issue
Block a user