Merge branch 'wp-f-reach'

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 10:04:26 +02:00
commit ca80219bbb
24 files changed
+1657 -124

No files matched your search

+4 -2
View File
@@ -12,13 +12,15 @@ task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`.
sources.py what a pull reads, changed by hand: boards and searches
added, switched on or off
discover.py which ATS a company's careers site runs on, from its pages
reach.py which countries a pull asks each big board for, from the agents
"""
from __future__ import annotations
from typing import Any
from app.jobs import cascade, discover, embed, extract, pull, score, sources, stages, summarise
from app.jobs import (cascade, discover, embed, extract, pull, reach, score, sources, stages,
summarise)
from app.jobs.pull import search_target, seed_searches
from app.jobs.score import seed_agents
@@ -38,4 +40,4 @@ def bootstrap(config_path: str = "") -> tuple[Any, Any]:
__all__ = ["BOOTSTRAP", "bootstrap", "cascade", "discover", "embed", "extract", "pull",
"score", "search_target", "seed_agents", "seed_searches", "sources", "stages", "summarise"]
"reach", "score", "search_target", "seed_agents", "seed_searches", "sources", "stages", "summarise"]
+19 -8
View File
@@ -17,8 +17,8 @@ cap) stores what it read and closes nothing; a paged board over the page cap
is swept across pulls and closes once the sweep reached its end (see
`plugins/readers/_runner.py`).
A board read with a filter (`ats:filtered:<target>`, from its subscription's
`params`) lists only the roles in reach, so its close keeps open the roles a
A board read with a filter (`ats:filtered:<target>`, from the agents' reach,
`app.jobs.reach`, or its subscription's `params`) lists only the roles in reach, so its close keeps open the roles a
person acted on: those with a vote or an application (`_close_unlisted`). A
filtered read, or one of a reader that declares `empty="suspect"`, that lists
under a tenth of what the same read listed last time is recorded by the
@@ -51,6 +51,7 @@ from typing import Any
from pipeline import NULL, RunContext, Tally, Task, job
from app.config import Config
from app.jobs.reach import reaches
from app.models import company_key
from app.storage import Store
@@ -131,8 +132,8 @@ def _boards(config: Config, store: Store, run: str = "") -> list[dict[str, Any]]
its own, and queues nothing when another run is in flight. A run with
nothing to read is settled here, since no settle will see a batch of it."""
seed_searches(config, store)
targets = [(source, [s.target for s in store.subscriptions(source=source, fetchable=True)])
for source in SOURCES]
subs = {source: store.subscriptions(source=source, fetchable=True) for source in SOURCES}
targets = [(source, [s.target for s in subs[source]]) for source in SOURCES]
if not any(found for _, found in targets):
if run:
release_run(store, run)
@@ -142,13 +143,20 @@ def _boards(config: Config, store: Store, run: str = "") -> list[dict[str, Any]]
log.warning("pull: %s since %s; this one reads nothing", ALREADY_RUNNING,
running(store))
return []
return [{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run}
for source, found in targets for i in range(0, len(found), BATCH_SIZE)]
# The agents' reach, once per run, each board's in its batch (`app.jobs.reach`).
reach_of = reaches(store, subs["ats"])
batches = [{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run}
for source, found in targets for i in range(0, len(found), BATCH_SIZE)]
for batch in batches:
if batch["source"] == "ats":
batch["reach"] = {t: reach_of[t] for t in batch["targets"] if t in reach_of}
return batches
@job("pull_batch", "Reading boards", resource="http")
def run_batch(config: Config, store: Store, *, targets: list[str], source: str = "ats",
run: str = "", context: RunContext = NULL) -> dict[str, Any]:
run: str = "", reach: dict[str, Any] | None = None,
context: RunContext = NULL) -> dict[str, Any]:
"""Fetch this batch's boards or searches and `upsert_role` every item
they list.
@@ -163,7 +171,9 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
`run` is the marker of the run that queued this batch, handed to the
plugin as its `run` option, so its stamps belong to that run whatever the
stored marker says by the time the batch runs.
stored marker says by the time the batch runs. `reach` is `target ->` the
reach the run computed for a board (`app.jobs.reach`), handed to the
plugin as its `reach` option.
"""
from plugins.ats import ATSSource
from plugins.search import SearchSource
@@ -189,6 +199,7 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
options["boards"] = [{"url": target, "label": sub.company or sub.label,
"params": sub.params}]
options["keys"] = dict(config.sources.get("search") or {})
options["reach"] = (reach or {}).get(target)
plugin = ATSSource(options)
plugin.state = store.state
for item in plugin.fetch():
+266
View File
@@ -0,0 +1,266 @@
"""The reach of a pull: which countries, and whether remote roles, each
company board is asked for, derived from the search agents that are switched
on. Computed once per pull (`plan`) and handed to each board's batch as the
`reach` the runner narrows to its reader's filters
(`plugins/readers/_runner.py`, `ask_for`).
- **Base reach**: the union of the enabled agents' `any` gates on country,
place (a town or region, mapped to its country through the gazetteer) and
remote. An agent whose `any` group is empty, or holds any other member (a
function, a remote value other than `remote`, a place the gazetteer does
not know), can take a role from anywhere, so the base reach is then
everywhere.
- **Favourite companies** (affinity 2, which skips the `any` gates in
scoring) are read with the `require` reach of the agent that starred them:
the intersection of its `require` gates on country and place, Europe
today, expanded to its countries through the gazetteer. A company is
starred per agent, so its reach is the union over the enabled agents of
the `require` reach where it is a favourite and the `any` reach where not.
- **Board override**: a board whose subscription `params` name a country, a
place, remote or `{"reach": "all"}` gets no reach here; `ask_for` reads the
params instead.
- **Small boards**: a board that listed under `SMALL` roles when last read
whole gets no filter; filtering it saves little and risks closing roles.
A reach of more countries than a `one`-filter host should be asked one by
one (a favourite's Europe on SmartRecruiters) is dropped by `ask_for`, which
knows the reader (`MAX_SERIES`).
"""
from __future__ import annotations
from collections.abc import Iterable, Mapping
from dataclasses import dataclass, field
from typing import Any
from app import places
from app.models import Agent, Gate, Subscription, company_key
from app.storage import Store
from plugins import readers
from plugins.readers import _runner
#: A board that listed fewer roles than this when last read whole is read
#: whole.
SMALL = 300
#: `state` key of a board's size, `ats:size:<target>`: the roles its last
#: unfiltered read listed, written by `plugins/ats.py`.
SIZE = "ats:size:"
#: The params that make a board's own reach (see `_runner._PARAM_DIMENSION`).
OVERRIDE_KEYS = frozenset(("countries", "country", "places", "place", "remote", "reach"))
@dataclass(frozen=True)
class Reach:
"""Countries (ISO codes) and remote, or everywhere."""
countries: frozenset[str] = frozenset()
remote: bool = False
everywhere: bool = False
def __or__(self, other: Reach) -> Reach:
if self.everywhere or other.everywhere:
return EVERYWHERE
return Reach(self.countries | other.countries, self.remote or other.remote)
def params(self) -> dict[str, Any] | None:
"""What `ask_for` takes as `reach`; None for everywhere."""
if self.everywhere:
return None
out: dict[str, Any] = {"countries": sorted(self.countries)}
if self.remote:
out["remote"] = True
return out
def describe(self) -> str:
""""DK, SE, remote", "31 countries", "everywhere"."""
if self.everywhere:
return "everywhere"
codes = sorted(self.countries)
words = [", ".join(codes)] if len(codes) <= 4 else [f"{len(codes)} countries"]
if self.remote:
words.append("remote")
return ", ".join(w for w in words if w) or "nothing"
EVERYWHERE = Reach(everywhere=True)
NOWHERE = Reach()
def _countries_of(ids: Iterable[int], gaz: places.Gazetteer) -> set[str]:
"""The ISO codes of the countries the places lie in, a continent's every
country."""
out: set[str] = set()
for place_id in ids:
place = gaz.places.get(place_id)
if place is None:
continue
if place.kind == "continent":
out |= {gaz.places[p].key for p in gaz.under((place_id,))
if gaz.places[p].kind == "country"}
elif code := gaz.country(place_id):
out.add(code)
return out
def gate_reach(gate: Gate, gaz: places.Gazetteer) -> Reach:
"""Where a role that meets `gate` can be: its countries, remote, or
everywhere for a gate that says nothing of place or names one the
gazetteer does not know."""
values = [str(v).strip() for v in gate.values if str(v).strip()]
if gate.field == "remote":
return Reach(remote=True) if values and all(v.lower() == "remote" for v in values) \
else EVERYWHERE
if gate.field not in ("country", "place") or not values:
return EVERYWHERE
found: set[str] = set()
for value in values:
named = gaz.lookup(f"country:{value}" if gate.field == "country" else value)
if named is None:
if gate.field == "country" and len(value) == 2 and value.isalpha():
found.add(value.upper())
continue
return EVERYWHERE
found |= _countries_of(named, gaz)
return Reach(frozenset(found)) if found else EVERYWHERE
def any_reach(agent: Agent, gaz: places.Gazetteer) -> Reach:
"""The union of the agent's `any` gates; everywhere with none."""
group = [g for g in agent.gates if g.mode == "any"]
if not group:
return EVERYWHERE
out = NOWHERE
for gate in group:
out = out | gate_reach(gate, gaz)
return out
def require_reach(agent: Agent, gaz: places.Gazetteer) -> Reach:
"""The intersection of the agent's `require` gates on country and place,
or remote alone for a `require` of remote only; everywhere with neither,
or when the intersection is empty."""
kept: frozenset[str] | None = None
remote = False
for gate in agent.gates:
if gate.mode != "require":
continue
reach = gate_reach(gate, gaz)
if reach.everywhere:
continue
if reach.countries:
kept = reach.countries if kept is None else kept & reach.countries
elif reach.remote:
remote = True
if kept:
return Reach(kept)
return Reach(remote=True) if remote and kept is None else EVERYWHERE
@dataclass
class Plan:
"""The reach of one pull: `base` for every board, `favourites` for the
companies some agent starred, and each board's size by target."""
base: Reach
favourites: dict[str, Reach] = field(default_factory=dict)
sizes: dict[str, int] = field(default_factory=dict)
def why(self, sub: Subscription) -> tuple[Reach, str]:
"""The reach `sub` is read with, and why: `override`, `small`,
`unknown` (a board never read whole), `favourite` or `agents`."""
if OVERRIDE_KEYS & set(sub.params or {}):
return EVERYWHERE, "override"
size = self.sizes.get(sub.target)
if size is None:
return EVERYWHERE, "unknown"
if size < SMALL:
return EVERYWHERE, "small"
key = company_key(sub.company or sub.label)
if key in self.favourites:
return self.favourites[key], "favourite"
return self.base, "agents"
def for_subscription(self, sub: Subscription) -> dict[str, Any] | None:
"""The `reach` a board's batch hands `ask_for`; None for none."""
return self.why(sub)[0].params()
def sizes(store: Store, subs: Iterable[Subscription]) -> dict[str, int]:
"""Each board's size: the roles its last unfiltered read listed
(`ats:size:<target>`), else, for a board read before that was kept and
never filtered, its open roles. A board never read has none."""
state = store.state
counts = store.open_countries_by_subscription()
out: dict[str, int] = {}
for sub in subs:
kept = state.get(f"{SIZE}{sub.target}", "")
if kept.isdigit():
out[sub.target] = int(kept)
elif state.get(f"ats:ok:{sub.target}") and not state.get(f"ats:filtered:{sub.target}"):
out[sub.target] = sum(counts.get(sub.id, {}).values())
return out
def plan(store: Store, subs: Iterable[Subscription] | None = None) -> Plan:
"""The reach of a pull, from the enabled agents and their favourites, and
the sizes of `subs` (every `ats` subscription when None)."""
gaz = store.gazetteer()
agents = store.agents(enabled_only=True)
reaches = {a.id: (any_reach(a, gaz), require_reach(a, gaz)) for a in agents}
base = NOWHERE
for any_of, _ in reaches.values():
base = base | any_of
if not agents:
base = EVERYWHERE
starred: dict[str, set[int]] = {}
for agent in agents:
for company in store.companies(agent.id):
if company.affinity == 2:
starred.setdefault(company.key, set()).add(agent.id)
favourites: dict[str, Reach] = {}
for key, by in starred.items():
reach = NOWHERE
for agent_id, (any_of, required) in reaches.items():
reach = reach | (required if agent_id in by else any_of)
favourites[key] = reach
subs = list(store.subscriptions(source="ats") if subs is None else subs)
return Plan(base, favourites, sizes(store, subs))
def reaches(store: Store, subs: Iterable[Subscription]) -> Mapping[str, dict[str, Any]]:
"""`target -> reach` for the boards that get one, what a pull puts in
its batches' payloads."""
subs = list(subs)
found = plan(store, subs)
return {s.target: r for s in subs if (r := found.for_subscription(s)) is not None}
#: Why a board is read whole, as the Companies list says it.
_WHOLE = {"small": f"under {SMALL} roles", "unknown": "not read whole yet",
"override": "set here", "favourite": "favourite, too many countries to ask",
"agents": "an agent takes any place"}
def line(found: Plan, sub: Subscription) -> str:
"""What a pull asks `sub`'s board for, once its reader has narrowed the
reach, in a few words: "DK, SE, remote · agents", "all roles · under 300
roles", "all roles · Greenhouse cannot filter"."""
module = readers.HOSTS.get(sub.target.partition(":")[0])
if module is None:
return ""
given, why = found.why(sub)
ask = _runner.ask_for(module, sub.params, given.params())
if _runner.filtered(ask):
words = sorted(ask.countries) if len(ask.countries) <= 4 \
else [f"{len(ask.countries)} countries"]
if ask.remote:
words.append("remote")
return f"{', '.join(words)} · {'set here' if why == 'override' else why}"
if why in ("agents", "favourite") and not given.everywhere:
if not module.READS.filters.declared():
return f"all roles · {module.LABEL} cannot filter"
if why == "agents":
return f"all roles · {module.LABEL} cannot ask for these"
return f"all roles · {_WHOLE[why]}"