Merge branch 'wp-f-reach'

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 10:04:26 +02:00
commit ca80219bbb
24 files changed
+1657 -124

No files matched your search

+4 -2
View File
@@ -12,13 +12,15 @@ task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`.
sources.py what a pull reads, changed by hand: boards and searches
added, switched on or off
discover.py which ATS a company's careers site runs on, from its pages
reach.py which countries a pull asks each big board for, from the agents
"""
from __future__ import annotations
from typing import Any
from app.jobs import cascade, discover, embed, extract, pull, score, sources, stages, summarise
from app.jobs import (cascade, discover, embed, extract, pull, reach, score, sources, stages,
summarise)
from app.jobs.pull import search_target, seed_searches
from app.jobs.score import seed_agents
@@ -38,4 +40,4 @@ def bootstrap(config_path: str = "") -> tuple[Any, Any]:
__all__ = ["BOOTSTRAP", "bootstrap", "cascade", "discover", "embed", "extract", "pull",
"score", "search_target", "seed_agents", "seed_searches", "sources", "stages", "summarise"]
"reach", "score", "search_target", "seed_agents", "seed_searches", "sources", "stages", "summarise"]
+19 -8
View File
@@ -17,8 +17,8 @@ cap) stores what it read and closes nothing; a paged board over the page cap
is swept across pulls and closes once the sweep reached its end (see
`plugins/readers/_runner.py`).
A board read with a filter (`ats:filtered:<target>`, from its subscription's
`params`) lists only the roles in reach, so its close keeps open the roles a
A board read with a filter (`ats:filtered:<target>`, from the agents' reach,
`app.jobs.reach`, or its subscription's `params`) lists only the roles in reach, so its close keeps open the roles a
person acted on: those with a vote or an application (`_close_unlisted`). A
filtered read, or one of a reader that declares `empty="suspect"`, that lists
under a tenth of what the same read listed last time is recorded by the
@@ -51,6 +51,7 @@ from typing import Any
from pipeline import NULL, RunContext, Tally, Task, job
from app.config import Config
from app.jobs.reach import reaches
from app.models import company_key
from app.storage import Store
@@ -131,8 +132,8 @@ def _boards(config: Config, store: Store, run: str = "") -> list[dict[str, Any]]
its own, and queues nothing when another run is in flight. A run with
nothing to read is settled here, since no settle will see a batch of it."""
seed_searches(config, store)
targets = [(source, [s.target for s in store.subscriptions(source=source, fetchable=True)])
for source in SOURCES]
subs = {source: store.subscriptions(source=source, fetchable=True) for source in SOURCES}
targets = [(source, [s.target for s in subs[source]]) for source in SOURCES]
if not any(found for _, found in targets):
if run:
release_run(store, run)
@@ -142,13 +143,20 @@ def _boards(config: Config, store: Store, run: str = "") -> list[dict[str, Any]]
log.warning("pull: %s since %s; this one reads nothing", ALREADY_RUNNING,
running(store))
return []
return [{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run}
for source, found in targets for i in range(0, len(found), BATCH_SIZE)]
# The agents' reach, once per run, each board's in its batch (`app.jobs.reach`).
reach_of = reaches(store, subs["ats"])
batches = [{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run}
for source, found in targets for i in range(0, len(found), BATCH_SIZE)]
for batch in batches:
if batch["source"] == "ats":
batch["reach"] = {t: reach_of[t] for t in batch["targets"] if t in reach_of}
return batches
@job("pull_batch", "Reading boards", resource="http")
def run_batch(config: Config, store: Store, *, targets: list[str], source: str = "ats",
run: str = "", context: RunContext = NULL) -> dict[str, Any]:
run: str = "", reach: dict[str, Any] | None = None,
context: RunContext = NULL) -> dict[str, Any]:
"""Fetch this batch's boards or searches and `upsert_role` every item
they list.
@@ -163,7 +171,9 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
`run` is the marker of the run that queued this batch, handed to the
plugin as its `run` option, so its stamps belong to that run whatever the
stored marker says by the time the batch runs.
stored marker says by the time the batch runs. `reach` is `target ->` the
reach the run computed for a board (`app.jobs.reach`), handed to the
plugin as its `reach` option.
"""
from plugins.ats import ATSSource
from plugins.search import SearchSource
@@ -189,6 +199,7 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
options["boards"] = [{"url": target, "label": sub.company or sub.label,
"params": sub.params}]
options["keys"] = dict(config.sources.get("search") or {})
options["reach"] = (reach or {}).get(target)
plugin = ATSSource(options)
plugin.state = store.state
for item in plugin.fetch():
+266
View File
@@ -0,0 +1,266 @@
"""The reach of a pull: which countries, and whether remote roles, each
company board is asked for, derived from the search agents that are switched
on. Computed once per pull (`plan`) and handed to each board's batch as the
`reach` the runner narrows to its reader's filters
(`plugins/readers/_runner.py`, `ask_for`).
- **Base reach**: the union of the enabled agents' `any` gates on country,
place (a town or region, mapped to its country through the gazetteer) and
remote. An agent whose `any` group is empty, or holds any other member (a
function, a remote value other than `remote`, a place the gazetteer does
not know), can take a role from anywhere, so the base reach is then
everywhere.
- **Favourite companies** (affinity 2, which skips the `any` gates in
scoring) are read with the `require` reach of the agent that starred them:
the intersection of its `require` gates on country and place, Europe
today, expanded to its countries through the gazetteer. A company is
starred per agent, so its reach is the union over the enabled agents of
the `require` reach where it is a favourite and the `any` reach where not.
- **Board override**: a board whose subscription `params` name a country, a
place, remote or `{"reach": "all"}` gets no reach here; `ask_for` reads the
params instead.
- **Small boards**: a board that listed under `SMALL` roles when last read
whole gets no filter; filtering it saves little and risks closing roles.
A reach of more countries than a `one`-filter host should be asked one by
one (a favourite's Europe on SmartRecruiters) is dropped by `ask_for`, which
knows the reader (`MAX_SERIES`).
"""
from __future__ import annotations
from collections.abc import Iterable, Mapping
from dataclasses import dataclass, field
from typing import Any
from app import places
from app.models import Agent, Gate, Subscription, company_key
from app.storage import Store
from plugins import readers
from plugins.readers import _runner
#: A board that listed fewer roles than this when last read whole is read
#: whole.
SMALL = 300
#: `state` key of a board's size, `ats:size:<target>`: the roles its last
#: unfiltered read listed, written by `plugins/ats.py`.
SIZE = "ats:size:"
#: The params that make a board's own reach (see `_runner._PARAM_DIMENSION`).
OVERRIDE_KEYS = frozenset(("countries", "country", "places", "place", "remote", "reach"))
@dataclass(frozen=True)
class Reach:
"""Countries (ISO codes) and remote, or everywhere."""
countries: frozenset[str] = frozenset()
remote: bool = False
everywhere: bool = False
def __or__(self, other: Reach) -> Reach:
if self.everywhere or other.everywhere:
return EVERYWHERE
return Reach(self.countries | other.countries, self.remote or other.remote)
def params(self) -> dict[str, Any] | None:
"""What `ask_for` takes as `reach`; None for everywhere."""
if self.everywhere:
return None
out: dict[str, Any] = {"countries": sorted(self.countries)}
if self.remote:
out["remote"] = True
return out
def describe(self) -> str:
""""DK, SE, remote", "31 countries", "everywhere"."""
if self.everywhere:
return "everywhere"
codes = sorted(self.countries)
words = [", ".join(codes)] if len(codes) <= 4 else [f"{len(codes)} countries"]
if self.remote:
words.append("remote")
return ", ".join(w for w in words if w) or "nothing"
EVERYWHERE = Reach(everywhere=True)
NOWHERE = Reach()
def _countries_of(ids: Iterable[int], gaz: places.Gazetteer) -> set[str]:
"""The ISO codes of the countries the places lie in, a continent's every
country."""
out: set[str] = set()
for place_id in ids:
place = gaz.places.get(place_id)
if place is None:
continue
if place.kind == "continent":
out |= {gaz.places[p].key for p in gaz.under((place_id,))
if gaz.places[p].kind == "country"}
elif code := gaz.country(place_id):
out.add(code)
return out
def gate_reach(gate: Gate, gaz: places.Gazetteer) -> Reach:
"""Where a role that meets `gate` can be: its countries, remote, or
everywhere for a gate that says nothing of place or names one the
gazetteer does not know."""
values = [str(v).strip() for v in gate.values if str(v).strip()]
if gate.field == "remote":
return Reach(remote=True) if values and all(v.lower() == "remote" for v in values) \
else EVERYWHERE
if gate.field not in ("country", "place") or not values:
return EVERYWHERE
found: set[str] = set()
for value in values:
named = gaz.lookup(f"country:{value}" if gate.field == "country" else value)
if named is None:
if gate.field == "country" and len(value) == 2 and value.isalpha():
found.add(value.upper())
continue
return EVERYWHERE
found |= _countries_of(named, gaz)
return Reach(frozenset(found)) if found else EVERYWHERE
def any_reach(agent: Agent, gaz: places.Gazetteer) -> Reach:
"""The union of the agent's `any` gates; everywhere with none."""
group = [g for g in agent.gates if g.mode == "any"]
if not group:
return EVERYWHERE
out = NOWHERE
for gate in group:
out = out | gate_reach(gate, gaz)
return out
def require_reach(agent: Agent, gaz: places.Gazetteer) -> Reach:
"""The intersection of the agent's `require` gates on country and place,
or remote alone for a `require` of remote only; everywhere with neither,
or when the intersection is empty."""
kept: frozenset[str] | None = None
remote = False
for gate in agent.gates:
if gate.mode != "require":
continue
reach = gate_reach(gate, gaz)
if reach.everywhere:
continue
if reach.countries:
kept = reach.countries if kept is None else kept & reach.countries
elif reach.remote:
remote = True
if kept:
return Reach(kept)
return Reach(remote=True) if remote and kept is None else EVERYWHERE
@dataclass
class Plan:
"""The reach of one pull: `base` for every board, `favourites` for the
companies some agent starred, and each board's size by target."""
base: Reach
favourites: dict[str, Reach] = field(default_factory=dict)
sizes: dict[str, int] = field(default_factory=dict)
def why(self, sub: Subscription) -> tuple[Reach, str]:
"""The reach `sub` is read with, and why: `override`, `small`,
`unknown` (a board never read whole), `favourite` or `agents`."""
if OVERRIDE_KEYS & set(sub.params or {}):
return EVERYWHERE, "override"
size = self.sizes.get(sub.target)
if size is None:
return EVERYWHERE, "unknown"
if size < SMALL:
return EVERYWHERE, "small"
key = company_key(sub.company or sub.label)
if key in self.favourites:
return self.favourites[key], "favourite"
return self.base, "agents"
def for_subscription(self, sub: Subscription) -> dict[str, Any] | None:
"""The `reach` a board's batch hands `ask_for`; None for none."""
return self.why(sub)[0].params()
def sizes(store: Store, subs: Iterable[Subscription]) -> dict[str, int]:
"""Each board's size: the roles its last unfiltered read listed
(`ats:size:<target>`), else, for a board read before that was kept and
never filtered, its open roles. A board never read has none."""
state = store.state
counts = store.open_countries_by_subscription()
out: dict[str, int] = {}
for sub in subs:
kept = state.get(f"{SIZE}{sub.target}", "")
if kept.isdigit():
out[sub.target] = int(kept)
elif state.get(f"ats:ok:{sub.target}") and not state.get(f"ats:filtered:{sub.target}"):
out[sub.target] = sum(counts.get(sub.id, {}).values())
return out
def plan(store: Store, subs: Iterable[Subscription] | None = None) -> Plan:
"""The reach of a pull, from the enabled agents and their favourites, and
the sizes of `subs` (every `ats` subscription when None)."""
gaz = store.gazetteer()
agents = store.agents(enabled_only=True)
reaches = {a.id: (any_reach(a, gaz), require_reach(a, gaz)) for a in agents}
base = NOWHERE
for any_of, _ in reaches.values():
base = base | any_of
if not agents:
base = EVERYWHERE
starred: dict[str, set[int]] = {}
for agent in agents:
for company in store.companies(agent.id):
if company.affinity == 2:
starred.setdefault(company.key, set()).add(agent.id)
favourites: dict[str, Reach] = {}
for key, by in starred.items():
reach = NOWHERE
for agent_id, (any_of, required) in reaches.items():
reach = reach | (required if agent_id in by else any_of)
favourites[key] = reach
subs = list(store.subscriptions(source="ats") if subs is None else subs)
return Plan(base, favourites, sizes(store, subs))
def reaches(store: Store, subs: Iterable[Subscription]) -> Mapping[str, dict[str, Any]]:
"""`target -> reach` for the boards that get one, what a pull puts in
its batches' payloads."""
subs = list(subs)
found = plan(store, subs)
return {s.target: r for s in subs if (r := found.for_subscription(s)) is not None}
#: Why a board is read whole, as the Companies list says it.
_WHOLE = {"small": f"under {SMALL} roles", "unknown": "not read whole yet",
"override": "set here", "favourite": "favourite, too many countries to ask",
"agents": "an agent takes any place"}
def line(found: Plan, sub: Subscription) -> str:
"""What a pull asks `sub`'s board for, once its reader has narrowed the
reach, in a few words: "DK, SE, remote · agents", "all roles · under 300
roles", "all roles · Greenhouse cannot filter"."""
module = readers.HOSTS.get(sub.target.partition(":")[0])
if module is None:
return ""
given, why = found.why(sub)
ask = _runner.ask_for(module, sub.params, given.params())
if _runner.filtered(ask):
words = sorted(ask.countries) if len(ask.countries) <= 4 \
else [f"{len(ask.countries)} countries"]
if ask.remote:
words.append("remote")
return f"{', '.join(words)} · {'set here' if why == 'override' else why}"
if why in ("agents", "favourite") and not given.everywhere:
if not module.READS.filters.declared():
return f"all roles · {module.LABEL} cannot filter"
if why == "agents":
return f"all roles · {module.LABEL} cannot ask for these"
return f"all roles · {_WHOLE[why]}"
+63 -1
View File
@@ -17,6 +17,11 @@
* into the Follow form's `#b-host`, and a slug. Saving posts to
* `/api/companies/{key}/board`, which reads the board once before keeping
* it; Detach stops reading it and keeps the company.
*
* A row with a board says what a pull asks it for ("reads DK, SE · agents",
* `reach` from app/jobs/reach.py), and its editor has a second line that
* sets the board's own reach: the agents', all roles, or its own countries
* and remote, posted to `/api/sources/{id}/reach`.
*/
(function () {
"use strict";
@@ -90,6 +95,7 @@
sub.appendChild(link);
}
sub.append(` · ${count(company.open_roles, "open role")}`);
if (company.reach) sub.append(` · reads ${company.reach}`);
if (company.failing_since) {
// The board has failed on every pull since then; after 21 days its
// roles close.
@@ -199,6 +205,58 @@
return form;
}
/** The open row's reach editor, for a row with a board: the agents' reach,
all roles, or the board's own countries and remote. */
function reachEditor(company, rerender) {
if (company.subscription_id == null) return null;
const own = company.reach_params || {};
const form = document.createElement("form");
form.className = "board-edit";
form.autocomplete = "off";
const mode = document.createElement("select");
mode.setAttribute("aria-label", `What ${company.name}'s board is asked for`);
for (const [value, label] of [["agents", "Agents' reach"], ["all", "All roles"],
["own", "These countries"]]) {
mode.appendChild(new Option(label, value));
}
mode.value = own.reach === "all" ? "all" : (own.countries || own.remote != null ? "own" : "agents");
const codes = document.createElement("input");
codes.type = "text";
codes.spellcheck = false;
codes.placeholder = "DK SE";
codes.value = (own.countries || []).join(" ");
codes.setAttribute("aria-label", "Country codes");
const remote = document.createElement("select");
remote.setAttribute("aria-label", "Remote roles");
remote.append(new Option("and remote", "yes"), new Option("no remote", "no"));
remote.value = own.remote ? "yes" : "no";
const save = document.createElement("button");
save.type = "submit";
save.textContent = "Set reach";
const shown = () => { codes.hidden = remote.hidden = mode.value !== "own"; };
mode.addEventListener("change", shown);
shown();
form.append(mode, codes, remote, save);
form.addEventListener("submit", async (event) => {
event.preventDefault();
save.disabled = true;
try {
const result = await api(`/api/sources/${company.subscription_id}/reach`, {
method: "POST",
body: { mode: mode.value, countries: codes.value.split(/[\s,]+/).filter(Boolean),
remote: remote.value === "yes" },
});
window.toast.info(`${company.name}: reads ${result.reach} from the next pull.`);
editing = null;
await loadCompanies(agentId());
rerender();
} catch (err) {
save.disabled = false;
}
});
return form;
}
/** The board switch, or nothing for a company with no board:
there is no subscription to switch. */
function onSwitch(company, rerender) {
@@ -274,7 +332,11 @@
row.appendChild(affinityControl(company.key, company.affinity, async () => render(), agent));
}
row.appendChild(subLine(company, render));
if (editing === company.key) row.appendChild(boardEditor(company, render));
if (editing === company.key) {
row.appendChild(boardEditor(company, render));
const reach = reachEditor(company, render);
if (reach) row.appendChild(reach);
}
body.appendChild(row);
}
const open = rows.reduce((sum, c) => sum + c.open_roles, 0);
+44
View File
@@ -15,6 +15,11 @@
*
* It is also the page's first script, so it shows the one section of the
* four the URL fragment names and marks it in the settings rail.
*
* Under the Follow form of Company boards, a fold lists what each reader
* declares (`/api/readers`): how its list arrives, where the text is, what
* it filters at the source, how it states places, and the robots.txt
* decision taken when it was built.
*/
(function () {
"use strict";
@@ -277,6 +282,44 @@
}
}
// ----------------------------------------------------------------- readers
/** "country one, remote ids", or "none". */
function filters(reader) {
const shown = Object.entries(reader.filters).map(([name, how]) => `${name} ${how}`);
return shown.length ? shown.join(", ") : "none";
}
function renderReaders(readers) {
const fold = document.createElement("details");
fold.className = "prune";
fold.id = "readers";
const summary = document.createElement("summary");
summary.textContent = `What each reader does (${readers.length})`;
const list = document.createElement("div");
list.className = "srows";
for (const reader of readers) {
const row = document.createElement("div");
row.className = "srow";
const name = document.createElement("span");
name.className = "name";
const label = document.createElement("span");
label.textContent = reader.label;
const scope = document.createElement("span");
scope.className = "badge";
scope.textContent = reader.scope === "search" ? "job board" : "company board";
name.append(label, scope);
const sub = document.createElement("span");
sub.className = "sub";
sub.textContent = [`list ${reader.listing}`, `text ${reader.text}`,
`filters ${filters(reader)}`, `places ${reader.places}`, reader.robots].join(" · ");
row.append(name, sub);
list.appendChild(row);
}
fold.append(summary, list);
$("#board-form").after(fold);
}
// ------------------------------------------------------ one section shown
// Sources shows one section: the one the fragment names or contains, or the
@@ -317,6 +360,7 @@
document.addEventListener("DOMContentLoaded", async () => {
if (!$("#searches-list")) return;
if ($("#board-form")) api("/api/readers").then(renderReaders, () => {});
try {
await load();
} catch (err) {
+71 -1
View File
@@ -473,6 +473,10 @@ class CompanyOut(BaseModel):
from_search: bool = False
hand_reason: str = ""
by_hand: bool = False
#: The board's own reach in its params (`jobs.reach.OVERRIDE_KEYS`), and
#: what a pull asks its board for (`jobs.reach.line`).
reach_params: dict[str, Any] = {}
reach: str = ""
def _company_boards(store: Store) -> dict[str, tuple[int, str, bool, str]]:
@@ -495,10 +499,13 @@ def _company_outs(store: Store, agent: int | None) -> list[CompanyOut]:
ever_boarded = {company_key(s.company) for s in store.subscriptions(source="ats")}
counts = store.open_role_counts()
failing = jobs.pull.failing_boards(store)
subs = {s.id: s for s in store.subscriptions(source="ats")}
plan = jobs.reach.plan(store, subs.values())
out = []
for c in store.companies(agent):
sub_id, host, on, target = boards.get(c.key, (None, "", None, ""))
since, reason = failing.get(target, ("", ""))
sub = subs.get(sub_id) if sub_id is not None else None
from_search = jobs.sources.known_from_search_only(c, ever_boarded)
out.append(CompanyOut(key=c.key, name=c.name, notes=c.notes, board=host,
subscription_id=sub_id, on=on, affinity=c.affinity,
@@ -508,7 +515,10 @@ def _company_outs(store: Store, agent: int | None) -> list[CompanyOut]:
watch=c.watch, careers_opened_at=c.careers_opened_at,
failing_since=since, failing_reason=reason,
from_search=from_search, hand_reason=c.hand_reason,
by_hand=c.key not in boards and not from_search))
by_hand=c.key not in boards and not from_search,
reach_params={k: v for k, v in (sub.params if sub else {}).items()
if k in jobs.reach.OVERRIDE_KEYS},
reach=jobs.reach.line(plan, sub) if sub else ""))
return out
@@ -803,6 +813,66 @@ def switch_source(sub_id: int, body: SwitchBody, store: StoreDep,
return {"source": _one_source(store, sub_id), "closed": closed}
class ReachBody(BaseModel):
#: `agents`: the enabled agents' reach; `all`: every role; `own`: the
#: `countries` (ISO codes) and remote given here.
mode: Literal["agents", "all", "own"]
countries: list[str] = []
remote: bool = False
@router.post("/sources/{sub_id}/reach")
def set_reach(sub_id: int, body: ReachBody, store: StoreDep) -> dict[str, Any]:
"""A board's own reach, which wins over the agents' (see
`app/jobs/reach.py`), or none. The next pull reads the board so; a
narrower reach closes the roles outside it then, except those with a
vote or an application."""
sub = store.subscription(sub_id)
if sub is None or sub.source != "ats":
raise HTTPException(404, f"no such board: {sub_id}")
codes = sorted({c.strip().upper() for c in body.countries if c.strip()})
if unknown := [c for c in codes if not places.known(f"country:{c}") or c == "EU"]:
raise HTTPException(422, f"Not a country code: {', '.join(unknown)}.")
if body.mode == "own" and not (codes or body.remote):
raise HTTPException(422, "Name a country or remote, or read all roles.")
params = {k: v for k, v in sub.params.items() if k not in jobs.reach.OVERRIDE_KEYS}
if body.mode == "all":
params["reach"] = "all"
elif body.mode == "own":
params |= {"countries": codes, "remote": body.remote}
sub = store.update_subscription(sub_id, params=params)
return {"source": _one_source(store, sub_id),
"reach": jobs.reach.line(jobs.reach.plan(store, [sub]), sub)}
class ReaderOut(BaseModel):
"""What a reader declares (`plugins/readers/_contract.py`), for the
Sources page: `filters` holds only the dimensions it filters."""
name: str
label: str
scope: str
listing: str
text: str
filters: dict[str, str]
places: str
robots: str
@router.get("/readers")
def list_readers() -> list[ReaderOut]:
"""Every reader that declares `READS`, boards first, by label."""
from plugins import readers
found = [m for m in readers.READERS.values() if isinstance(getattr(m, "READS", None),
readers.Reads)]
return [ReaderOut(name=m.NAME, label=m.LABEL, scope=m.READS.scope,
listing=m.READS.listing, text=m.READS.text,
filters=m.READS.filters.declared(), places=m.READS.places,
robots=m.READS.robots)
for m in sorted(found, key=lambda m: (m.READS.scope != "board", m.LABEL.lower()))]
class SearchBody(BaseModel):
provider: str
name: str = ""