"""The reach of a pull: which countries, and whether remote roles, each company board is asked for, derived from the search agents that are switched on. Computed once per pull (`plan`) and handed to each board's batch as the `reach` the runner narrows to its reader's filters (`plugins/readers/_runner.py`, `ask_for`). - **Base reach**: the union of the enabled agents' `any` gates on country, place (a town or region, mapped to its country through the gazetteer) and remote; a `remote_from` gate is remote. An agent whose `any` group is empty, or holds any other member (a function, a remote value other than `remote`, a place the gazetteer does not know), can take a role from anywhere, so the base reach is then everywhere. - **Wide reach, for the companies set to All countries** (`companies.all_countries`, global): the union over the enabled agents of their `require` reach, the intersection of each one's `require` gates on country and place, Europe today, expanded to its countries through the gazetteer, and the base reach, so the override never reads less than the board would without it. An agent with no `require` gate on place makes it everywhere. A `require` gate on remote or `remote_from` next to one on country adds nothing to it, so remote never widens it past the countries. A role in Freiburg may be open to remote, or worth asking the company about for Denmark. Favourite (affinity 2) is scoring only, where it skips the `any` gates; it does not widen what is fetched. - **Board override**: a board whose subscription `params` name a country, a place or remote gets no reach here; `ask_for` reads the params instead. - **Small boards**: a board of under `SMALL` roles gets no filter; filtering it saves little and risks closing roles. A board's size is set once, when it is added or rechecked (`plugins/ats.py`, `board_size`), and never by a pull. A board with no size yet is handed the reach it would have as a big one, with `unsized` and `small`, and its batch sizes it before it reads it, so a big new board is not read whole first; a board whose reader cannot narrow that reach is sized by its whole read instead. A reach of more countries than a `one`-filter host should be asked one by one (the wide reach's Europe on SmartRecruiters) is dropped by `ask_for`, which knows the reader (`MAX_SERIES`), so such a board is read whole. **Fetch terms** (`Agent.fetch_terms`) narrow a wide read by words, so a big board sends the roles that match one of them rather than every role: - The terms of a pull are the union over the enabled agents. An enabled agent with none wants every role, so then there are none. - They go only with the wide reach: a company set to All countries (`wide` in the reach), and a board whose reach `ask_for` drops as too many countries for a `one` host. The narrow reach (DK, SE) already leaves at most about 150 roles on any big board, and small, unknown and overridden boards get no reach at all, so none of those take terms. - A company set to All roles (`companies.all_roles`, global) takes none on any of its boards, and no later role filter (category, level) either. It changes nothing on the narrow reach today, which takes no terms. - Only a reader that declares `query="one"` takes them, one series a term; `ask_for` has the rule for terms times the reach's series. - A read with terms is never the whole board: the pull closes nothing against it (`app/jobs/pull.py`). - The search providers take none. Every saved search has a query of its own, which terms would replace; a search is a query already. """ from __future__ import annotations import json from collections.abc import Iterable, Mapping from dataclasses import dataclass, field from typing import Any from app import places from app.models import Agent, Gate, Subscription, company_key from app.storage import Store from plugins import readers from plugins.readers import _runner #: A board that listed fewer roles than this when last read whole is read #: whole. SMALL = 300 #: `state` keys of a board's size, `ats:size:`, the roles it listed #: when it was sized, and of the day it was sized (ISO), both written by #: `plugins/ats.py` (`keep_size`). SIZE = "ats:size:" SIZE_CHECKED = "ats:sizechecked:" #: `state` key of the host ids a board's site states (`{"checked", #: "facets"}`), kept by the runner (`_runner.facets_for`). FACETS = "ats:facets:" #: The params that make a board's own reach (see `_runner._PARAM_DIMENSION`). OVERRIDE_KEYS = frozenset(("countries", "country", "places", "place", "remote")) @dataclass(frozen=True) class Reach: """Countries (ISO codes) and remote, or everywhere.""" countries: frozenset[str] = frozenset() remote: bool = False everywhere: bool = False def __or__(self, other: Reach) -> Reach: if self.everywhere or other.everywhere: return EVERYWHERE return Reach(self.countries | other.countries, self.remote or other.remote) def params(self) -> dict[str, Any] | None: """What `ask_for` takes as `reach`; None for everywhere.""" if self.everywhere: return None out: dict[str, Any] = {"countries": sorted(self.countries)} if self.remote: out["remote"] = True return out def describe(self) -> str: """"DK, SE, remote", "31 countries", "everywhere".""" if self.everywhere: return "everywhere" codes = sorted(self.countries) words = [", ".join(codes)] if len(codes) <= 4 else [f"{len(codes)} countries"] if self.remote: words.append("remote") return ", ".join(w for w in words if w) or "nothing" EVERYWHERE = Reach(everywhere=True) NOWHERE = Reach() def _countries_of(ids: Iterable[int], gaz: places.Gazetteer) -> set[str]: """The ISO codes of the countries the places lie in, a continent's every country.""" out: set[str] = set() for place_id in ids: place = gaz.places.get(place_id) if place is None: continue if place.kind == "continent": out |= {gaz.places[p].key for p in gaz.under((place_id,)) if gaz.places[p].kind == "country"} elif code := gaz.country(place_id): out.add(code) return out def gate_reach(gate: Gate, gaz: places.Gazetteer) -> Reach: """Where a role that meets `gate` can be: its countries, remote, or everywhere for a gate that says nothing of place or names one the gazetteer does not know.""" values = [str(v).strip() for v in gate.values if str(v).strip()] if gate.field == "remote_from": return Reach(remote=True) if gate.field == "remote": return Reach(remote=True) if values and all(v.lower() == "remote" for v in values) \ else EVERYWHERE if gate.field not in ("country", "place") or not values: return EVERYWHERE found: set[str] = set() for value in values: named = gaz.lookup(f"country:{value}" if gate.field == "country" else value) if named is None: if gate.field == "country" and len(value) == 2 and value.isalpha(): found.add(value.upper()) continue return EVERYWHERE found |= _countries_of(named, gaz) return Reach(frozenset(found)) if found else EVERYWHERE def any_reach(agent: Agent, gaz: places.Gazetteer) -> Reach: """The union of the agent's `any` gates; everywhere with none.""" group = [g for g in agent.gates if g.mode == "any"] if not group: return EVERYWHERE out = NOWHERE for gate in group: out = out | gate_reach(gate, gaz) return out def require_reach(agent: Agent, gaz: places.Gazetteer) -> Reach: """The intersection of the agent's `require` gates on country and place, or remote alone for a `require` of remote only; everywhere with neither, or when the intersection is empty.""" kept: frozenset[str] | None = None remote = False for gate in agent.gates: if gate.mode != "require": continue reach = gate_reach(gate, gaz) if reach.everywhere: continue if reach.countries: kept = reach.countries if kept is None else kept & reach.countries elif reach.remote: remote = True if kept: return Reach(kept) return Reach(remote=True) if remote and kept is None else EVERYWHERE def fetch_terms(agents: Iterable[Agent]) -> tuple[str, ...]: """The fetch terms of a pull: the union of the agents' terms, one of each regardless of case, sorted; none when an agent has none, or with no agent.""" found: dict[str, str] = {} listed = False for agent in agents: listed = True terms = [t.strip() for t in agent.fetch_terms if t.strip()] if not terms: return () for term in terms: found.setdefault(term.casefold(), term) return tuple(found[k] for k in sorted(found)) if listed else () def wide_names(agents: Iterable[Agent], gaz: places.Gazetteer) -> str: """The wide reach as the agents' `require` gates name it, for the Companies list: "Europe" for a gate on that place, and for a gate that lists countries one by one, the continent they all lie in; up to three names as written; "everywhere" when an agent has no such gate; "" when none of that fits, which `line` says as a count.""" names: dict[str, str] = {} for agent in agents: gates = [g for g in agent.gates if g.mode == "require" and g.field in ("country", "place") and not gate_reach(g, gaz).everywhere] if not gates: return "everywhere" for gate in gates: values = [str(v).strip() for v in gate.values if str(v).strip()] if len(values) > 3: values = [_continent(gate_reach(gate, gaz).countries, gaz)] for text in values: names.setdefault(text.casefold(), text) return ", ".join(names.values()) if len(names) <= 3 and "" not in names else "" def _continent(codes: Iterable[str], gaz: places.Gazetteer) -> str: """The continent every one of `codes` lies in, "" for none. A code the gazetteer has no country for (EU) is skipped.""" known = {p.key for p in gaz.places.values() if p.kind == "country"} wanted = set(codes) & known for place in gaz.places.values(): if place.kind != "continent" or not wanted: continue under = {gaz.places[p].key for p in gaz.under((place.id,)) if gaz.places[p].kind == "country"} if wanted <= under: return place.name return "" @dataclass class Plan: """The reach of one pull: `base` for every board, `wide` for the boards of the companies in `all_countries`, each board's size by target, the fetch `terms` of the wide reads, which skip the companies in `all_roles`, `wide_names`, what the agents call the wide reach, and `facets`, the host ids the runner keeps for a board on a host that filters by them, for the Companies line.""" base: Reach wide: Reach = EVERYWHERE sizes: dict[str, int] = field(default_factory=dict) all_countries: frozenset[str] = frozenset() terms: tuple[str, ...] = () all_roles: frozenset[str] = frozenset() wide_names: str = "" facets: dict[str, dict[str, dict[str, str]]] = field(default_factory=dict) def why(self, sub: Subscription) -> tuple[Reach, str]: """The reach `sub` is read with, and why: `override`, `small`, `unknown` (a board never read whole), `countries` (its company is set to All countries) or `agents`.""" if OVERRIDE_KEYS & set(sub.params or {}): return EVERYWHERE, "override" size = self.sizes.get(sub.target) if size is None: return EVERYWHERE, "unknown" if size < SMALL: return EVERYWHERE, "small" return self._big(sub) def _big(self, sub: Subscription) -> tuple[Reach, str]: """The reach of `sub` as a board of `SMALL` roles or more.""" if company_key(sub.company or sub.label) in self.all_countries: return self.wide, "countries" return self.base, "agents" def narrows_roles(self, sub: Subscription) -> bool: """Whether `sub`'s board may be narrowed by role type: its company is not set to All roles.""" return company_key(sub.company or sub.label) not in self.all_roles def for_subscription(self, sub: Subscription) -> dict[str, Any] | None: """The `reach` a board's batch hands `ask_for`; None for none. With fetch terms, and unless its company is set to All roles, a board on the wide reach carries them and `wide`, and a board on the agents' reach carries them for `ask_for` to use if it drops the countries as too many (see the module docstring). A board with no size carries `unsized` and `small`, and the reach of a big board unless its params set their own, for its batch to size it first.""" found, why = self.why(sub) if why == "unknown": found, why = self._big(sub) out = found.params() if self.terms and why in ("countries", "agents") and self.narrows_roles(sub): out = (out or {}) | {"terms": list(self.terms)} if why == "countries": out["wide"] = True if sub.target not in self.sizes: out = (out or {}) | {"unsized": True, "small": SMALL} return out def sizes(store: Store, subs: Iterable[Subscription]) -> dict[str, int]: """Each board's size as kept (`ats:size:`); a board never sized has none.""" state = store.state out: dict[str, int] = {} for sub in subs: kept = state.get(f"{SIZE}{sub.target}", "") if kept.isdigit(): out[sub.target] = int(kept) return out def site_facets(store: Store, subs: Iterable[Subscription]) -> dict[str, dict[str, dict[str, str]]]: """The host ids the runner keeps for each board on a host whose country or remote filter is `ids` (`ats:facets:`); a board with none kept is left out.""" out: dict[str, dict[str, dict[str, str]]] = {} for sub in subs: module = readers.HOSTS.get(sub.target.partition(":")[0]) if module is None \ or "ids" not in (module.READS.filters.country, module.READS.filters.remote): continue try: kept = json.loads(store.state.get(f"{FACETS}{sub.target}") or "{}") except (TypeError, ValueError): continue found = kept.get("facets") if isinstance(kept, dict) else None if isinstance(found, dict): out[sub.target] = found return out def plan(store: Store, subs: Iterable[Subscription] | None = None) -> Plan: """The reach of a pull, from the enabled agents and the companies' two overrides, and the sizes of `subs` (every `ats` subscription when None).""" gaz = store.gazetteer() agents = store.agents(enabled_only=True) base = wide = NOWHERE for agent in agents: base = base | any_reach(agent, gaz) wide = wide | require_reach(agent, gaz) if not agents: base = wide = EVERYWHERE subs = list(store.subscriptions(source="ats") if subs is None else subs) rows = store.companies() # All countries never reads less than the board would without it: the # base's remote, which a `require` of Europe does not name, is kept. return Plan(base, wide | base, sizes(store, subs), all_countries=frozenset(c.key for c in rows if c.all_countries), terms=fetch_terms(agents), all_roles=frozenset(c.key for c in rows if c.all_roles), wide_names=wide_names(agents, gaz) if agents else "everywhere", facets=site_facets(store, subs)) def reaches(store: Store, subs: Iterable[Subscription]) -> Mapping[str, dict[str, Any]]: """`target -> reach` for the boards that get one, what a pull puts in its batches' payloads.""" subs = list(subs) found = plan(store, subs) return {s.target: r for s in subs if (r := found.for_subscription(s)) is not None} #: Why a board is read whole, as the Companies list says it. _WHOLE = {"small": f"under {SMALL} roles", "unknown": "size not known yet", "override": "set here", "countries": "all countries (everywhere)", "agents": "an agent takes any place"} def _codes(codes: Iterable[str]) -> list[str]: """Country codes as the line names them: up to four, else a count.""" found = sorted(codes) return found if len(found) <= 4 else [f"{len(found)} countries"] def _places(ask: readers.Ask) -> str: words = _codes(ask.countries) if ask.remote: words.append("remote") return ", ".join(words) def _site(module: Any, ask: readers.Ask, facets: Mapping[str, Mapping[str, str]] | None, lead: str, roles: str) -> str: """The line of a board whose site does not know every value `ask` wants on an `ids` filter, from the host ids the runner keeps (`facets`, None before any are kept), as `_runner.series` drops them: what is asked, then what the site does not know ("DK · SE not on this site · no remote option"), after `lead` and before `roles`; or, with nothing left to ask, "site lists none of DK, SE: nothing asked, not closing", since the runner reads such a board as incomplete. "" when the site knows every value, or `lead` is "" and something is still asked.""" if facets is None: return "" reads = module.READS.filters known, missing = list(ask.countries), [] if reads.country == "ids" and ask.countries and "country" in facets: listed = facets.get("country") or {} known = [c for c in ask.countries if c in listed] missing = [c for c in ask.countries if c not in listed] no_remote = bool(ask.remote) and reads.remote == "ids" and not facets.get("remote") if not (missing or no_remote): return "" if no_remote and not (ask.countries or ask.places) and "remote" not in facets: # Remote alone, on a site that states no remote facet, is read whole. return f"all places, no remote option on this site · {roles}" if lead else "" remote = bool(ask.remote) and not no_remote if not (known or ask.places or remote): if not missing: return "site has no remote option: nothing asked, not closing" named = _codes(missing) + (["remote"] if no_remote else []) return f"site lists none of {', '.join(named)}: nothing asked, not closing" if not lead: return "" words = [", ".join(_codes(known) + list(ask.places) + (["remote"] if remote else []))] if missing: words.append(f"{', '.join(_codes(missing))} not on this site") if no_remote: words.append("no remote option") return f"{lead}: {' · '.join(w for w in words if w)} · {roles}" def _roles(ask: readers.Ask) -> str: """The role-type half of the line: the agents' terms the board is asked for, or all roles.""" if not ask.terms: return "all roles" terms = ", ".join(ask.terms) if len(ask.terms) <= 5 else f"{len(ask.terms)} terms" return f"follows agents' terms: {terms}" def line(found: Plan, sub: Subscription) -> str: """What a pull asks `sub`'s board for, once its reader has narrowed the reach, in plain words, countries first and role types second: "follows agents: DK, SE, remote · all roles", "all countries (Europe) · follows agents' terms: engineer, product, data", "all countries (Europe) · all roles", "set here: DE · all roles", "all places, too many countries to ask · follows agents' terms: engineer", or, for a board read whole, why: "under 300 roles: read whole", "Greenhouse cannot filter: read whole". A board with no size yet says it is sized on the next pull. On a host that filters by its own ids, the line names what the site does not know once the runner has asked it (`_site`): "follows agents: DK · SE not on this site · no remote option · all roles", or "site lists none of DK, SE: nothing asked, not closing", or, on a host with no remote filter while the agents want remote, "follows agents: DK, SE · remote not asked on SmartRecruiters · all roles".""" module = readers.HOSTS.get(sub.target.partition(":")[0]) if module is None: return "" given, why = found.why(sub) if why == "unknown": return f"{_WHOLE[why]}: sized on the next pull" ask = _runner.ask_for(module, sub.params, found.for_subscription(sub)) roles = _roles(ask) if _runner.filtered(ask): lead = {"agents": "follows agents", "override": "set here"}.get(why, "") if site := _site(module, ask, found.facets.get(sub.target), lead, roles): return site # The reach wants remote roles and the host cannot be asked for them: # the read leaves them out, and the settle keeps the ones it has # (`app.jobs.pull`), so the line says so rather than "DK, SE". unasked = (f" · remote not asked on {module.LABEL}" if module.READS.filters.remote == "none" and _runner.remote_wanted(module, sub.params, found.for_subscription(sub)) else "") if why == "countries": named = found.wide_names or f"{len(found.wide.countries)} countries" return f"all countries ({named}){unasked} · {roles}" if why == "override": return f"set here: {_places(ask)}{unasked} · {roles}" return f"follows agents: {_places(ask)}{unasked} · {roles}" if ask.terms: if given.everywhere: return f"all countries (everywhere) · {roles}" return f"all places, too many countries to ask · {roles}" if why in ("agents", "countries") and given.everywhere: return f"{_WHOLE[why]}: read whole" if why in ("agents", "countries", "override") and not module.READS.filters.declared(): return f"{module.LABEL} cannot filter: read whole" reads = module.READS.filters if why in ("agents", "countries") and reads.country == "one" \ and len(given.countries) > _runner.MAX_SERIES: return "too many countries to ask: read whole" if why in ("agents", "countries"): return f"{module.LABEL} cannot ask for these: read whole" return f"{_WHOLE[why]}: read whole"