diff --git a/app/jobs/__init__.py b/app/jobs/__init__.py index 3f5b182..35ed510 100644 --- a/app/jobs/__init__.py +++ b/app/jobs/__init__.py @@ -12,13 +12,15 @@ task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`. sources.py what a pull reads, changed by hand: boards and searches added, switched on or off discover.py which ATS a company's careers site runs on, from its pages + reach.py which countries a pull asks each big board for, from the agents """ from __future__ import annotations from typing import Any -from app.jobs import cascade, discover, embed, extract, pull, score, sources, stages, summarise +from app.jobs import (cascade, discover, embed, extract, pull, reach, score, sources, stages, + summarise) from app.jobs.pull import search_target, seed_searches from app.jobs.score import seed_agents @@ -38,4 +40,4 @@ def bootstrap(config_path: str = "") -> tuple[Any, Any]: __all__ = ["BOOTSTRAP", "bootstrap", "cascade", "discover", "embed", "extract", "pull", - "score", "search_target", "seed_agents", "seed_searches", "sources", "stages", "summarise"] + "reach", "score", "search_target", "seed_agents", "seed_searches", "sources", "stages", "summarise"] diff --git a/app/jobs/pull.py b/app/jobs/pull.py index 41d8819..c0a1cab 100644 --- a/app/jobs/pull.py +++ b/app/jobs/pull.py @@ -17,8 +17,8 @@ cap) stores what it read and closes nothing; a paged board over the page cap is swept across pulls and closes once the sweep reached its end (see `plugins/readers/_runner.py`). -A board read with a filter (`ats:filtered:`, from its subscription's -`params`) lists only the roles in reach, so its close keeps open the roles a +A board read with a filter (`ats:filtered:`, from the agents' reach, +`app.jobs.reach`, or its subscription's `params`) lists only the roles in reach, so its close keeps open the roles a person acted on: those with a vote or an application (`_close_unlisted`). A filtered read, or one of a reader that declares `empty="suspect"`, that lists under a tenth of what the same read listed last time is recorded by the @@ -51,6 +51,7 @@ from typing import Any from pipeline import NULL, RunContext, Tally, Task, job from app.config import Config +from app.jobs.reach import reaches from app.models import company_key from app.storage import Store @@ -131,8 +132,8 @@ def _boards(config: Config, store: Store, run: str = "") -> list[dict[str, Any]] its own, and queues nothing when another run is in flight. A run with nothing to read is settled here, since no settle will see a batch of it.""" seed_searches(config, store) - targets = [(source, [s.target for s in store.subscriptions(source=source, fetchable=True)]) - for source in SOURCES] + subs = {source: store.subscriptions(source=source, fetchable=True) for source in SOURCES} + targets = [(source, [s.target for s in subs[source]]) for source in SOURCES] if not any(found for _, found in targets): if run: release_run(store, run) @@ -142,13 +143,20 @@ def _boards(config: Config, store: Store, run: str = "") -> list[dict[str, Any]] log.warning("pull: %s since %s; this one reads nothing", ALREADY_RUNNING, running(store)) return [] - return [{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run} - for source, found in targets for i in range(0, len(found), BATCH_SIZE)] + # The agents' reach, once per run, each board's in its batch (`app.jobs.reach`). + reach_of = reaches(store, subs["ats"]) + batches = [{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run} + for source, found in targets for i in range(0, len(found), BATCH_SIZE)] + for batch in batches: + if batch["source"] == "ats": + batch["reach"] = {t: reach_of[t] for t in batch["targets"] if t in reach_of} + return batches @job("pull_batch", "Reading boards", resource="http") def run_batch(config: Config, store: Store, *, targets: list[str], source: str = "ats", - run: str = "", context: RunContext = NULL) -> dict[str, Any]: + run: str = "", reach: dict[str, Any] | None = None, + context: RunContext = NULL) -> dict[str, Any]: """Fetch this batch's boards or searches and `upsert_role` every item they list. @@ -163,7 +171,9 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str = `run` is the marker of the run that queued this batch, handed to the plugin as its `run` option, so its stamps belong to that run whatever the - stored marker says by the time the batch runs. + stored marker says by the time the batch runs. `reach` is `target ->` the + reach the run computed for a board (`app.jobs.reach`), handed to the + plugin as its `reach` option. """ from plugins.ats import ATSSource from plugins.search import SearchSource @@ -189,6 +199,7 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str = options["boards"] = [{"url": target, "label": sub.company or sub.label, "params": sub.params}] options["keys"] = dict(config.sources.get("search") or {}) + options["reach"] = (reach or {}).get(target) plugin = ATSSource(options) plugin.state = store.state for item in plugin.fetch(): diff --git a/app/jobs/reach.py b/app/jobs/reach.py new file mode 100644 index 0000000..4b09be6 --- /dev/null +++ b/app/jobs/reach.py @@ -0,0 +1,266 @@ +"""The reach of a pull: which countries, and whether remote roles, each +company board is asked for, derived from the search agents that are switched +on. Computed once per pull (`plan`) and handed to each board's batch as the +`reach` the runner narrows to its reader's filters +(`plugins/readers/_runner.py`, `ask_for`). + +- **Base reach**: the union of the enabled agents' `any` gates on country, + place (a town or region, mapped to its country through the gazetteer) and + remote. An agent whose `any` group is empty, or holds any other member (a + function, a remote value other than `remote`, a place the gazetteer does + not know), can take a role from anywhere, so the base reach is then + everywhere. +- **Favourite companies** (affinity 2, which skips the `any` gates in + scoring) are read with the `require` reach of the agent that starred them: + the intersection of its `require` gates on country and place, Europe + today, expanded to its countries through the gazetteer. A company is + starred per agent, so its reach is the union over the enabled agents of + the `require` reach where it is a favourite and the `any` reach where not. +- **Board override**: a board whose subscription `params` name a country, a + place, remote or `{"reach": "all"}` gets no reach here; `ask_for` reads the + params instead. +- **Small boards**: a board that listed under `SMALL` roles when last read + whole gets no filter; filtering it saves little and risks closing roles. + +A reach of more countries than a `one`-filter host should be asked one by +one (a favourite's Europe on SmartRecruiters) is dropped by `ask_for`, which +knows the reader (`MAX_SERIES`). +""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, field +from typing import Any + +from app import places +from app.models import Agent, Gate, Subscription, company_key +from app.storage import Store +from plugins import readers +from plugins.readers import _runner + +#: A board that listed fewer roles than this when last read whole is read +#: whole. +SMALL = 300 + +#: `state` key of a board's size, `ats:size:`: the roles its last +#: unfiltered read listed, written by `plugins/ats.py`. +SIZE = "ats:size:" + +#: The params that make a board's own reach (see `_runner._PARAM_DIMENSION`). +OVERRIDE_KEYS = frozenset(("countries", "country", "places", "place", "remote", "reach")) + + +@dataclass(frozen=True) +class Reach: + """Countries (ISO codes) and remote, or everywhere.""" + + countries: frozenset[str] = frozenset() + remote: bool = False + everywhere: bool = False + + def __or__(self, other: Reach) -> Reach: + if self.everywhere or other.everywhere: + return EVERYWHERE + return Reach(self.countries | other.countries, self.remote or other.remote) + + def params(self) -> dict[str, Any] | None: + """What `ask_for` takes as `reach`; None for everywhere.""" + if self.everywhere: + return None + out: dict[str, Any] = {"countries": sorted(self.countries)} + if self.remote: + out["remote"] = True + return out + + def describe(self) -> str: + """"DK, SE, remote", "31 countries", "everywhere".""" + if self.everywhere: + return "everywhere" + codes = sorted(self.countries) + words = [", ".join(codes)] if len(codes) <= 4 else [f"{len(codes)} countries"] + if self.remote: + words.append("remote") + return ", ".join(w for w in words if w) or "nothing" + + +EVERYWHERE = Reach(everywhere=True) +NOWHERE = Reach() + + +def _countries_of(ids: Iterable[int], gaz: places.Gazetteer) -> set[str]: + """The ISO codes of the countries the places lie in, a continent's every + country.""" + out: set[str] = set() + for place_id in ids: + place = gaz.places.get(place_id) + if place is None: + continue + if place.kind == "continent": + out |= {gaz.places[p].key for p in gaz.under((place_id,)) + if gaz.places[p].kind == "country"} + elif code := gaz.country(place_id): + out.add(code) + return out + + +def gate_reach(gate: Gate, gaz: places.Gazetteer) -> Reach: + """Where a role that meets `gate` can be: its countries, remote, or + everywhere for a gate that says nothing of place or names one the + gazetteer does not know.""" + values = [str(v).strip() for v in gate.values if str(v).strip()] + if gate.field == "remote": + return Reach(remote=True) if values and all(v.lower() == "remote" for v in values) \ + else EVERYWHERE + if gate.field not in ("country", "place") or not values: + return EVERYWHERE + found: set[str] = set() + for value in values: + named = gaz.lookup(f"country:{value}" if gate.field == "country" else value) + if named is None: + if gate.field == "country" and len(value) == 2 and value.isalpha(): + found.add(value.upper()) + continue + return EVERYWHERE + found |= _countries_of(named, gaz) + return Reach(frozenset(found)) if found else EVERYWHERE + + +def any_reach(agent: Agent, gaz: places.Gazetteer) -> Reach: + """The union of the agent's `any` gates; everywhere with none.""" + group = [g for g in agent.gates if g.mode == "any"] + if not group: + return EVERYWHERE + out = NOWHERE + for gate in group: + out = out | gate_reach(gate, gaz) + return out + + +def require_reach(agent: Agent, gaz: places.Gazetteer) -> Reach: + """The intersection of the agent's `require` gates on country and place, + or remote alone for a `require` of remote only; everywhere with neither, + or when the intersection is empty.""" + kept: frozenset[str] | None = None + remote = False + for gate in agent.gates: + if gate.mode != "require": + continue + reach = gate_reach(gate, gaz) + if reach.everywhere: + continue + if reach.countries: + kept = reach.countries if kept is None else kept & reach.countries + elif reach.remote: + remote = True + if kept: + return Reach(kept) + return Reach(remote=True) if remote and kept is None else EVERYWHERE + + +@dataclass +class Plan: + """The reach of one pull: `base` for every board, `favourites` for the + companies some agent starred, and each board's size by target.""" + + base: Reach + favourites: dict[str, Reach] = field(default_factory=dict) + sizes: dict[str, int] = field(default_factory=dict) + + def why(self, sub: Subscription) -> tuple[Reach, str]: + """The reach `sub` is read with, and why: `override`, `small`, + `unknown` (a board never read whole), `favourite` or `agents`.""" + if OVERRIDE_KEYS & set(sub.params or {}): + return EVERYWHERE, "override" + size = self.sizes.get(sub.target) + if size is None: + return EVERYWHERE, "unknown" + if size < SMALL: + return EVERYWHERE, "small" + key = company_key(sub.company or sub.label) + if key in self.favourites: + return self.favourites[key], "favourite" + return self.base, "agents" + + def for_subscription(self, sub: Subscription) -> dict[str, Any] | None: + """The `reach` a board's batch hands `ask_for`; None for none.""" + return self.why(sub)[0].params() + + +def sizes(store: Store, subs: Iterable[Subscription]) -> dict[str, int]: + """Each board's size: the roles its last unfiltered read listed + (`ats:size:`), else, for a board read before that was kept and + never filtered, its open roles. A board never read has none.""" + state = store.state + counts = store.open_countries_by_subscription() + out: dict[str, int] = {} + for sub in subs: + kept = state.get(f"{SIZE}{sub.target}", "") + if kept.isdigit(): + out[sub.target] = int(kept) + elif state.get(f"ats:ok:{sub.target}") and not state.get(f"ats:filtered:{sub.target}"): + out[sub.target] = sum(counts.get(sub.id, {}).values()) + return out + + +def plan(store: Store, subs: Iterable[Subscription] | None = None) -> Plan: + """The reach of a pull, from the enabled agents and their favourites, and + the sizes of `subs` (every `ats` subscription when None).""" + gaz = store.gazetteer() + agents = store.agents(enabled_only=True) + reaches = {a.id: (any_reach(a, gaz), require_reach(a, gaz)) for a in agents} + base = NOWHERE + for any_of, _ in reaches.values(): + base = base | any_of + if not agents: + base = EVERYWHERE + starred: dict[str, set[int]] = {} + for agent in agents: + for company in store.companies(agent.id): + if company.affinity == 2: + starred.setdefault(company.key, set()).add(agent.id) + favourites: dict[str, Reach] = {} + for key, by in starred.items(): + reach = NOWHERE + for agent_id, (any_of, required) in reaches.items(): + reach = reach | (required if agent_id in by else any_of) + favourites[key] = reach + subs = list(store.subscriptions(source="ats") if subs is None else subs) + return Plan(base, favourites, sizes(store, subs)) + + +def reaches(store: Store, subs: Iterable[Subscription]) -> Mapping[str, dict[str, Any]]: + """`target -> reach` for the boards that get one, what a pull puts in + its batches' payloads.""" + subs = list(subs) + found = plan(store, subs) + return {s.target: r for s in subs if (r := found.for_subscription(s)) is not None} + + +#: Why a board is read whole, as the Companies list says it. +_WHOLE = {"small": f"under {SMALL} roles", "unknown": "not read whole yet", + "override": "set here", "favourite": "favourite, too many countries to ask", + "agents": "an agent takes any place"} + + +def line(found: Plan, sub: Subscription) -> str: + """What a pull asks `sub`'s board for, once its reader has narrowed the + reach, in a few words: "DK, SE, remote · agents", "all roles · under 300 + roles", "all roles · Greenhouse cannot filter".""" + module = readers.HOSTS.get(sub.target.partition(":")[0]) + if module is None: + return "" + given, why = found.why(sub) + ask = _runner.ask_for(module, sub.params, given.params()) + if _runner.filtered(ask): + words = sorted(ask.countries) if len(ask.countries) <= 4 \ + else [f"{len(ask.countries)} countries"] + if ask.remote: + words.append("remote") + return f"{', '.join(words)} · {'set here' if why == 'override' else why}" + if why in ("agents", "favourite") and not given.everywhere: + if not module.READS.filters.declared(): + return f"all roles · {module.LABEL} cannot filter" + if why == "agents": + return f"all roles · {module.LABEL} cannot ask for these" + return f"all roles · {_WHOLE[why]}" diff --git a/app/static/companies.js b/app/static/companies.js index 6791537..966da97 100644 --- a/app/static/companies.js +++ b/app/static/companies.js @@ -17,6 +17,11 @@ * into the Follow form's `#b-host`, and a slug. Saving posts to * `/api/companies/{key}/board`, which reads the board once before keeping * it; Detach stops reading it and keeps the company. + * + * A row with a board says what a pull asks it for ("reads DK, SE · agents", + * `reach` from app/jobs/reach.py), and its editor has a second line that + * sets the board's own reach: the agents', all roles, or its own countries + * and remote, posted to `/api/sources/{id}/reach`. */ (function () { "use strict"; @@ -90,6 +95,7 @@ sub.appendChild(link); } sub.append(` · ${count(company.open_roles, "open role")}`); + if (company.reach) sub.append(` · reads ${company.reach}`); if (company.failing_since) { // The board has failed on every pull since then; after 21 days its // roles close. @@ -199,6 +205,58 @@ return form; } + /** The open row's reach editor, for a row with a board: the agents' reach, + all roles, or the board's own countries and remote. */ + function reachEditor(company, rerender) { + if (company.subscription_id == null) return null; + const own = company.reach_params || {}; + const form = document.createElement("form"); + form.className = "board-edit"; + form.autocomplete = "off"; + const mode = document.createElement("select"); + mode.setAttribute("aria-label", `What ${company.name}'s board is asked for`); + for (const [value, label] of [["agents", "Agents' reach"], ["all", "All roles"], + ["own", "These countries"]]) { + mode.appendChild(new Option(label, value)); + } + mode.value = own.reach === "all" ? "all" : (own.countries || own.remote != null ? "own" : "agents"); + const codes = document.createElement("input"); + codes.type = "text"; + codes.spellcheck = false; + codes.placeholder = "DK SE"; + codes.value = (own.countries || []).join(" "); + codes.setAttribute("aria-label", "Country codes"); + const remote = document.createElement("select"); + remote.setAttribute("aria-label", "Remote roles"); + remote.append(new Option("and remote", "yes"), new Option("no remote", "no")); + remote.value = own.remote ? "yes" : "no"; + const save = document.createElement("button"); + save.type = "submit"; + save.textContent = "Set reach"; + const shown = () => { codes.hidden = remote.hidden = mode.value !== "own"; }; + mode.addEventListener("change", shown); + shown(); + form.append(mode, codes, remote, save); + form.addEventListener("submit", async (event) => { + event.preventDefault(); + save.disabled = true; + try { + const result = await api(`/api/sources/${company.subscription_id}/reach`, { + method: "POST", + body: { mode: mode.value, countries: codes.value.split(/[\s,]+/).filter(Boolean), + remote: remote.value === "yes" }, + }); + window.toast.info(`${company.name}: reads ${result.reach} from the next pull.`); + editing = null; + await loadCompanies(agentId()); + rerender(); + } catch (err) { + save.disabled = false; + } + }); + return form; + } + /** The board switch, or nothing for a company with no board: there is no subscription to switch. */ function onSwitch(company, rerender) { @@ -274,7 +332,11 @@ row.appendChild(affinityControl(company.key, company.affinity, async () => render(), agent)); } row.appendChild(subLine(company, render)); - if (editing === company.key) row.appendChild(boardEditor(company, render)); + if (editing === company.key) { + row.appendChild(boardEditor(company, render)); + const reach = reachEditor(company, render); + if (reach) row.appendChild(reach); + } body.appendChild(row); } const open = rows.reduce((sum, c) => sum + c.open_roles, 0); diff --git a/app/static/sources.js b/app/static/sources.js index f2b484d..c986b8b 100644 --- a/app/static/sources.js +++ b/app/static/sources.js @@ -15,6 +15,11 @@ * * It is also the page's first script, so it shows the one section of the * four the URL fragment names and marks it in the settings rail. + * + * Under the Follow form of Company boards, a fold lists what each reader + * declares (`/api/readers`): how its list arrives, where the text is, what + * it filters at the source, how it states places, and the robots.txt + * decision taken when it was built. */ (function () { "use strict"; @@ -277,6 +282,44 @@ } } + // ----------------------------------------------------------------- readers + + /** "country one, remote ids", or "none". */ + function filters(reader) { + const shown = Object.entries(reader.filters).map(([name, how]) => `${name} ${how}`); + return shown.length ? shown.join(", ") : "none"; + } + + function renderReaders(readers) { + const fold = document.createElement("details"); + fold.className = "prune"; + fold.id = "readers"; + const summary = document.createElement("summary"); + summary.textContent = `What each reader does (${readers.length})`; + const list = document.createElement("div"); + list.className = "srows"; + for (const reader of readers) { + const row = document.createElement("div"); + row.className = "srow"; + const name = document.createElement("span"); + name.className = "name"; + const label = document.createElement("span"); + label.textContent = reader.label; + const scope = document.createElement("span"); + scope.className = "badge"; + scope.textContent = reader.scope === "search" ? "job board" : "company board"; + name.append(label, scope); + const sub = document.createElement("span"); + sub.className = "sub"; + sub.textContent = [`list ${reader.listing}`, `text ${reader.text}`, + `filters ${filters(reader)}`, `places ${reader.places}`, reader.robots].join(" · "); + row.append(name, sub); + list.appendChild(row); + } + fold.append(summary, list); + $("#board-form").after(fold); + } + // ------------------------------------------------------ one section shown // Sources shows one section: the one the fragment names or contains, or the @@ -317,6 +360,7 @@ document.addEventListener("DOMContentLoaded", async () => { if (!$("#searches-list")) return; + if ($("#board-form")) api("/api/readers").then(renderReaders, () => {}); try { await load(); } catch (err) { diff --git a/app/web/api.py b/app/web/api.py index fcd820f..9c7b5a1 100644 --- a/app/web/api.py +++ b/app/web/api.py @@ -473,6 +473,10 @@ class CompanyOut(BaseModel): from_search: bool = False hand_reason: str = "" by_hand: bool = False + #: The board's own reach in its params (`jobs.reach.OVERRIDE_KEYS`), and + #: what a pull asks its board for (`jobs.reach.line`). + reach_params: dict[str, Any] = {} + reach: str = "" def _company_boards(store: Store) -> dict[str, tuple[int, str, bool, str]]: @@ -495,10 +499,13 @@ def _company_outs(store: Store, agent: int | None) -> list[CompanyOut]: ever_boarded = {company_key(s.company) for s in store.subscriptions(source="ats")} counts = store.open_role_counts() failing = jobs.pull.failing_boards(store) + subs = {s.id: s for s in store.subscriptions(source="ats")} + plan = jobs.reach.plan(store, subs.values()) out = [] for c in store.companies(agent): sub_id, host, on, target = boards.get(c.key, (None, "", None, "")) since, reason = failing.get(target, ("", "")) + sub = subs.get(sub_id) if sub_id is not None else None from_search = jobs.sources.known_from_search_only(c, ever_boarded) out.append(CompanyOut(key=c.key, name=c.name, notes=c.notes, board=host, subscription_id=sub_id, on=on, affinity=c.affinity, @@ -508,7 +515,10 @@ def _company_outs(store: Store, agent: int | None) -> list[CompanyOut]: watch=c.watch, careers_opened_at=c.careers_opened_at, failing_since=since, failing_reason=reason, from_search=from_search, hand_reason=c.hand_reason, - by_hand=c.key not in boards and not from_search)) + by_hand=c.key not in boards and not from_search, + reach_params={k: v for k, v in (sub.params if sub else {}).items() + if k in jobs.reach.OVERRIDE_KEYS}, + reach=jobs.reach.line(plan, sub) if sub else "")) return out @@ -803,6 +813,66 @@ def switch_source(sub_id: int, body: SwitchBody, store: StoreDep, return {"source": _one_source(store, sub_id), "closed": closed} +class ReachBody(BaseModel): + #: `agents`: the enabled agents' reach; `all`: every role; `own`: the + #: `countries` (ISO codes) and remote given here. + mode: Literal["agents", "all", "own"] + countries: list[str] = [] + remote: bool = False + + +@router.post("/sources/{sub_id}/reach") +def set_reach(sub_id: int, body: ReachBody, store: StoreDep) -> dict[str, Any]: + """A board's own reach, which wins over the agents' (see + `app/jobs/reach.py`), or none. The next pull reads the board so; a + narrower reach closes the roles outside it then, except those with a + vote or an application.""" + sub = store.subscription(sub_id) + if sub is None or sub.source != "ats": + raise HTTPException(404, f"no such board: {sub_id}") + codes = sorted({c.strip().upper() for c in body.countries if c.strip()}) + if unknown := [c for c in codes if not places.known(f"country:{c}") or c == "EU"]: + raise HTTPException(422, f"Not a country code: {', '.join(unknown)}.") + if body.mode == "own" and not (codes or body.remote): + raise HTTPException(422, "Name a country or remote, or read all roles.") + params = {k: v for k, v in sub.params.items() if k not in jobs.reach.OVERRIDE_KEYS} + if body.mode == "all": + params["reach"] = "all" + elif body.mode == "own": + params |= {"countries": codes, "remote": body.remote} + sub = store.update_subscription(sub_id, params=params) + return {"source": _one_source(store, sub_id), + "reach": jobs.reach.line(jobs.reach.plan(store, [sub]), sub)} + + +class ReaderOut(BaseModel): + """What a reader declares (`plugins/readers/_contract.py`), for the + Sources page: `filters` holds only the dimensions it filters.""" + + name: str + label: str + scope: str + listing: str + text: str + filters: dict[str, str] + places: str + robots: str + + +@router.get("/readers") +def list_readers() -> list[ReaderOut]: + """Every reader that declares `READS`, boards first, by label.""" + from plugins import readers + + found = [m for m in readers.READERS.values() if isinstance(getattr(m, "READS", None), + readers.Reads)] + return [ReaderOut(name=m.NAME, label=m.LABEL, scope=m.READS.scope, + listing=m.READS.listing, text=m.READS.text, + filters=m.READS.filters.declared(), places=m.READS.places, + robots=m.READS.robots) + for m in sorted(found, key=lambda m: (m.READS.scope != "board", m.LABEL.lower()))] + + class SearchBody(BaseModel): provider: str name: str = "" diff --git a/plugins/README.md b/plugins/README.md index 50951c6..cf69506 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -69,9 +69,11 @@ Numbers are from the live database after the pull of 2026-09-29. For the "list, | robots.txt | read before the list for the 14 hosts that name a page: Workday, SuccessFactors, Oracle, Phenom, Eightfold, Jibe, Gem, BambooHR, Pinpoint, Deel, Jobylon, Rippling, HiBob, HR Manager; kept for one day per board | A disallow fails the board with "robots.txt disallows it" and counts toward the 21 days. An unreachable file (5xx, 429, timeout) fails the board for this pull and is asked again next pull. A 4xx means no rules, which allows. `Crawl-delay` is not honoured. The eight API hosts (Greenhouse, Lever, Ashby, Workable, SmartRecruiters, Recruitee, Personio, Teamtailor) and the search providers read no robots.txt. | | 429 | first page: the board fails; a later page: the read stops, keeps what it read and is incomplete; a per-role read: the per-role reads of that board stop until the next pull | Nothing is closed on a list cut short by a 429. A search that fails is skipped for the pull. | +How far a board is read: a pull asks each board over 300 roles for the reach of the search agents that are switched on (`app/jobs/reach.py`): the union of their `any` gates on country, town (as its country) and remote, or every role when one agent can take a role from anywhere. A company starred as favourite is asked for its agent's `require` reach, Europe today; a host that asks one country a request reads a favourite whole instead of 40 times. A board's own reach, set on the Companies list, wins. A complete filtered read closes the roles outside the reach, except those with a vote or an application. + What no source does: -- Filter by country or place on the host's side. Every board is read whole; only the search providers take a region. +- Filter on the host's side for a host other than SmartRecruiters, SuccessFactors, Workday, Oracle, Phenom and Eightfold. Greenhouse, Ashby, Recruitee and the rest are read whole. - Read more than one board per company. DFDS is read from its Danish Teamtailor board only. - Use a key taken from a site's script. Maersk, New Work and Welcome to the Jungle are on the Watchlist for that reason. - Read a page that needs a browser to run its script. @@ -223,6 +225,7 @@ What no source does: | Remote | `location.remote`, true or false (122 remote) | | Pay | none: the list states no pay | | Found by discovery | yes, from a `jobs.`, `careers.` or `api.smartrecruiters.com` link; a guessed name is probed | +| Filter at the source | `?country=dk`, one country a request; remote is ignored by the API, so a remote role outside the reach is not read. Bosch DK: 2 of 4,830 (2026-09-30). Which countries is the agents' reach, see below | | Limitations | 10,000 roles per board (100 pages). The identifier is read without case, so a lowercased slug works. An unknown identifier answers an empty list, not an error: a mistyped slug reads as a complete board with no roles and would close a company's roles. | The reader's "Fetch the posting" button shows for these roles, since their description is empty; it reads one posting page on demand and does not feed `embed` or `extract`. Whether it reads a SmartRecruiters page cleanly was not checked. @@ -307,6 +310,7 @@ The reader's "Fetch the posting" button shows for these roles, since their descr | Remote | none | | Pay | none | | Found by discovery | yes, from a link to a role, `/go/` page or search on the site's own host; a link to the SAP back end names the host only; not probed. A site on the company's own domain whose pages link only the back end is set by hand | +| Filter at the source | `optionsFacetsDD_country=DK`, one country a request; no remote facet. Vestas DK: 64 of 668 (2026-09-30). Which countries is the agents' reach, see below | | Limitations | Only the default locale's roles are listed: a role that exists only in another locale (Carlsberg's French brand, 37 of 105) is not read. Only `/search/` and `/job/` are read, never `/services/` or the `successfactors.eu` back end, so a company with only the back end (Bang & Olufsen) cannot be read. 100 pages per board, so 1,000 to 10,000 roles depending on the page size. A third layout or a bot wall that answers 200 fails the board with "not a SuccessFactors search page". Read from this machine only; not yet from the deployed server. | ### Workday @@ -323,6 +327,7 @@ The reader's "Fetch the posting" button shows for these roles, since their descr | Remote | `remoteType`, where the tenant sends it: remote, hybrid, onsite (585 with a value) | | Pay | none: no pay field; pay appears only inside the text | | Found by discovery | yes, from a `myworkdayjobs.com` link with a site; not probed | +| Filter at the source | the tenant's own facets in `appliedFacets`, ids read by `facets` once a day: the facet whose values are countries, and any value that reads as remote. Two facets intersect, so remote is a series of its own. LEGO DK: 97 of 455 (2026-09-30). A tenant with no country facet is read whole. Which countries is the agents' reach, see below | | Limitations | 2,000 roles per board (100 pages of 20). A tenant has several sites and each is its own board. robots.txt paths are matched without case, as Workday serves any case. Every tenant is behind Cloudflare and was read from this machine only. | ### Oracle Recruiting Cloud @@ -339,6 +344,7 @@ The reader's "Fetch the posting" button shows for these roles, since their descr | Remote | `WorkplaceTypeCode`: remote, hybrid, onsite (741 with a value) | | Pay | none: the resource states no pay | | Found by discovery | yes, from a candidate site or role link on the pod, or a REST call naming `siteNumber`; not probed | +| Filter at the source | `selectedLocationsFacet` with the pod's geography ids, and `selectedWorkplaceTypesFacet=ORA_REMOTE` as a series of its own. `locationsFacet` lists the ten biggest countries only, so a country missing from a full list is read unfiltered: fa-evmr (Nokia) has neither DK nor SE listed and is read whole. Which countries is the agents' reach, see below | | Limitations | The host is the tenant's Fusion pod, not the company's domain (`jobs.nokia.com` redirects to it). Links are built for locale `en`. The department fields were empty on every role of the four boards. | ### Phenom @@ -355,6 +361,7 @@ The reader's "Fetch the posting" button shows for these roles, since their descr | Remote | per role only, from the page's `checkRemote` (120 with a value) | | Pay | none | | Found by discovery | yes, from a `///search-results` link; a link to Phenom itself names the host only; not probed | +| Filter at the source | `selected_fields.country`, a list of English names in one request; no remote. Philips NL and DE: 111 of 873 (2026-09-30). Which countries is the agents' reach, see below | | Limitations | The locale is built as `_`, the spelling of the three sites read; another site may spell it differently. 10,000 roles per board. A role page without a description fails the read and is retried on a later pull. | ### Eightfold @@ -371,7 +378,8 @@ The reader's "Fetch the posting" button shows for these roles, since their descr | Remote | `workLocationOption`: onsite, hybrid, remote (628 with a value) | | Pay | none | | Found by discovery | yes, from an `eightfold.ai` link that carries `domain=`; a bare `eightfold.ai` link names the host only; not probed | -| Limitations | With 10 per request and 100 pages, a board over 1,000 roles is never read to the end; Infineon (1,289) and Microsoft (about 2,300) are not attached for that reason. The search takes a location filter, but a target has nowhere to state one yet (planned in wave 2). Verified on three tenants only. | +| Filter at the source | `location=`, one country a request; no remote. Ericsson SE: 154 of 618 (2026-09-30). Which countries is the agents' reach, see below | +| Limitations | With 10 per request and 100 pages, a board over 1,000 roles is never read to the end; unless the country filter brings it under the cap: Infineon (1,289, 163 in DK and SE) and Microsoft (about 2,300, 60) become readable that way. Verified on three tenants only. | ### Gem diff --git a/plugins/ats.py b/plugins/ats.py index 5b718dc..db982de 100644 --- a/plugins/ats.py +++ b/plugins/ats.py @@ -17,8 +17,12 @@ A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`, `lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what its slug is. Every board is read independently; one that fails is logged, recorded as failing under `ats:failing:`, and skipped. A board's -subscription `params` (`{"countries": ["DK"], "remote": true}`) are its -filter, narrowed to what its reader declares (`_runner.ask_for`). +filter is the `reach` option the pull hands it (the agents', see +`app/jobs/reach.py`), over which its subscription `params` (`{"countries": +["DK"], "remote": true}`, `{"reach": "all"}`) win, narrowed to what its +reader declares (`_runner.ask_for`). An unfiltered read records the board's +size under `ats:size:`, which the reach reads to leave small boards +whole. A pull reads listings only. A host that states a role's text on the role's own page (`text="detail"`) is read there by `app/jobs/enrich.py` after @@ -353,7 +357,7 @@ class ATSSource(Source): raise ValueError(f"{target!r} is not host:slug for a known ATS host") module = readers.HOSTS[host] get = _getter(client, host, slug) - ask = _runner.ask_for(module, self.params_for(target), + ask = _runner.ask_for(module, self.params_for(target), self.options.get("reach"), keys=self.options.get("keys")) listing = _runner.read(self.state, "ats", target, module, slug, get, ask, limits=_limits()) @@ -392,6 +396,12 @@ class ATSSource(Source): # as the whole of it. _runner.stamp(self.state, "ats", target, run_marker, listing, [compound(i) for i in listing.ids]) + if not listing.filtered: + size = len(listing.ids) + if not listing.complete: + kept = self.state.get(f"ats:size:{target}", "") + size = max(size, int(kept) if kept.isdigit() else 0) + self.state[f"ats:size:{target}"] = str(size) if detail_sets is not None: # A complete listing drops the ids it no longer lists. for key, ids in zip(detail_keys(target), detail_sets, strict=False): diff --git a/plugins/readers/_contract.py b/plugins/readers/_contract.py index 250d494..4f8305f 100644 --- a/plugins/readers/_contract.py +++ b/plugins/readers/_contract.py @@ -21,6 +21,11 @@ Signatures of a module that declares `READS`, with `Get` and `Page` from def detail(get: Get, slug: str, row: Row) -> dict[str, Any] # text="detail" def facets(get: Get, slug: str) -> dict[str, dict[str, str]] # an "ids" filter +`facets` maps each `ids` dimension to `{value: host id}`: an ISO code for +`country`, a place name for `place`, any word for `remote`. A dimension the +host has no facet for is left out, and the runner then reads it unfiltered; +a dimension present without a wanted value means the host has none of it. + A module without `READS` keeps the signatures without `ask` until it declares; `undeclared()` in the package lists those modules. """ @@ -146,11 +151,20 @@ class Ask(_Frozen): carries one value. With `ids`, `countries` and `places` hold the host's ids from `facets`, not codes or names. `params` are a search's saved params (and a resolved key under `"key"`). An empty `Ask()` is an - unfiltered read.""" + unfiltered read. + + `remote=True` asks for remote roles. A reach of countries and remote + means "in these countries, or remote", which no host can ask in one + request (two facets intersect), so the runner makes the remote roles a + series of their own: a reader never gets both `remote=True` and a + country or a place. With `remote="ids"`, `remote_ids` holds the host's + ids of its remote values (`facets(...)["remote"]`, any key), and + `remote` stays True.""" countries: tuple[str, ...] = () places: tuple[str, ...] = () remote: bool | None = None + remote_ids: tuple[str, ...] = () query: str = "" params: dict[str, Any] = {} diff --git a/plugins/readers/_countries.py b/plugins/readers/_countries.py new file mode 100644 index 0000000..2acea40 --- /dev/null +++ b/plugins/readers/_countries.py @@ -0,0 +1,93 @@ +"""Country codes and the English names hosts write, for the readers that +filter by a country's name (Eightfold, Phenom) or name a host id by it +(Workday, Oracle), and for SuccessFactors, which writes a place's country as +its code. + +`NAMES` is ISO 3166-1 alpha-2 to the English short name. `code_of` reads a +name back, with the spellings hosts use that are not the short name +("United States of America", "Korea, Republic of", "Czech Republic"). +""" + +from __future__ import annotations + +import unicodedata + +#: ISO 3166-1 alpha-2 to the English name, which is what `country_of` reads: +#: a bare "DE" or "SE" in a location line is taken for Delaware or nothing. +NAMES: dict[str, str] = dict(entry.strip().split(" ", 1) for entry in """ +AD Andorra | AE United Arab Emirates | AF Afghanistan | AG Antigua and Barbuda | AI +Anguilla | AL Albania | AM Armenia | AO Angola | AR Argentina | AS American Samoa | AT +Austria | AU Australia | AW Aruba | AX Åland Islands | AZ Azerbaijan | BA Bosnia and +Herzegovina | BB Barbados | BD Bangladesh | BE Belgium | BF Burkina Faso | BG Bulgaria +| BH Bahrain | BI Burundi | BJ Benin | BM Bermuda | BN Brunei | BO Bolivia | BQ +Caribbean Netherlands | BR Brazil | BS Bahamas | BT Bhutan | BW Botswana | BY Belarus +| BZ Belize | CA Canada | CD Democratic Republic of the Congo | CF Central African +Republic | CG Republic of the Congo | CH Switzerland | CI Côte d'Ivoire | CK Cook +Islands | CL Chile | CM Cameroon | CN China | CO Colombia | CR Costa Rica | CU Cuba | +CV Cabo Verde | CW Curaçao | CY Cyprus | CZ Czechia | DE Germany | DJ Djibouti | DK +Denmark | DM Dominica | DO Dominican Republic | DZ Algeria | EC Ecuador | EE Estonia | +EG Egypt | ER Eritrea | ES Spain | ET Ethiopia | FI Finland | FJ Fiji | FK Falkland +Islands | FM Micronesia | FO Faroe Islands | FR France | GA Gabon | GB United Kingdom +| GD Grenada | GE Georgia | GF French Guiana | GG Guernsey | GH Ghana | GI Gibraltar | +GL Greenland | GM Gambia | GN Guinea | GP Guadeloupe | GQ Equatorial Guinea | GR +Greece | GT Guatemala | GU Guam | GW Guinea-Bissau | GY Guyana | HK Hong Kong | HN +Honduras | HR Croatia | HT Haiti | HU Hungary | ID Indonesia | IE Ireland | IL Israel +| IM Isle of Man | IN India | IQ Iraq | IR Iran | IS Iceland | IT Italy | JE Jersey | +JM Jamaica | JO Jordan | JP Japan | KE Kenya | KG Kyrgyzstan | KH Cambodia | KI +Kiribati | KM Comoros | KN Saint Kitts and Nevis | KP North Korea | KR South Korea | +KW Kuwait | KY Cayman Islands | KZ Kazakhstan | LA Laos | LB Lebanon | LC Saint Lucia +| LI Liechtenstein | LK Sri Lanka | LR Liberia | LS Lesotho | LT Lithuania | LU +Luxembourg | LV Latvia | LY Libya | MA Morocco | MC Monaco | MD Moldova | ME +Montenegro | MG Madagascar | MH Marshall Islands | MK North Macedonia | ML Mali | MM +Myanmar | MN Mongolia | MO Macao | MP Northern Mariana Islands | MQ Martinique | MR +Mauritania | MS Montserrat | MT Malta | MU Mauritius | MV Maldives | MW Malawi | MX +Mexico | MY Malaysia | MZ Mozambique | NA Namibia | NC New Caledonia | NE Niger | NG +Nigeria | NI Nicaragua | NL Netherlands | NO Norway | NP Nepal | NR Nauru | NU Niue | +NZ New Zealand | OM Oman | PA Panama | PE Peru | PF French Polynesia | PG Papua New +Guinea | PH Philippines | PK Pakistan | PL Poland | PR Puerto Rico | PS Palestine | PT +Portugal | PW Palau | PY Paraguay | QA Qatar | RE Réunion | RO Romania | RS Serbia | +RU Russia | RW Rwanda | SA Saudi Arabia | SB Solomon Islands | SC Seychelles | SD +Sudan | SE Sweden | SG Singapore | SI Slovenia | SK Slovakia | SL Sierra Leone | SM +San Marino | SN Senegal | SO Somalia | SR Suriname | SS South Sudan | ST São Tomé and +Príncipe | SV El Salvador | SY Syria | SZ Eswatini | TC Turks and Caicos Islands | TD +Chad | TG Togo | TH Thailand | TJ Tajikistan | TL Timor-Leste | TM Turkmenistan | TN +Tunisia | TO Tonga | TR Turkey | TT Trinidad and Tobago | TV Tuvalu | TW Taiwan | TZ +Tanzania | UA Ukraine | UG Uganda | US United States | UY Uruguay | UZ Uzbekistan | VA +Vatican City | VC Saint Vincent and the Grenadines | VE Venezuela | VG British Virgin +Islands | VI U.S. Virgin Islands | VN Vietnam | VU Vanuatu | WS Samoa | XK Kosovo | YE +Yemen | ZA South Africa | ZM Zambia | ZW Zimbabwe +""".replace("\n", " ").split(" | ")) + +#: Other names hosts give a country: Workday's "United States of America" and +#: "Korea, Republic of", older and local spellings. +_OTHER = { + "united states of america": "US", "usa": "US", "u.s.a.": "US", "u.s.": "US", + "united kingdom of great britain and northern ireland": "GB", "great britain": "GB", + "uk": "GB", "england": "GB", "korea, republic of": "KR", "republic of korea": "KR", + "korea": "KR", "czech republic": "CZ", "turkiye": "TR", "viet nam": "VN", + "russian federation": "RU", "hong kong sar": "HK", "hong kong sar, china": "HK", + "taiwan, province of china": "TW", "macau": "MO", "the netherlands": "NL", + "holland": "NL", "slovak republic": "SK", "republic of ireland": "IE", + "ivory coast": "CI", "cote d'ivoire": "CI", "north macedonia, republic of": "MK", + "macedonia": "MK", "iran, islamic republic of": "IR", "moldova, republic of": "MD", + "tanzania, united republic of": "TZ", "united arab emirates (uae)": "AE", "uae": "AE", +} + + +def _fold(name: str) -> str: + decomposed = unicodedata.normalize("NFKD", name.strip().casefold()) + return " ".join("".join(c for c in decomposed if not unicodedata.combining(c)).split()) + + +_BY_NAME: dict[str, str] = {_fold(name): code for code, name in NAMES.items()} | _OTHER + + +def code_of(name: str) -> str: + """The ISO code of a country written by name, '' for a name not known. + Case and accents do not matter.""" + return _BY_NAME.get(_fold(name or ""), "") + + +def name_of(code: str) -> str: + """The English name of an ISO code, '' for a code not known.""" + return NAMES.get((code or "").strip().upper(), "") diff --git a/plugins/readers/_runner.py b/plugins/readers/_runner.py index 73bf67e..bcdb189 100644 --- a/plugins/readers/_runner.py +++ b/plugins/readers/_runner.py @@ -289,14 +289,22 @@ def resolve_key(key: Key, keys: Mapping[str, Any] | None) -> dict[str, str]: return out +#: The most series a reach may cost on a `one` filter: a reach of more +#: countries (a favourite's Europe) reads such a host unfiltered instead. +MAX_SERIES = 8 + + def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None = None, reach: Mapping[str, Any] | None = None, *, keys: Mapping[str, Any] | None = None) -> Ask: - """The `Ask` one target resolves to. `reach` (from the agents, package F) - and then the subscription's own `params` (a board's override, e.g. - `{"countries": ["DK"], "remote": true}`, or a search's saved params) set - the dimensions, params winning per dimension, narrowed to the ones the - reader declares in `READS.filters`. A search's params go whole into + """The `Ask` one target resolves to. `reach` (from the agents, see + `app/jobs/reach.py`) and then the subscription's own `params` (a board's + override, e.g. `{"countries": ["DK"], "remote": true}`, or a search's + saved params) set the dimensions, params winning per dimension, narrowed + to the ones the reader declares in `READS.filters`. `{"reach": "all"}` in + the params drops the reach, and so does a reach of more than + `MAX_SERIES` countries on a reader whose country filter is `one`. A + `remote` of False asks nothing. A search's params go whole into `Ask.params`; a reader with `READS.key` gets the resolved key under `Ask.params["key"]` (see `resolve_key`, which raises when it is missing). A module without `READS` gets `Ask()`.""" @@ -304,7 +312,12 @@ def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None = return Ask() reads: Reads = module.READS params = dict(subscription_params or {}) - wanted = _dimensions(module, reach or {}) | _dimensions(module, params) + reached = {} if params.get("reach") == "all" else _dimensions(module, reach or {}) + if reads.filters.country == "one" and len(reached.get("country", ())) > MAX_SERIES: + reached = {} + wanted = reached | _dimensions(module, params) + if wanted.get("remote") is False: + del wanted["remote"] fields: dict[str, Any] = {_ASK_FIELD[name]: value for name, value in wanted.items() if name in reads.filters.declared()} extra: dict[str, Any] = dict(params) if reads.scope == "search" else {} @@ -320,29 +333,61 @@ def filtered(ask: Ask) -> bool: def series(module: ModuleType, ask: Ask, facets: Mapping[str, Mapping[str, str]] | None = None) -> list[Ask]: - """The series of requests `ask` takes on this reader: one per value of a - dimension declared `one` (the product when two are), one with every - value for `many`, and host ids from `facets` for `ids` (values `facets` - does not know are dropped; an empty list when none is left, since the - host then has nothing in reach). `[ask]` for a module without `READS`.""" + """The series of requests `ask` takes on this reader. Remote with a + country or a place is two parts, "in these places, or remote": the + places without remote, and remote alone. Each part is one series per + value of a dimension declared `one` (the product when two are), one + with every value for `many`, and host ids from `facets` for `ids`: + values `facets` does not know are dropped, and a part with none left is + no series, since the host has nothing of it; a dimension `facets` does + not state at all is read unfiltered, and so is remote alone, while the + remote part beside places is then left out. `[ask]` for a module + without `READS`.""" if not declared(module): return [ask] + if ask.remote and (ask.countries or ask.places): + return [*_series(module, ask.model_copy(update={"remote": None}), facets), + *_series(module, ask.model_copy(update={"countries": (), "places": ()}), + facets, beside=True)] + return _series(module, ask, facets) + + +def _series(module: ModuleType, ask: Ask, facets: Mapping[str, Mapping[str, str]] | None, + beside: bool = False) -> list[Ask]: + """`series` for an ask that is places or remote, not both. `beside` marks + the remote part of an ask that also has places.""" reach = module.READS.filters + known_facets = facets or {} countries, places = ask.countries, ask.places for dimension in ("country", "place"): values = countries if dimension == "country" else places if getattr(reach, dimension) != "ids" or not values: continue - known = (facets or {}).get(dimension) or {} - mapped = tuple(known[v] for v in values if v in known) - if not mapped: - log.info("readers: %s knows none of %s %s", module.NAME, dimension, - ", ".join(values)) - return [] + if dimension not in known_facets: + log.info("readers: %s states no %s facet; read unfiltered", module.NAME, dimension) + mapped: tuple[str, ...] = () + else: + known = known_facets.get(dimension) or {} + mapped = tuple(dict.fromkeys(known[v] for v in values if v in known)) + if not mapped: + log.info("readers: %s knows none of %s %s", module.NAME, dimension, + ", ".join(values)) + return [] if dimension == "country": countries = mapped else: places = mapped + if reach.remote == "ids" and ask.remote: + if "remote" not in known_facets: + if beside: + log.info("readers: %s states no remote facet; remote left out", module.NAME) + return [] + ask = ask.model_copy(update={"remote": None}) + else: + ids = tuple(dict.fromkeys((known_facets.get("remote") or {}).values())) + if not ids: + return [] + ask = ask.model_copy(update={"remote_ids": ids}) by_country = [(c,) for c in countries] if reach.country == "one" and countries \ else [countries] by_place = [(p,) for p in places] if reach.place == "one" and places else [places] @@ -505,12 +550,13 @@ def _load(state: Mapping[str, str], key: str) -> dict[str, Any]: def facets_for(module: ModuleType, slug: str, get: Get, ask: Ask, state: MutableMapping[str, str] | None = None, key: str = "") -> dict[str, Any]: - """The reader's `facets` when `ask` sets a dimension it declares `ids`, + """The reader's `facets` when `ask` sets a dimension it declares `ids` + (a country, a place, or remote), kept in `state[key]` for `FACETS_TTL`; `{}` otherwise.""" if not declared(module) or not hasattr(module, "facets"): return {} ids = [name for name, reach in module.READS.filters.declared().items() if reach == "ids"] - if not any(getattr(ask, _ASK_FIELD[name]) for name in ids if name in ("country", "place")): + if not any(getattr(ask, _ASK_FIELD[name]) for name in ids if name != "query"): return {} now = datetime.now(UTC) if state is not None and key: diff --git a/plugins/readers/eightfold.py b/plugins/readers/eightfold.py index fe2364f..075a7c9 100644 --- a/plugins/readers/eightfold.py +++ b/plugins/readers/eightfold.py @@ -15,14 +15,17 @@ links. `/api/apply/v2/jobs` answers 403 "Not authorized for PCSX" on the sites tried, so it is not used. The search ignores `num` and answers ten roles a page, so a board costs one -request per ten roles, and `ats.MAX_PAGES` (100) stops a read at 1,000 roles: a -larger board (Infineon's 1,292, Microsoft's 2,319) is read incomplete every pull, -and an incomplete read never closes the roles it did not list. The rows are -sorted newest first (`sort_by=timestamp`) so that what the cap keeps is the -newest thousand and the pages do not shuffle between two requests, unlike the -default order, which puts hot roles first. The search does take a `location` -free text (Ericsson's "Sweden" found 157 of 637), which could bring a read -under the cap, but a target names no filter and none is built here. +request per ten roles, and `ats.MAX_PAGES` (100) stops a read at 1,000 roles. +The rows are sorted newest first (`sort_by=timestamp`) so that what the cap +keeps is the newest thousand and the pages do not shuffle between two +requests, unlike the default order, which puts hot roles first. + +A country is filtered on the server by `location=`, one a +request (`Filters(country="one")`): Ericsson's "Sweden" answered 154 of 618 on +2026-09-30, a role at several places is kept when one of them matches, and +the review measured Infineon at 163 of 1,288 and Microsoft at 60 of 2,320 for +Denmark and Sweden, which brings both under the cap. A code with no English +name in `_countries` is read unfiltered. Discovery names Eightfold from an `eightfold.ai` link, and the host and the domain from one that carries `domain=`. @@ -35,15 +38,17 @@ import json import re from collections.abc import Callable from typing import Any +from urllib.parse import quote from desk.sources import strip_html -from ._contract import Ask, Fields, Place, Reads, Row +from ._contract import Ask, Fields, Filters, Place, Reads, Row +from ._countries import name_of from ._shared import Get, Page, is_web_url, only, parse_dt READS = Reads( scope="board", listing="paged", page_size=10, text="detail", complete="stated", - places="many", + places="many", filters=Filters(country="one"), fields=Fields(posted="listed", remote="stated"), robots="read: robots.txt allows /careers (checked 2026-09-29)", ) @@ -95,12 +100,14 @@ def careers_url(slug: str) -> str: def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page: """One page of `/api/pcsx/search` -> `{"status": 200, "data": {"positions": - [...], "count": n}}`; the next starts at the offset after this page's - positions, and an empty page ends the read.""" + [...], "count": n}}`, in the one country `ask` names; the next starts at + the offset after this page's positions, and an empty page ends the read.""" host, domain = parts(slug) start = int(cursor or 0) + country = name_of(ask.countries[0]) if ask.countries else "" + where = f"&location={quote(country)}" if country else "" answer = json.loads(get(f"https://{host}/api/pcsx/search?domain={domain}" - f"&sort_by=timestamp&start={start}")) + f"&sort_by=timestamp&start={start}{where}")) data = answer.get("data") if isinstance(answer, dict) else None positions = data.get("positions") if isinstance(data, dict) else None if not isinstance(positions, list): diff --git a/plugins/readers/oracle.py b/plugins/readers/oracle.py index e75057a..c02826f 100644 --- a/plugins/readers/oracle.py +++ b/plugins/readers/oracle.py @@ -8,6 +8,18 @@ number is `CX_1001` or `CX_1` on the tenants read so far, and the site's own name (`SinchCareer`) is taken in its place. Written from live boards on 2026-09-29 (Milestone Systems, Nokia, Danske Bank, Sinch); jobhunt-hq did not read Oracle. + +Countries are filtered on the server by `selectedLocationsFacet`, the pod's +geography ids (`Filters(country="ids")`), which `facets` reads off +`locationsFacet`; several ids go in one request, joined with `;`, the way the +site's own search sends a multiple choice. That list holds only the pod's ten +biggest countries (fa-evmr's, 2026-09-30), so when it is full a country +missing from it may still have roles: `facets` then maps every other country +to `*`, and a series asking for one is read without a country filter. An id +the pod does not know answers the whole board (review). Remote is +`selectedWorkplaceTypesFacet=ORA_REMOTE` (`remote="one"`); two facets are an +intersection (Finland and remote answered 0 on 2026-09-30), which is why the +runner asks for remote in a series of its own. """ from __future__ import annotations @@ -21,7 +33,8 @@ from urllib.parse import quote from desk.sources import strip_html -from ._contract import Ask, Fields, Place, Reads +from ._contract import Ask, Fields, Filters, Place, Reads +from ._countries import NAMES, code_of from ._shared import Get, Page, Row, only, parse_dt NAME = "oracle" @@ -41,7 +54,7 @@ PATTERNS = ( READS = Reads( scope="board", listing="paged", page_size=200, text="detail", teaser=True, - complete="stated", places="many", + complete="stated", places="many", filters=Filters(country="ids", remote="one"), fields=Fields(posted="trusted", remote="stated", country="stated"), robots=("read: robots.txt allows /hcmUI/CandidateExperience/en/sites//jobs " "(checked 2026-09-29)"), @@ -103,7 +116,7 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page: data = json.loads(get( f"{_api(slug)}/recruitingCEJobRequisitions?onlyData=true" f"&expand=requisitionList.secondaryLocations" - f"&finder=findReqs;siteNumber={site},limit={PAGE_SIZE},offset={offset}")) + f"&finder=findReqs;siteNumber={site},{_selected(ask)}limit={PAGE_SIZE},offset={offset}")) search = (data.get("items") or [{}])[0] roles = search.get("requisitionList") or [] # `_paged` counts `id`, which Oracle spells `Id`. @@ -111,6 +124,42 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page: return search, results, search.get("TotalJobsCount"), offset + len(roles) if roles else None +#: How many countries `locationsFacet` lists at most; a list this long may +#: leave some out. +FACET_LIMIT = 10 + + +def _selected(ask: Ask) -> str: + """The finder's facet parameters for `ask`, each ending in a comma: the + location ids (none when one is `*`, a country the pod did not list) and + remote.""" + out = "" + ids = [i for i in ask.countries if re.fullmatch(r"\d+", i)] + if ids and "*" not in ask.countries: + out += f"selectedLocationsFacet={'%3B'.join(ids)}," + if ask.remote: + out += "selectedWorkplaceTypesFacet=ORA_REMOTE," + return out + + +def facets(get: Get, slug: str) -> dict[str, dict[str, str]]: + """The pod's geography id of each country `locationsFacet` names, read + with the facets and no roles. When the list is `FACET_LIMIT` long, every + country it leaves out maps to `*`: its roles may be there, and a read + for it goes unfiltered.""" + site = _parts(slug)[1] + data = json.loads(get( + f"{_api(slug)}/recruitingCEJobRequisitions?onlyData=true" + f"&finder=findReqs;siteNumber={site},facetsList=LOCATIONS,limit=1,offset=0")) + listed = (data.get("items") or [{}])[0].get("locationsFacet") or [] + country = {code: str(entry["Id"]) for entry in listed + if isinstance(entry, dict) and entry.get("Id") + and (code := code_of(str(entry.get("Name") or "")))} + if len(listed) >= FACET_LIMIT: + country = dict.fromkeys(NAMES, "*") | country + return {"country": country} + + def parse(body: bytes, slug: str) -> list[Row]: return rows((json.loads(body).get("items") or [{}])[0], slug) diff --git a/plugins/readers/phenom.py b/plugins/readers/phenom.py index aaeb916..caa623c 100644 --- a/plugins/readers/phenom.py +++ b/plugins/readers/phenom.py @@ -9,6 +9,14 @@ answered without a cookie, an `Origin` header, a `pageId` or the site's `country`, both read off the slug's path) and the paging. Nothing is taken from the page's script. Written from live sites on 2026-09-29 (King, Just Eat Takeaway, Philips); jobhunt-hq did not read Phenom. + +Countries are filtered on the server as a list of English names in +`selected_fields.country`, all in one request (`Filters(country="many")`); the +list is a union, and a role at several places is kept when one of them +matches. Philips answered 111 of 873 for Netherlands and Germany on +2026-09-30. There is no remote field to filter on. A code with no English +name in `_countries` is left out of the list, and a list left empty reads the +whole board. """ from __future__ import annotations @@ -21,7 +29,8 @@ from typing import Any from desk.sources import strip_html -from ._contract import Ask, Fields, Place, Reads +from ._contract import Ask, Fields, Filters, Place, Reads +from ._countries import name_of from ._shared import Get, Page, Row, only, parse_dt NAME = "phenom" @@ -37,7 +46,8 @@ PATTERNS = ( READS = Reads( scope="board", listing="paged", page_size=100, text="detail", teaser=True, - complete="stated", places="many", fields=Fields(posted="trusted", remote="stated"), + complete="stated", places="many", filters=Filters(country="many"), + fields=Fields(posted="trusted", remote="stated"), robots="read: robots.txt allows ///search-results (checked 2026-09-29)", ) @@ -77,19 +87,21 @@ def careers_url(slug: str) -> str: return f"{_root(slug)}/search-results" -def _body(slug: str, offset: int) -> dict[str, Any]: +def _body(slug: str, offset: int, countries: tuple[str, ...] = ()) -> dict[str, Any]: """What the site's search page sends for a listing of every role, newest - first, less the parts that only tell the page what to draw. `lang` is - `en_global` for `/global/en` and `en_us` for `/us/en`, the site's own - spelling on the three sites read.""" + first, less the parts that only tell the page what to draw, in the + `countries` (ISO codes) given. `lang` is `en_global` for `/global/en` and + `en_us` for `/us/en`, the site's own spelling on the three sites read.""" _host, country, lang = _parts(slug) + names = [name for code in countries if (name := name_of(code))] return { "lang": f"{lang}_{country}", "deviceType": "desktop", "country": country, "pageName": "search-results", "ddoKey": "refineSearch", "sortBy": "", "subsearch": "", "from": offset, "jobs": True, "counts": True, "all_fetch": False, "pageType": "landingPage", "size": PAGE_SIZE, "clearAll": False, "jdsource": "facets", "isSliderEnable": False, "siteType": "external", - "location": "", "keywords": "", "global": True, "selected_fields": {}, + "location": "", "keywords": "", "global": True, + "selected_fields": {"country": names} if names else {}, "sort": {"order": "desc", "field": "postedDate"}, "locationData": {}, } @@ -106,7 +118,7 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page: offset = int(cursor or 0) host = _parts(slug)[0] data = json.loads(get(f"https://{host}/widgets", method="POST", - payload=_body(slug, offset))) + payload=_body(slug, offset, ask.countries))) search = data.get("refineSearch") if not isinstance(search, dict) or search.get("status", 200) != 200: raise ValueError("not a Phenom search answer: no refineSearch") diff --git a/plugins/readers/smartrecruiters.py b/plugins/readers/smartrecruiters.py index 021cc0f..b58907f 100644 --- a/plugins/readers/smartrecruiters.py +++ b/plugins/readers/smartrecruiters.py @@ -3,16 +3,22 @@ with a stated total. The identifier is case-sensitive (`BoschGroup`). Written from one real board on 2026-09-23; jobhunt-hq did not read SmartRecruiters. + +A country is filtered on the server, `?country=dk`, one a request +(`Filters(country="one")`): Bosch's DE, SE and DK summed to what the app held +(review, 2026-09-29), and DK answered 2 of 4,830 on 2026-09-30. The API +ignores a remote parameter, so remote is not filtered. """ from __future__ import annotations import json import logging +import re from collections.abc import Callable from typing import Any -from ._contract import Ask, Fields, Place, Reads +from ._contract import Ask, Fields, Filters, Place, Reads from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok log = logging.getLogger(__name__) @@ -27,7 +33,7 @@ PATTERNS = ( #: An unknown identifier answers an empty list, which is `empty="suspect"`. READS = Reads( scope="board", listing="paged", page_size=100, text="none", complete="stated", - places="one", empty="suspect", + places="one", empty="suspect", filters=Filters(country="one"), fields=Fields(posted="trusted", remote="stated", country="stated", seniority="stated", employment="stated"), robots="api: no robots.txt check needed (public board API), 2026-09-30", @@ -73,9 +79,13 @@ def probe_ok(body: bytes) -> bool: def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page: + """One page, in the one country `ask` names, which the API takes in lower + case.""" offset = int(cursor or 0) + country = ask.countries[0].lower() if ask.countries else "" + where = f"&country={country}" if re.fullmatch(r"[a-z]{2}", country) else "" data = json.loads(get(f"https://{_API}/v1/companies/{slug}/postings" - f"?offset={offset}&limit={PAGE_SIZE}")) + f"?offset={offset}&limit={PAGE_SIZE}{where}")) content = data.get("content") or [] return data, content, data.get("totalFound"), offset + len(content) if content else None diff --git a/plugins/readers/successfactors.py b/plugins/readers/successfactors.py index c9feb57..0124020 100644 --- a/plugins/readers/successfactors.py +++ b/plugins/readers/successfactors.py @@ -30,8 +30,13 @@ module reads, and neither is a translation of a listed role. A path prefix names a brand of a shared site. `//search/` lists only that brand on the server (Universal Robots 30 of Teradyne's 486, MiR 1), so the read is as small as the company; `rows` still keeps only the roles -under `//job/`, in case a site does not scope. There is no server-side -country filter in this version. +under `//job/`, in case a site does not scope. + +A country is filtered on the server by the search's own facet, +`optionsFacetsDD_country=DK`, one country a request (`Filters(country="one")`); +it worked on the four tenants tried in the review and on Vestas (64 of 668, +2026-09-30). A role at several places is listed when one of them is in the +country. There is no remote facet. A role's own page carries microdata (`itemtype` schema.org JobPosting) with `datePosted` and the address; `detail` reads the description, the date and the @@ -52,7 +57,8 @@ from urllib.parse import urljoin, urlsplit from desk.sources import strip_html -from ._contract import Ask, Fields, Place, Reads +from ._contract import Ask, Fields, Filters, Place, Reads +from ._countries import NAMES from ._shared import Get, Page, Row, is_web_url, only, parse_dt NAME = "successfactors" @@ -79,6 +85,7 @@ KEEP_CASE = True READS = Reads( scope="board", listing="paged", text="detail", complete="stated", places="many", + filters=Filters(country="one"), fields=Fields(posted="trusted", country="stated"), robots="read: robots.txt allows /search/ and /job/ (checked 2026-09-29)", ) @@ -122,52 +129,10 @@ _COUNTRY_LABELS = ("country", "land") _MONTHS = {name: number for number, name in enumerate( ("jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"), 1)} -#: ISO 3166-1 alpha-2 to the English name, which is what `country_of` reads: -#: a bare "DE" or "SE" in a location line is taken for Delaware or nothing. A -#: code that is not here stays as the site wrote it. -_COUNTRIES = dict(entry.strip().split(" ", 1) for entry in """ -AD Andorra | AE United Arab Emirates | AF Afghanistan | AG Antigua and Barbuda | AI -Anguilla | AL Albania | AM Armenia | AO Angola | AR Argentina | AS American Samoa | AT -Austria | AU Australia | AW Aruba | AX Åland Islands | AZ Azerbaijan | BA Bosnia and -Herzegovina | BB Barbados | BD Bangladesh | BE Belgium | BF Burkina Faso | BG Bulgaria -| BH Bahrain | BI Burundi | BJ Benin | BM Bermuda | BN Brunei | BO Bolivia | BQ -Caribbean Netherlands | BR Brazil | BS Bahamas | BT Bhutan | BW Botswana | BY Belarus -| BZ Belize | CA Canada | CD Democratic Republic of the Congo | CF Central African -Republic | CG Republic of the Congo | CH Switzerland | CI Côte d'Ivoire | CK Cook -Islands | CL Chile | CM Cameroon | CN China | CO Colombia | CR Costa Rica | CU Cuba | -CV Cabo Verde | CW Curaçao | CY Cyprus | CZ Czechia | DE Germany | DJ Djibouti | DK -Denmark | DM Dominica | DO Dominican Republic | DZ Algeria | EC Ecuador | EE Estonia | -EG Egypt | ER Eritrea | ES Spain | ET Ethiopia | FI Finland | FJ Fiji | FK Falkland -Islands | FM Micronesia | FO Faroe Islands | FR France | GA Gabon | GB United Kingdom -| GD Grenada | GE Georgia | GF French Guiana | GG Guernsey | GH Ghana | GI Gibraltar | -GL Greenland | GM Gambia | GN Guinea | GP Guadeloupe | GQ Equatorial Guinea | GR -Greece | GT Guatemala | GU Guam | GW Guinea-Bissau | GY Guyana | HK Hong Kong | HN -Honduras | HR Croatia | HT Haiti | HU Hungary | ID Indonesia | IE Ireland | IL Israel -| IM Isle of Man | IN India | IQ Iraq | IR Iran | IS Iceland | IT Italy | JE Jersey | -JM Jamaica | JO Jordan | JP Japan | KE Kenya | KG Kyrgyzstan | KH Cambodia | KI -Kiribati | KM Comoros | KN Saint Kitts and Nevis | KP North Korea | KR South Korea | -KW Kuwait | KY Cayman Islands | KZ Kazakhstan | LA Laos | LB Lebanon | LC Saint Lucia -| LI Liechtenstein | LK Sri Lanka | LR Liberia | LS Lesotho | LT Lithuania | LU -Luxembourg | LV Latvia | LY Libya | MA Morocco | MC Monaco | MD Moldova | ME -Montenegro | MG Madagascar | MH Marshall Islands | MK North Macedonia | ML Mali | MM -Myanmar | MN Mongolia | MO Macao | MP Northern Mariana Islands | MQ Martinique | MR -Mauritania | MS Montserrat | MT Malta | MU Mauritius | MV Maldives | MW Malawi | MX -Mexico | MY Malaysia | MZ Mozambique | NA Namibia | NC New Caledonia | NE Niger | NG -Nigeria | NI Nicaragua | NL Netherlands | NO Norway | NP Nepal | NR Nauru | NU Niue | -NZ New Zealand | OM Oman | PA Panama | PE Peru | PF French Polynesia | PG Papua New -Guinea | PH Philippines | PK Pakistan | PL Poland | PR Puerto Rico | PS Palestine | PT -Portugal | PW Palau | PY Paraguay | QA Qatar | RE Réunion | RO Romania | RS Serbia | -RU Russia | RW Rwanda | SA Saudi Arabia | SB Solomon Islands | SC Seychelles | SD -Sudan | SE Sweden | SG Singapore | SI Slovenia | SK Slovakia | SL Sierra Leone | SM -San Marino | SN Senegal | SO Somalia | SR Suriname | SS South Sudan | ST São Tomé and -Príncipe | SV El Salvador | SY Syria | SZ Eswatini | TC Turks and Caicos Islands | TD -Chad | TG Togo | TH Thailand | TJ Tajikistan | TL Timor-Leste | TM Turkmenistan | TN -Tunisia | TO Tonga | TR Turkey | TT Trinidad and Tobago | TV Tuvalu | TW Taiwan | TZ -Tanzania | UA Ukraine | UG Uganda | US United States | UY Uruguay | UZ Uzbekistan | VA -Vatican City | VC Saint Vincent and the Grenadines | VE Venezuela | VG British Virgin -Islands | VI U.S. Virgin Islands | VN Vietnam | VU Vanuatu | WS Samoa | XK Kosovo | YE -Yemen | ZA South Africa | ZM Zambia | ZW Zimbabwe -""".replace("\n", " ").split(" | ")) +#: ISO 3166-1 alpha-2 to the English name (`_countries.NAMES`): a bare "DE" +#: or "SE" in a location line is taken for Delaware or nothing. A code that +#: is not there stays as the site wrote it. +_COUNTRIES = NAMES def _parts(slug: str) -> tuple[str, str]: @@ -190,9 +155,12 @@ def careers_url(slug: str) -> str: return f"https://{host}{prefix}/search/" -def _search_url(slug: str, start: int) -> str: +def _search_url(slug: str, start: int, country: str = "") -> str: + """The search page from `start`, in one country when `country` (an ISO + code) is given.""" host, prefix = _parts(slug) - return f"https://{host}{prefix}/search/?q=&locationsearch=&startrow={start}" + facet = f"&optionsFacetsDD_country={country}" if re.fullmatch(r"[A-Z]{2}", country) else "" + return f"https://{host}{prefix}/search/?q=&locationsearch={facet}&startrow={start}" # ------------------------------------------------------------------ listing @@ -296,12 +264,13 @@ def _entries(text: str) -> list[dict[str, Any]]: def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page: - """One search page. The cursor is `(start row, first role of the page + """One search page, in the one country `ask` names. The cursor is `(start row, first role of the page before)`: the next start row is what this page returned, and a page that starts with the same role as the last one is the site ignoring `startrow`, which ends the read (incomplete) instead of looping.""" start, previous = cursor or (0, None) - text = get(_search_url(slug, start)).decode("utf-8", "replace") + country = ask.countries[0] if ask.countries else "" + text = get(_search_url(slug, start, country)).decode("utf-8", "replace") if not _SEARCH_PAGE.search(text): raise ValueError("not a SuccessFactors search page") found = _entries(text) diff --git a/plugins/readers/workday.py b/plugins/readers/workday.py index b6f8738..fb9b25e 100644 --- a/plugins/readers/workday.py +++ b/plugins/readers/workday.py @@ -12,6 +12,17 @@ keeps the case the URL wrote (`KEEP_CASE`). Written from live career sites on 2026-09-29 (`docs/plan-hosts/workday.md`). The listing carries no description, so `detail` reads each role's own JSON. + +Countries and remote are filtered on the server by the tenant's own facets +(`Filters(country="ids", remote="ids")`): `appliedFacets` takes +`{: [, ...]}`, ids within one facet are a union, and two +facets are an intersection, which is why the runner asks for remote in a +series of its own. Each tenant names its facets its own way (LEGO's +`Location` lists countries, others nest `locationCountry` under +`locationMainGroup`), so `facets` reads them off the first page and picks the +one whose values are countries. A host id is `=`. LEGO +answered 97 of 455 for Denmark on 2026-09-30; a role at several places is +kept when one of them is in the country. """ from __future__ import annotations @@ -26,7 +37,8 @@ from typing import Any from desk.sources import strip_html -from ._contract import Ask, Fields, Place, Reads +from ._contract import Ask, Fields, Filters, Place, Reads +from ._countries import code_of from ._shared import Get, Page, Row, only, parse_dt log = logging.getLogger(__name__) @@ -37,7 +49,7 @@ KEEP_CASE = True READS = Reads( scope="board", listing="paged", page_size=20, ceiling=2000, text="detail", - complete="stated", places="many", + complete="stated", places="many", filters=Filters(country="ids", remote="ids"), fields=Fields(posted="trusted", remote="stated", country="stated", employment="stated"), robots="read: robots.txt allows // (checked 2026-09-29)", ) @@ -104,7 +116,8 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page: offset, total = cursor or (0, None) data = json.loads(get( f"https://{_host(slug)}/wday/cxs/{tenant}/{site}/jobs", method="POST", - payload={"appliedFacets": {}, "limit": PAGE_SIZE, "offset": offset, "searchText": ""})) + payload={"appliedFacets": _applied(ask), "limit": PAGE_SIZE, "offset": offset, + "searchText": ""})) if cursor is None: stated = data.get("total") total = stated if isinstance(stated, int) and not isinstance(stated, bool) else None @@ -118,6 +131,69 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page: return data, results, total, (following, total) if more else None +def _applied(ask: Ask) -> dict[str, list[str]]: + """`appliedFacets` for the host ids `ask` carries, `=` + each, grouped by facet.""" + out: dict[str, list[str]] = {} + for value in (*ask.countries, *ask.remote_ids): + facet, sep, value_id = value.partition("=") + if sep and facet and value_id: + out.setdefault(facet, []).append(value_id) + return out + + +#: A facet whose values are countries: at least this share of them name one. +_COUNTRY_SHARE = 0.8 + + +def _facet_lists(facets: Any) -> list[tuple[str, list[dict[str, Any]]]]: + """`(facetParameter, values)` of every facet of an answer, the facets nested + in a group (`locationMainGroup` holds `locationCountry` on some tenants) + included.""" + out: list[tuple[str, list[dict[str, Any]]]] = [] + for facet in facets if isinstance(facets, list) else []: + if not isinstance(facet, dict): + continue + values = [v for v in facet.get("values") or [] if isinstance(v, dict)] + nested = [v for v in values if "values" in v and "facetParameter" in v] + out += _facet_lists(nested) + plain = [v for v in values if v not in nested] + if plain and str(facet.get("facetParameter") or ""): + out.append((str(facet["facetParameter"]), plain)) + return out + + +def facets(get: Get, slug: str) -> dict[str, dict[str, str]]: + """The tenant's host ids, read off one page of one role: `country` maps + an ISO code to `=` in the facet whose values are + countries (most of them, `_COUNTRY_SHARE`; the one with most countries + when two are), and `remote` maps each facet value that `_REMOTE` reads as + remote ("Remote", "Fully Flexible"), in whatever facet, to its id. A + tenant with neither states no such dimension, and the runner then does + not filter on it.""" + tenant, _dc, site = _parts(slug) # type: ignore[misc] + data = json.loads(get( + f"https://{_host(slug)}/wday/cxs/{tenant}/{site}/jobs", method="POST", + payload={"appliedFacets": {}, "limit": 1, "offset": 0, "searchText": ""})) + country: dict[str, str] = {} + remote: dict[str, str] = {} + for name, values in _facet_lists(data.get("facets")): + found = {code: f"{name}={v['id']}" for v in values + if v.get("id") and (code := code_of(str(v.get("descriptor") or "")))} + if len(found) >= _COUNTRY_SHARE * len(values) and len(found) > len(country): + country = found + for v in values: + word = " ".join(str(v.get("descriptor") or "").split()) + if v.get("id") and _remote(word) == "remote": + remote[word.lower()] = f"{name}={v['id']}" + out: dict[str, dict[str, str]] = {} + if country: + out["country"] = country + if remote: + out["remote"] = remote + return out + + def _posting_id(posting: dict[str, Any], position: int) -> str: return str(posting.get("externalPath") or "".join(posting.get("bulletFields") or []) or f"#{position}") diff --git a/tests/fixtures/ats/oracle-facets.json b/tests/fixtures/ats/oracle-facets.json new file mode 100644 index 0000000..04532dc --- /dev/null +++ b/tests/fixtures/ats/oracle-facets.json @@ -0,0 +1,87 @@ +{ + "items": [ + { + "SearchId": 1, + "Facets": "LOCATIONS;WORKPLACE_TYPES", + "Offset": 0, + "Limit": 1, + "TotalJobsCount": 550, + "SiteNumber": "CX_1", + "locationsFacet": [ + { + "Id": 300000000480126, + "Name": "United States", + "TotalCount": 215 + }, + { + "Id": 300000000471745, + "Name": "India", + "TotalCount": 123 + }, + { + "Id": 300000000471544, + "Name": "Canada", + "TotalCount": 79 + }, + { + "Id": 300000000471982, + "Name": "Portugal", + "TotalCount": 38 + }, + { + "Id": 300000000471904, + "Name": "Malaysia", + "TotalCount": 31 + }, + { + "Id": 300000000471640, + "Name": "Finland", + "TotalCount": 23 + }, + { + "Id": 300000000471967, + "Name": "Poland", + "TotalCount": 21 + }, + { + "Id": 300000000480081, + "Name": "Thailand", + "TotalCount": 17 + }, + { + "Id": 300000000471760, + "Name": "Italy", + "TotalCount": 12 + }, + { + "Id": 300000000471601, + "Name": "Germany", + "TotalCount": 10 + } + ], + "workplaceTypesFacet": [ + { + "Id": "ORA_HYBRID", + "Name": "Hybrid", + "TotalCount": 317 + }, + { + "Id": "ORA_ON_SITE", + "Name": "Onsite", + "TotalCount": 230 + }, + { + "Id": "ORA_REMOTE", + "Name": "Remote", + "TotalCount": 3 + } + ], + "categoriesFacet": [], + "requisitionList": [] + } + ], + "count": 1, + "hasMore": false, + "limit": 25, + "offset": 0 +} \ No newline at end of file diff --git a/tests/fixtures/ats/workday-facets.json b/tests/fixtures/ats/workday-facets.json new file mode 100644 index 0000000..91e188b --- /dev/null +++ b/tests/fixtures/ats/workday-facets.json @@ -0,0 +1,109 @@ +{ + "total": 455, + "jobPostings": [], + "facets": [ + { + "facetParameter": "Location", + "descriptor": "Location", + "values": [ + { + "descriptor": "United States of America", + "id": "bc33aa3152ec42d4995f4791a106ed09", + "count": 173 + }, + { + "descriptor": "Denmark", + "id": "49ab063f422741e2aef271de00efeac8", + "count": 97 + }, + { + "descriptor": "China", + "id": "6cb77610a8a543aea2d6bc10457e35d4", + "count": 30 + }, + { + "descriptor": "United Kingdom", + "id": "29247e57dbaf46fb855b224e03170bc7", + "count": 29 + }, + { + "descriptor": "Czechia", + "id": "fc078443155c4ad294201ecf5a61a499", + "count": 25 + }, + { + "descriptor": "Hungary", + "id": "9db257f5937e4421b2fac64eec6832f8", + "count": 21 + }, + { + "descriptor": "Germany", + "id": "dcc5b7608d8644b3a93716604e78e995", + "count": 18 + }, + { + "descriptor": "Vietnam", + "id": "db69e8c8446c11de98360015c5e6daf6", + "count": 17 + } + ] + }, + { + "facetParameter": "timeType", + "descriptor": "Time Type", + "values": [ + { + "descriptor": "Full time", + "id": "07564020a44510063cb1bcb298e20001", + "count": 270 + }, + { + "descriptor": "Part time", + "id": "07564020a44510063cb1bcb298e20000", + "count": 162 + } + ] + }, + { + "facetParameter": "locationMainGroup", + "values": [ + { + "facetParameter": "locationRegionStateProvince", + "descriptor": "Region/State/Province", + "values": [ + { + "descriptor": "Alabama", + "id": "31475924e5494080a8a458bf4fa293ed", + "count": 2 + }, + { + "descriptor": "Alberta", + "id": "588eb24f950e4c5b93d5ad3f37aebad2", + "count": 3 + }, + { + "descriptor": "Arizona", + "id": "c7b20b0d4bc04711a00900569e9afabd", + "count": 4 + }, + { + "descriptor": "Arkansas", + "id": "cea6c5355e1b4983b5fd0640310385b5", + "count": 1 + }, + { + "descriptor": "Bavaria", + "id": "bd89fff5ffa6440a9abeaf2c5c1dc37c", + "count": 1 + }, + { + "descriptor": "Beijing", + "id": "4b194fc4ae3b4cbe95a1b379454e1775", + "count": 1 + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/test_ats_eightfold.py b/tests/test_ats_eightfold.py index ed5e5f0..ad70888 100644 --- a/tests/test_ats_eightfold.py +++ b/tests/test_ats_eightfold.py @@ -204,10 +204,10 @@ def test_discovery_reads_the_host_and_domain_from_a_link_that_has_both() -> None assert "eightfold" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS} -def test_it_declares_a_paged_read_of_ten_with_a_total_and_no_filter_yet() -> None: +def test_it_declares_a_paged_read_of_ten_with_a_total_and_one_country_a_request() -> None: reads = eightfold.READS assert (reads.listing, reads.page_size, reads.text, reads.complete, reads.places) == ( "paged", 10, "detail", "stated", "many") - assert reads.filters.declared() == {} and reads.fields.posted == "listed" + assert reads.filters.declared() == {"country": "one"} and reads.fields.posted == "listed" assert reads.robots.startswith("read: robots.txt allows /careers") assert not hasattr(eightfold, "robots_page") diff --git a/tests/test_reach.py b/tests/test_reach.py new file mode 100644 index 0000000..2b99a09 --- /dev/null +++ b/tests/test_reach.py @@ -0,0 +1,212 @@ +"""The reach of a pull (`app.jobs.reach`): from the enabled agents' gates and +favourites to each board's `reach`, and through the pull's batches to the +runner's `Ask`. No network: the runner's listing is stubbed.""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import ClassVar + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from pipeline import Tally # noqa: E402 +from app.jobs import pull, reach # noqa: E402 +from app.models import Gate # noqa: E402 +from app.storage import Store # noqa: E402 +from plugins.readers import Ask, _runner # noqa: E402 + +EUROPE = ["AT", "BE", "BG", "HR", "CY", "CZ", "DK", "EE", "FI", "FR", "DE", "GR", "HU", "IE", + "IT", "LV", "LT", "LU", "MT", "NL", "PL", "PT", "RO", "SK", "SI", "ES", "SE", "IS", + "LI", "NO", "GB", "CH", "EU"] + + +class FakeConfig: + sources: ClassVar[dict[str, object]] = {} + searches: ClassVar[list[object]] = [] + + +@pytest.fixture +def store(tmp_path: Path) -> Store: + return Store(tmp_path / "roledesk.db") + + +def _wolf(store: Store, **extra: object) -> int: + """The live agent 2 of 2026-09-30: Europe required, Denmark, Sweden, remote + or a town in Skåne or Denmark.""" + gates = [Gate(field="country", values=EUROPE, mode="require"), + Gate(field="country", values=["DK", "SE"], mode="any"), + Gate(field="remote", values=["remote"], mode="any"), + Gate(field="place", values=["Malmö", "Lund", "copenhagen", "odense"], mode="any"), + Gate(field="remote", values=["hybrid"], mode="prefer", weight=0.05)] + return store.save_agent("Wolf", gates=gates, **extra).id # type: ignore[arg-type] + + +def _board(store: Store, target: str, company: str, size: int | None = 1000, + params: dict[str, object] | None = None) -> None: + store.add_subscription("ats", target, label=company, company=company, + board=target.partition(":")[0], params=params or {}) + store.add_company(company) + if size is not None: + store.state[f"ats:size:{target}"] = str(size) + + +# ------------------------------------------------------------------ gates + +def test_the_base_reach_is_the_union_of_the_any_gates_with_towns_as_countries( + store: Store) -> None: + _wolf(store) + store.save_agent("Berlin", gates=[Gate(field="place", values=["Berlin"], mode="any")]) + plan = reach.plan(store) + assert plan.base == reach.Reach(frozenset({"DK", "SE", "DE"}), remote=True) + assert plan.base.params() == {"countries": ["DE", "DK", "SE"], "remote": True} + + +@pytest.mark.parametrize("gates", [ + [], + [Gate(field="country", values=["DK"], mode="any"), + Gate(field="function", values=["engineering"], mode="any")], + [Gate(field="remote", values=["hybrid"], mode="any")], + [Gate(field="place", values=["Nowhereville"], mode="any")], +], ids=["no any gates", "a function in the group", "hybrid anywhere", "an unknown town"]) +def test_an_agent_that_can_take_a_role_from_anywhere_makes_the_reach_everywhere( + store: Store, gates: list[Gate]) -> None: + _wolf(store) + store.save_agent("Other", gates=gates) + assert reach.plan(store).base == reach.EVERYWHERE + + +def test_an_agent_switched_off_does_not_widen_the_reach(store: Store) -> None: + _wolf(store) + store.save_agent("Off", gates=[], enabled=False) + assert reach.plan(store).base.countries == {"DK", "SE"} + + +def test_with_no_agent_switched_on_every_board_is_read_whole(store: Store) -> None: + _wolf(store, enabled=False) + assert reach.plan(store).base == reach.EVERYWHERE + + +def test_the_require_reach_intersects_and_expands_europe(store: Store) -> None: + gaz = store.gazetteer() + agent = store.agent(_wolf(store)) + assert agent is not None + europe = reach.require_reach(agent, gaz) + assert {"DK", "SE", "DE", "FR", "GB", "NO", "CH"} <= europe.countries + assert "US" not in europe.countries and "EU" not in europe.countries + narrower = agent.model_copy(update={"gates": [ + *agent.gates, Gate(field="country", values=["DK", "US"], mode="require")]}) + assert reach.require_reach(narrower, gaz) == reach.Reach(frozenset({"DK"})) + plain = agent.model_copy(update={"gates": []}) + assert reach.require_reach(plain, gaz) == reach.EVERYWHERE + + +# ------------------------------------------------------------- per board + +def test_a_favourite_is_read_across_the_require_reach(store: Store) -> None: + wolf = _wolf(store) + _board(store, "smartrecruiters:mistral", "Mistral AI") + _board(store, "smartrecruiters:acme", "Acme") + store.set_affinity(wolf, "mistral-ai", 2) + plan = reach.plan(store) + subs = {s.target: s for s in store.subscriptions(source="ats")} + favourite, why = plan.why(subs["smartrecruiters:mistral"]) + assert why == "favourite" and {"DE", "FR", "DK"} <= favourite.countries + assert plan.why(subs["smartrecruiters:acme"]) == (plan.base, "agents") + + +def test_a_company_starred_by_one_agent_keeps_the_others_reach(store: Store) -> None: + wolf = _wolf(store) + store.save_agent("Anywhere", gates=[Gate(field="country", values=["US"], mode="any")]) + _board(store, "smartrecruiters:mistral", "Mistral AI") + store.set_affinity(wolf, "mistral-ai", 2) + favourite = reach.plan(store).favourites["mistral-ai"] + assert "US" in favourite.countries and "FR" in favourite.countries + + +@pytest.mark.parametrize("size, params, why", [ + (299, {}, "small"), + (None, {}, "unknown"), + (5000, {"reach": "all"}, "override"), + (5000, {"countries": ["DE"]}, "override"), + (300, {}, "agents"), +]) +def test_small_unknown_and_overridden_boards_get_no_reach( + store: Store, size: int | None, params: dict[str, object], why: str) -> None: + _wolf(store) + _board(store, "workday:acme.wd3/Jobs", "Acme", size, params) + [sub] = store.subscriptions(source="ats") + found, said = reach.plan(store).why(sub) + assert said == why + assert (found.params() is None) == (why != "agents") + + +def test_a_board_read_before_sizes_were_kept_is_sized_by_its_open_roles(store: Store) -> None: + _board(store, "workday:acme.wd3/Jobs", "Acme", size=None) + [sub] = store.subscriptions(source="ats") + store.state["ats:ok:workday:acme.wd3/Jobs"] = "run-0" + assert reach.sizes(store, [sub]) == {"workday:acme.wd3/Jobs": 0} + # A filtered board's open roles are not its size. + store.state["ats:filtered:workday:acme.wd3/Jobs"] = "1" + assert reach.sizes(store, [sub]) == {} + + +def test_describe_is_short() -> None: + assert reach.Reach(frozenset({"SE", "DK"}), True).describe() == "DK, SE, remote" + assert reach.Reach(frozenset(EUROPE[:10])).describe() == "10 countries" + assert reach.EVERYWHERE.describe() == "everywhere" + + +# ---------------------------------------------------------------- the pull + +def _row(job_id: str) -> dict[str, object]: + return {"source_id": job_id, "title": f"Engineer {job_id}", "url": f"https://x.example/{job_id}", + "places": [{"text": "Copenhagen", "remote": None}], "posted": None, + "description": "", "remote": None, "salary_text": ""} + + +def _pull(store: Store) -> list[dict[str, object]]: + batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type] + results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type] + for batch in batches] + pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type] + return batches + + +def test_the_pull_hands_each_board_its_reach_and_the_runner_narrows_it( + store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + _wolf(store) + _board(store, "smartrecruiters:BoschGroup", "Bosch") + _board(store, "smartrecruiters:tiny", "Tiny", size=12) + asked: dict[str, Ask] = {} + + def fake(module: object, slug: str, get: object, ask: Ask, **kwargs: object + ) -> _runner.Listing: + asked[slug] = ask + return _runner.Listing([_row("1")], True, ["1"], _runner.filtered(ask)) # type: ignore[list-item] + + monkeypatch.setattr(_runner, "read_listing", fake) + [batch] = _pull(store) + assert batch["reach"] == {"smartrecruiters:BoschGroup": {"countries": ["DK", "SE"], + "remote": True}} + # SmartRecruiters filters countries one a request and cannot ask for remote. + assert asked["BoschGroup"] == Ask(countries=("DK", "SE")) + assert asked["tiny"] == Ask() + assert store.state["ats:filtered:smartrecruiters:BoschGroup"] == "1" + # The small board, read whole, keeps its size; the filtered one keeps its last. + assert store.state["ats:size:smartrecruiters:tiny"] == "1" + assert store.state["ats:size:smartrecruiters:BoschGroup"] == "1000" + + +def test_a_batch_queued_before_the_reach_existed_still_runs( + store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + _board(store, "smartrecruiters:acme", "Acme") + monkeypatch.setattr(_runner, "read_listing", lambda *a, **k: _runner.Listing( + [_row("1")], True, ["1"])) # type: ignore[list-item] + result = pull.run_batch(FakeConfig(), store, # type: ignore[arg-type] + targets=["smartrecruiters:acme"], run="r") + assert result["roles"] == 1 diff --git a/tests/test_readers_filters.py b/tests/test_readers_filters.py new file mode 100644 index 0000000..a33cca1 --- /dev/null +++ b/tests/test_readers_filters.py @@ -0,0 +1,268 @@ +"""Filtering at the source: what the six hosts that filter by country (and +Workday and Oracle by remote) send for an `Ask`, their `facets`, and the +runner's series for a reach of countries and remote. Against saved answers; +no network.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any +from urllib.parse import parse_qs, unquote, urlsplit + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from plugins.readers import ( # noqa: E402 + Ask, _countries, _runner, eightfold, oracle, phenom, smartrecruiters, successfactors, + workday, +) + +FIXTURES = ROOT / "tests" / "fixtures" / "ats" + + +class Recorder: + """A `get` that answers `body` to every request and keeps what was asked.""" + + def __init__(self, body: bytes) -> None: + self.body = body + self.asked: list[tuple[str, dict[str, Any] | None]] = [] + + def __call__(self, url: str, *, method: str = "GET", payload: dict[str, Any] | None = None, + headers: dict[str, str] | None = None) -> bytes: + self.asked.append((url, payload)) + return self.body + + +def _query(url: str) -> dict[str, list[str]]: + return parse_qs(urlsplit(url).query) + + +# ------------------------------------------------------------ declarations + +def test_the_six_hosts_declare_what_their_search_filters() -> None: + declared = {module.NAME: module.READS.filters.declared() for module in + (smartrecruiters, eightfold, workday, successfactors, phenom, oracle)} + assert declared == { + "smartrecruiters": {"country": "one"}, + "eightfold": {"country": "one"}, + "workday": {"country": "ids", "remote": "ids"}, + "successfactors": {"country": "one"}, + "phenom": {"country": "many"}, + "oracle": {"country": "ids", "remote": "one"}, + } + + +def test_country_names_read_both_ways_with_the_hosts_own_spellings() -> None: + assert _countries.name_of("dk") == "Denmark" and _countries.name_of("ZZ") == "" + assert _countries.code_of("United States of America") == "US" + assert _countries.code_of("Korea, Republic of") == "KR" + assert _countries.code_of(" czech republic ") == "CZ" + assert _countries.code_of("Türkiye") == "TR" and _countries.code_of("Turkey") == "TR" + assert _countries.code_of("Atlantis") == "" + assert successfactors._COUNTRIES["SE"] == "Sweden" + + +# -------------------------------------------------------- one a request + +def test_smartrecruiters_asks_one_country_in_lower_case() -> None: + get = Recorder(b'{"content": [], "totalFound": 0}') + smartrecruiters.page(get, "BoschGroup", None, Ask(countries=("DK",))) + smartrecruiters.page(get, "BoschGroup", 100, Ask()) + assert _query(get.asked[0][0])["country"] == ["dk"] + assert "country" not in _query(get.asked[1][0]) + + +def test_eightfold_asks_one_country_by_its_english_name() -> None: + get = Recorder(b'{"status": 200, "data": {"positions": [], "count": 0}}') + eightfold.page(get, "jobs.ericsson.com/ericsson.com", None, Ask(countries=("SE",))) + eightfold.page(get, "jobs.ericsson.com/ericsson.com", None, Ask(countries=("GB",))) + eightfold.page(get, "jobs.ericsson.com/ericsson.com", None, Ask(countries=("ZZ",))) + assert _query(get.asked[0][0])["location"] == ["Sweden"] + assert _query(get.asked[1][0])["location"] == ["United Kingdom"] + # A code with no name is read unfiltered: a superset, never too little. + assert "location" not in _query(get.asked[2][0]) + + +def test_successfactors_asks_one_country_by_its_facet() -> None: + body = (FIXTURES / "successfactors-table.html").read_bytes() + get = Recorder(body) + successfactors.page(get, "careers.vestas.com", None, Ask(countries=("DK",))) + successfactors.page(get, "jobs.teradyne.com/MiR", None, Ask(countries=("DE",))) + successfactors.page(get, "careers.vestas.com", None, Ask()) + assert get.asked[0][0] == ("https://careers.vestas.com/search/?q=&locationsearch=" + "&optionsFacetsDD_country=DK&startrow=0") + assert get.asked[1][0].startswith("https://jobs.teradyne.com/MiR/search/?") + assert _query(get.asked[1][0])["optionsFacetsDD_country"] == ["DE"] + assert "optionsFacetsDD_country" not in get.asked[2][0] + + +# ------------------------------------------------------------ a list + +def test_phenom_asks_every_country_by_name_in_one_request() -> None: + get = Recorder((FIXTURES / "phenom.json").read_bytes()) + phenom.page(get, "www.careers.philips.com/global/en", None, + Ask(countries=("NL", "DE", "ZZ"))) + phenom.page(get, "www.careers.philips.com/global/en", None, Ask()) + phenom.page(get, "www.careers.philips.com/global/en", None, Ask(countries=("ZZ",))) + assert get.asked[0][1]["selected_fields"] == {"country": ["Netherlands", "Germany"]} + assert get.asked[1][1]["selected_fields"] == {} + assert get.asked[2][1]["selected_fields"] == {} + + +# ------------------------------------------------------------ host ids + +LEGO = "lego.wd103/LEGO_External" + + +def _workday_facets(**extra: Any) -> dict[str, Any]: + saved = json.loads((FIXTURES / "workday-facets.json").read_text()) + saved["facets"] += extra.get("facets", []) + return saved + + +def test_workday_facets_find_the_facet_of_countries_and_the_remote_values() -> None: + remote = {"facetParameter": "remoteType", "descriptor": "Remote Type", "values": [ + {"descriptor": "Fully Flexible", "id": "ff1", "count": 30}, + {"descriptor": "Hybrid", "id": "hy1", "count": 20}, + {"descriptor": "Remote", "id": "rm1", "count": 3}]} + get = Recorder(json.dumps(_workday_facets(facets=[remote])).encode()) + found = workday.facets(get, LEGO) + assert found["country"]["DK"] == "Location=49ab063f422741e2aef271de00efeac8" + assert found["country"]["US"] == "Location=bc33aa3152ec42d4995f4791a106ed09" + # The regions nested under `locationMainGroup` and the time types are + # not countries. + assert all(v.startswith("Location=") for v in found["country"].values()) + assert found["remote"] == {"fully flexible": "remoteType=ff1", "remote": "remoteType=rm1"} + url, payload = get.asked[0] + assert url == "https://lego.wd103.myworkdayjobs.com/wday/cxs/lego/LEGO_External/jobs" + assert payload == {"appliedFacets": {}, "limit": 1, "offset": 0, "searchText": ""} + + +def test_workday_facets_of_a_tenant_without_remote_state_no_remote() -> None: + found = workday.facets(Recorder(json.dumps(_workday_facets()).encode()), LEGO) + assert set(found) == {"country"} + + +def test_workday_applies_the_host_ids_grouped_by_facet() -> None: + get = Recorder(b'{"total": 0, "jobPostings": []}') + workday.page(get, LEGO, None, Ask(countries=("Location=a", "Location=b"))) + workday.page(get, LEGO, None, Ask(remote=True, remote_ids=("remoteType=r", "Location=x"))) + workday.page(get, LEGO, None, Ask()) + assert get.asked[0][1]["appliedFacets"] == {"Location": ["a", "b"]} + assert get.asked[1][1]["appliedFacets"] == {"remoteType": ["r"], "Location": ["x"]} + assert get.asked[2][1]["appliedFacets"] == {} + + +POD = "fa-evmr-saasfaprod1.fa.ocs.oraclecloud.com/CX_1" + + +def test_oracle_facets_map_the_listed_countries_and_mark_the_rest_unknown() -> None: + get = Recorder((FIXTURES / "oracle-facets.json").read_bytes()) + found = oracle.facets(get, POD)["country"] + assert found["FI"] == "300000000471640" and found["US"] == "300000000480126" + # Ten listed: the pod lists only its biggest, so Denmark may be there. + assert found["DK"] == "*" + assert "facetsList=LOCATIONS" in get.asked[0][0] and "expand" not in get.asked[0][0] + + +def test_oracle_facets_of_a_short_list_leave_the_rest_out() -> None: + saved = json.loads((FIXTURES / "oracle-facets.json").read_text()) + saved["items"][0]["locationsFacet"] = saved["items"][0]["locationsFacet"][:3] + found = oracle.facets(Recorder(json.dumps(saved).encode()), POD)["country"] + assert set(found) == {"US", "IN", "CA"} + + +def test_oracle_selects_locations_and_remote_in_the_finder() -> None: + get = Recorder(b'{"items": [{"requisitionList": [], "TotalJobsCount": 0}]}') + oracle.page(get, POD, None, Ask(countries=("300000000471640", "300000000480126"))) + oracle.page(get, POD, None, Ask(remote=True)) + oracle.page(get, POD, None, Ask(countries=("300000000471640", "*"))) + finders = [unquote(_query(url)["finder"][0]) for url, _ in get.asked] + assert finders[0] == ("findReqs;siteNumber=CX_1,selectedLocationsFacet=" + "300000000471640;300000000480126,limit=200,offset=0") + assert finders[1] == ("findReqs;siteNumber=CX_1,selectedWorkplaceTypesFacet=ORA_REMOTE," + "limit=200,offset=0") + # A country the pod did not list: the series reads the whole board. + assert finders[2] == "findReqs;siteNumber=CX_1,limit=200,offset=0" + + +# --------------------------------------------------------------- series + +def test_countries_and_remote_are_two_series_on_a_host_that_filters_both() -> None: + ask = _runner.ask_for(workday, {}, {"countries": ["DK", "SE"], "remote": True}) + facets = {"country": {"DK": "Location=dk", "SE": "Location=se"}, + "remote": {"remote": "remoteType=r"}} + asks = _runner.series(workday, ask, facets) + assert [(a.countries, a.remote, a.remote_ids) for a in asks] == [ + (("Location=dk", "Location=se"), None, ()), + ((), True, ("remoteType=r",))] + + +def test_remote_is_left_out_beside_countries_where_the_host_has_no_remote_facet() -> None: + ask = Ask(countries=("DK",), remote=True) + asks = _runner.series(workday, ask, {"country": {"DK": "Location=dk"}}) + assert [(a.countries, a.remote) for a in asks] == [(("Location=dk",), None)] + # Remote alone on such a host is the whole board. + [alone] = _runner.series(workday, Ask(remote=True), {"country": {}}) + assert (alone.countries, alone.remote, alone.remote_ids) == ((), None, ()) + + +def test_a_host_with_no_country_facet_is_read_unfiltered_not_emptied() -> None: + [whole] = _runner.series(workday, Ask(countries=("DK",)), {}) + assert whole.countries == () + + +def test_oracle_reads_countries_and_remote_as_two_series() -> None: + ask = _runner.ask_for(oracle, {}, {"countries": ["FI"], "remote": True}) + asks = _runner.series(oracle, ask, {"country": {"FI": "300000000471640"}}) + assert [(a.countries, a.remote) for a in asks] == [ + (("300000000471640",), None), ((), True)] + + +def test_a_one_host_gets_a_series_a_country_and_no_remote() -> None: + ask = _runner.ask_for(smartrecruiters, {}, {"countries": ["DK", "SE"], "remote": True}) + assert ask.remote is None + assert [a.countries for a in _runner.series(smartrecruiters, ask)] == [("DK",), ("SE",)] + + +def test_a_reach_too_wide_for_a_one_host_reads_it_unfiltered() -> None: + europe = [c for c in _countries.NAMES if c in { + "AT", "BE", "BG", "HR", "CY", "CZ", "DK", "EE", "FI", "FR", "DE", "GR", "HU", "IE"}] + assert len(europe) > _runner.MAX_SERIES + assert _runner.ask_for(eightfold, {}, {"countries": europe}) == Ask() + # A list in one request takes it whole. + assert _runner.ask_for(phenom, {}, {"countries": europe}).countries == tuple(europe) + # A board's own params are kept however many. + assert len(_runner.ask_for(eightfold, {"countries": europe}, None).countries) == len(europe) + + +@pytest.mark.parametrize("params, expected", [ + ({"reach": "all"}, Ask()), + ({"countries": ["DE"], "remote": False}, Ask(countries=("DE",))), + ({"remote": True}, Ask(countries=("DK",), remote=True)), +]) +def test_a_boards_params_win_over_the_reach(params: dict[str, Any], expected: Ask) -> None: + reach = {"countries": ["DK"], "remote": True} + assert _runner.ask_for(oracle, params, reach) == expected + + +def test_facets_are_asked_for_remote_alone_on_a_host_that_names_it_by_id() -> None: + calls: list[str] = [] + + def facets(get: Any, slug: str) -> dict[str, dict[str, str]]: + calls.append(slug) + return {"remote": {"remote": "remoteType=r"}} + + state: dict[str, str] = {} + original = workday.facets + try: + workday.facets = facets # type: ignore[assignment] + found = _runner.facets_for(workday, LEGO, Recorder(b""), Ask(remote=True), state, "k") + finally: + workday.facets = original # type: ignore[assignment] + assert found == {"remote": {"remote": "remoteType=r"}} and calls == [LEGO] diff --git a/tests/test_web.py b/tests/test_web.py index da12bec..c4677db 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -537,7 +537,8 @@ def test_companies_lists_board_and_open_role_count( "ats": "greenhouse", "ats_slug": "", "careers_url": "", "country": "DK", "city": "", "domain": "", "watch": None, "careers_opened_at": "", "failing_since": "", "failing_reason": "", - "from_search": False, "hand_reason": "", "by_hand": False} + "from_search": False, "hand_reason": "", "by_hand": False, + "reach_params": {}, "reach": "all roles · not read whole yet"} [other] = [c for c in res.json() if c["key"] == "other"] assert (other["board"], other["subscription_id"], other["on"], other["ats"]) == ( "", None, None, "") @@ -1386,7 +1387,7 @@ def test_a_company_is_added_by_hand_with_a_reason_and_a_link( "ats": "", "ats_slug": "", "careers_url": "https://revolut.example/careers", "country": "", "city": "", "watch": None, "careers_opened_at": "", "failing_since": "", "failing_reason": "", "from_search": False, - "hand_reason": "added", "by_hand": True} + "hand_reason": "added", "by_hand": True, "reach_params": {}, "reach": ""} assert store.company("revolut").hand_reason == "added" # type: ignore[union-attr] other = client.post("/api/companies", json={"name": "Meta", "hand_reason": "blocked"}) diff --git a/tests/test_web_reach.py b/tests/test_web_reach.py new file mode 100644 index 0000000..b3353e4 --- /dev/null +++ b/tests/test_web_reach.py @@ -0,0 +1,107 @@ +"""A board's own reach on the Companies list, and the readers' declarations +on the Sources page, through the JSON API. No network.""" + +from __future__ import annotations + +from collections.abc import Iterator +from pathlib import Path + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from app.config import Config, ServerConfig +from app.main import create_app +from app.models import Gate +from app.storage import Store + + +@pytest.fixture +def app(tmp_path: Path) -> FastAPI: + return create_app(Config(server=ServerConfig(host="127.0.0.1", port=0), + database=tmp_path / "roledesk.db")) + + +@pytest.fixture +def store(app: FastAPI) -> Store: + return app.state.store + + +@pytest.fixture +def client(app: FastAPI) -> Iterator[TestClient]: + with TestClient(app) as c: + yield c + + +@pytest.fixture +def bosch(store: Store) -> int: + """A big SmartRecruiters board and an agent that wants Denmark or remote.""" + store.save_agent("Wolf", gates=[Gate(field="country", values=["DK"], mode="any"), + Gate(field="remote", values=["remote"], mode="any")]) + sub = store.add_subscription("ats", "smartrecruiters:BoschGroup", label="Bosch", + company="Bosch", board="smartrecruiters", + params={"note": "kept"}) + store.add_company("Bosch") + store.state["ats:size:smartrecruiters:BoschGroup"] = "4830" + return sub.id + + +def _row(client: TestClient) -> dict[str, object]: + return next(c for c in client.get("/api/companies").json() if c["key"] == "bosch") + + +def test_a_company_row_says_what_its_board_is_asked_for(client: TestClient, bosch: int) -> None: + row = _row(client) + # SmartRecruiters cannot ask for remote, so only the country is asked. + assert (row["reach"], row["reach_params"]) == ("DK · agents", {}) + + +def test_a_boards_own_reach_is_kept_in_its_params_and_wins( + client: TestClient, store: Store, bosch: int) -> None: + res = client.post(f"/api/sources/{bosch}/reach", + json={"mode": "own", "countries": ["de", "se"], "remote": True}) + assert res.status_code == 200, res.text + assert res.json()["reach"] == "DE, SE · set here" + sub = store.subscription(bosch) + assert sub is not None + assert sub.params == {"note": "kept", "countries": ["DE", "SE"], "remote": True} + assert _row(client)["reach_params"] == {"countries": ["DE", "SE"], "remote": True} + + res = client.post(f"/api/sources/{bosch}/reach", json={"mode": "all"}) + assert res.json()["reach"] == "all roles · set here" + assert store.subscription(bosch).params == {"note": "kept", "reach": "all"} # type: ignore[union-attr] + + res = client.post(f"/api/sources/{bosch}/reach", json={"mode": "agents"}) + assert res.json()["reach"] == "DK · agents" + assert store.subscription(bosch).params == {"note": "kept"} # type: ignore[union-attr] + + +@pytest.mark.parametrize("body, status", [ + ({"mode": "own"}, 422), + ({"mode": "own", "countries": ["XX"]}, 422), + ({"mode": "own", "countries": ["EU"]}, 422), + ({"mode": "nowhere"}, 422), +]) +def test_a_reach_that_names_nothing_is_refused( + client: TestClient, bosch: int, body: dict[str, object], status: int) -> None: + assert client.post(f"/api/sources/{bosch}/reach", json=body).status_code == status + + +def test_a_search_or_an_unknown_board_has_no_reach(client: TestClient, store: Store) -> None: + search = store.add_subscription("search", "jobindex:x", label="X", board="jobindex", + params={"q": "python"}) + assert client.post(f"/api/sources/{search.id}/reach", json={"mode": "all"}).status_code == 404 + assert client.post("/api/sources/999/reach", json={"mode": "all"}).status_code == 404 + + +def test_the_readers_are_listed_with_what_they_declare(client: TestClient) -> None: + found = {r["name"]: r for r in client.get("/api/readers").json()} + assert found["workday"] == { + "name": "workday", "label": "Workday", "scope": "board", "listing": "paged", + "text": "detail", "filters": {"country": "ids", "remote": "ids"}, "places": "many", + "robots": found["workday"]["robots"]} + assert found["workday"]["robots"].startswith("read:") + assert found["greenhouse"]["filters"] == {} + scopes = [r["scope"] for r in client.get("/api/readers").json()] + assert scopes == sorted(scopes) # boards first, then the searches + assert "search" in scopes