diff --git a/app/jobs/pull.py b/app/jobs/pull.py index 41d8819..c0a1cab 100644 --- a/app/jobs/pull.py +++ b/app/jobs/pull.py @@ -17,8 +17,8 @@ cap) stores what it read and closes nothing; a paged board over the page cap is swept across pulls and closes once the sweep reached its end (see `plugins/readers/_runner.py`). -A board read with a filter (`ats:filtered:`, from its subscription's -`params`) lists only the roles in reach, so its close keeps open the roles a +A board read with a filter (`ats:filtered:`, from the agents' reach, +`app.jobs.reach`, or its subscription's `params`) lists only the roles in reach, so its close keeps open the roles a person acted on: those with a vote or an application (`_close_unlisted`). A filtered read, or one of a reader that declares `empty="suspect"`, that lists under a tenth of what the same read listed last time is recorded by the @@ -51,6 +51,7 @@ from typing import Any from pipeline import NULL, RunContext, Tally, Task, job from app.config import Config +from app.jobs.reach import reaches from app.models import company_key from app.storage import Store @@ -131,8 +132,8 @@ def _boards(config: Config, store: Store, run: str = "") -> list[dict[str, Any]] its own, and queues nothing when another run is in flight. A run with nothing to read is settled here, since no settle will see a batch of it.""" seed_searches(config, store) - targets = [(source, [s.target for s in store.subscriptions(source=source, fetchable=True)]) - for source in SOURCES] + subs = {source: store.subscriptions(source=source, fetchable=True) for source in SOURCES} + targets = [(source, [s.target for s in subs[source]]) for source in SOURCES] if not any(found for _, found in targets): if run: release_run(store, run) @@ -142,13 +143,20 @@ def _boards(config: Config, store: Store, run: str = "") -> list[dict[str, Any]] log.warning("pull: %s since %s; this one reads nothing", ALREADY_RUNNING, running(store)) return [] - return [{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run} - for source, found in targets for i in range(0, len(found), BATCH_SIZE)] + # The agents' reach, once per run, each board's in its batch (`app.jobs.reach`). + reach_of = reaches(store, subs["ats"]) + batches = [{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run} + for source, found in targets for i in range(0, len(found), BATCH_SIZE)] + for batch in batches: + if batch["source"] == "ats": + batch["reach"] = {t: reach_of[t] for t in batch["targets"] if t in reach_of} + return batches @job("pull_batch", "Reading boards", resource="http") def run_batch(config: Config, store: Store, *, targets: list[str], source: str = "ats", - run: str = "", context: RunContext = NULL) -> dict[str, Any]: + run: str = "", reach: dict[str, Any] | None = None, + context: RunContext = NULL) -> dict[str, Any]: """Fetch this batch's boards or searches and `upsert_role` every item they list. @@ -163,7 +171,9 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str = `run` is the marker of the run that queued this batch, handed to the plugin as its `run` option, so its stamps belong to that run whatever the - stored marker says by the time the batch runs. + stored marker says by the time the batch runs. `reach` is `target ->` the + reach the run computed for a board (`app.jobs.reach`), handed to the + plugin as its `reach` option. """ from plugins.ats import ATSSource from plugins.search import SearchSource @@ -189,6 +199,7 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str = options["boards"] = [{"url": target, "label": sub.company or sub.label, "params": sub.params}] options["keys"] = dict(config.sources.get("search") or {}) + options["reach"] = (reach or {}).get(target) plugin = ATSSource(options) plugin.state = store.state for item in plugin.fetch(): diff --git a/app/jobs/reach.py b/app/jobs/reach.py new file mode 100644 index 0000000..e494b96 --- /dev/null +++ b/app/jobs/reach.py @@ -0,0 +1,235 @@ +"""The reach of a pull: which countries, and whether remote roles, each +company board is asked for, derived from the search agents that are switched +on. Computed once per pull (`plan`) and handed to each board's batch as the +`reach` the runner narrows to its reader's filters +(`plugins/readers/_runner.py`, `ask_for`). + +- **Base reach**: the union of the enabled agents' `any` gates on country, + place (a town or region, mapped to its country through the gazetteer) and + remote. An agent whose `any` group is empty, or holds any other member (a + function, a remote value other than `remote`, a place the gazetteer does + not know), can take a role from anywhere, so the base reach is then + everywhere. +- **Favourite companies** (affinity 2, which skips the `any` gates in + scoring) are read with the `require` reach of the agent that starred them: + the intersection of its `require` gates on country and place, Europe + today, expanded to its countries through the gazetteer. A company is + starred per agent, so its reach is the union over the enabled agents of + the `require` reach where it is a favourite and the `any` reach where not. +- **Board override**: a board whose subscription `params` name a country, a + place, remote or `{"reach": "all"}` gets no reach here; `ask_for` reads the + params instead. +- **Small boards**: a board that listed under `SMALL` roles when last read + whole gets no filter; filtering it saves little and risks closing roles. + +A reach of more countries than a `one`-filter host should be asked one by +one (a favourite's Europe on SmartRecruiters) is dropped by `ask_for`, which +knows the reader (`MAX_SERIES`). +""" + +from __future__ import annotations + +from collections.abc import Iterable, Mapping +from dataclasses import dataclass, field +from typing import Any + +from app import places +from app.models import Agent, Gate, Subscription, company_key +from app.storage import Store + +#: A board that listed fewer roles than this when last read whole is read +#: whole. +SMALL = 300 + +#: `state` key of a board's size, `ats:size:`: the roles its last +#: unfiltered read listed, written by `plugins/ats.py`. +SIZE = "ats:size:" + +#: The params that make a board's own reach (see `_runner._PARAM_DIMENSION`). +OVERRIDE_KEYS = frozenset(("countries", "country", "places", "place", "remote", "reach")) + + +@dataclass(frozen=True) +class Reach: + """Countries (ISO codes) and remote, or everywhere.""" + + countries: frozenset[str] = frozenset() + remote: bool = False + everywhere: bool = False + + def __or__(self, other: Reach) -> Reach: + if self.everywhere or other.everywhere: + return EVERYWHERE + return Reach(self.countries | other.countries, self.remote or other.remote) + + def params(self) -> dict[str, Any] | None: + """What `ask_for` takes as `reach`; None for everywhere.""" + if self.everywhere: + return None + out: dict[str, Any] = {"countries": sorted(self.countries)} + if self.remote: + out["remote"] = True + return out + + def describe(self) -> str: + """"DK, SE, remote", "31 countries", "everywhere".""" + if self.everywhere: + return "everywhere" + codes = sorted(self.countries) + words = [", ".join(codes)] if len(codes) <= 4 else [f"{len(codes)} countries"] + if self.remote: + words.append("remote") + return ", ".join(w for w in words if w) or "nothing" + + +EVERYWHERE = Reach(everywhere=True) +NOWHERE = Reach() + + +def _countries_of(ids: Iterable[int], gaz: places.Gazetteer) -> set[str]: + """The ISO codes of the countries the places lie in, a continent's every + country.""" + out: set[str] = set() + for place_id in ids: + place = gaz.places.get(place_id) + if place is None: + continue + if place.kind == "continent": + out |= {gaz.places[p].key for p in gaz.under((place_id,)) + if gaz.places[p].kind == "country"} + elif code := gaz.country(place_id): + out.add(code) + return out + + +def gate_reach(gate: Gate, gaz: places.Gazetteer) -> Reach: + """Where a role that meets `gate` can be: its countries, remote, or + everywhere for a gate that says nothing of place or names one the + gazetteer does not know.""" + values = [str(v).strip() for v in gate.values if str(v).strip()] + if gate.field == "remote": + return Reach(remote=True) if values and all(v.lower() == "remote" for v in values) \ + else EVERYWHERE + if gate.field not in ("country", "place") or not values: + return EVERYWHERE + found: set[str] = set() + for value in values: + named = gaz.lookup(f"country:{value}" if gate.field == "country" else value) + if named is None: + if gate.field == "country" and len(value) == 2 and value.isalpha(): + found.add(value.upper()) + continue + return EVERYWHERE + found |= _countries_of(named, gaz) + return Reach(frozenset(found)) if found else EVERYWHERE + + +def any_reach(agent: Agent, gaz: places.Gazetteer) -> Reach: + """The union of the agent's `any` gates; everywhere with none.""" + group = [g for g in agent.gates if g.mode == "any"] + if not group: + return EVERYWHERE + out = NOWHERE + for gate in group: + out = out | gate_reach(gate, gaz) + return out + + +def require_reach(agent: Agent, gaz: places.Gazetteer) -> Reach: + """The intersection of the agent's `require` gates on country and place, + or remote alone for a `require` of remote only; everywhere with neither, + or when the intersection is empty.""" + kept: frozenset[str] | None = None + remote = False + for gate in agent.gates: + if gate.mode != "require": + continue + reach = gate_reach(gate, gaz) + if reach.everywhere: + continue + if reach.countries: + kept = reach.countries if kept is None else kept & reach.countries + elif reach.remote: + remote = True + if kept: + return Reach(kept) + return Reach(remote=True) if remote and kept is None else EVERYWHERE + + +@dataclass +class Plan: + """The reach of one pull: `base` for every board, `favourites` for the + companies some agent starred, and each board's size by target.""" + + base: Reach + favourites: dict[str, Reach] = field(default_factory=dict) + sizes: dict[str, int] = field(default_factory=dict) + + def why(self, sub: Subscription) -> tuple[Reach, str]: + """The reach `sub` is read with, and why: `override`, `small`, + `unknown` (a board never read whole), `favourite` or `agents`.""" + if OVERRIDE_KEYS & set(sub.params or {}): + return EVERYWHERE, "override" + size = self.sizes.get(sub.target) + if size is None: + return EVERYWHERE, "unknown" + if size < SMALL: + return EVERYWHERE, "small" + key = company_key(sub.company or sub.label) + if key in self.favourites: + return self.favourites[key], "favourite" + return self.base, "agents" + + def for_subscription(self, sub: Subscription) -> dict[str, Any] | None: + """The `reach` a board's batch hands `ask_for`; None for none.""" + return self.why(sub)[0].params() + + +def sizes(store: Store, subs: Iterable[Subscription]) -> dict[str, int]: + """Each board's size: the roles its last unfiltered read listed + (`ats:size:`), else, for a board read before that was kept and + never filtered, its open roles. A board never read has none.""" + state = store.state + counts = store.open_countries_by_subscription() + out: dict[str, int] = {} + for sub in subs: + kept = state.get(f"{SIZE}{sub.target}", "") + if kept.isdigit(): + out[sub.target] = int(kept) + elif state.get(f"ats:ok:{sub.target}") and not state.get(f"ats:filtered:{sub.target}"): + out[sub.target] = sum(counts.get(sub.id, {}).values()) + return out + + +def plan(store: Store, subs: Iterable[Subscription] | None = None) -> Plan: + """The reach of a pull, from the enabled agents and their favourites, and + the sizes of `subs` (every `ats` subscription when None).""" + gaz = store.gazetteer() + agents = store.agents(enabled_only=True) + reaches = {a.id: (any_reach(a, gaz), require_reach(a, gaz)) for a in agents} + base = NOWHERE + for any_of, _ in reaches.values(): + base = base | any_of + if not agents: + base = EVERYWHERE + starred: dict[str, set[int]] = {} + for agent in agents: + for company in store.companies(agent.id): + if company.affinity == 2: + starred.setdefault(company.key, set()).add(agent.id) + favourites: dict[str, Reach] = {} + for key, by in starred.items(): + reach = NOWHERE + for agent_id, (any_of, required) in reaches.items(): + reach = reach | (required if agent_id in by else any_of) + favourites[key] = reach + subs = list(store.subscriptions(source="ats") if subs is None else subs) + return Plan(base, favourites, sizes(store, subs)) + + +def reaches(store: Store, subs: Iterable[Subscription]) -> Mapping[str, dict[str, Any]]: + """`target -> reach` for the boards that get one, what a pull puts in + its batches' payloads.""" + subs = list(subs) + found = plan(store, subs) + return {s.target: r for s in subs if (r := found.for_subscription(s)) is not None} diff --git a/plugins/ats.py b/plugins/ats.py index 167835d..83c6074 100644 --- a/plugins/ats.py +++ b/plugins/ats.py @@ -17,8 +17,12 @@ A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`, `lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what its slug is. Every board is read independently; one that fails is logged, recorded as failing under `ats:failing:`, and skipped. A board's -subscription `params` (`{"countries": ["DK"], "remote": true}`) are its -filter, narrowed to what its reader declares (`_runner.ask_for`). +filter is the `reach` option the pull hands it (the agents', see +`app/jobs/reach.py`), over which its subscription `params` (`{"countries": +["DK"], "remote": true}`, `{"reach": "all"}`) win, narrowed to what its +reader declares (`_runner.ask_for`). An unfiltered read records the board's +size under `ats:size:`, which the reach reads to leave small boards +whole. The limits below are read when a board is read, so a test that replaces one on this module reaches the runner. @@ -349,7 +353,7 @@ class ATSSource(Source): raise ValueError(f"{target!r} is not host:slug for a known ATS host") module = readers.HOSTS[host] get = _getter(client, host, slug) - ask = _runner.ask_for(module, self.params_for(target), + ask = _runner.ask_for(module, self.params_for(target), self.options.get("reach"), keys=self.options.get("keys")) listing = _runner.read(self.state, "ats", target, module, slug, get, ask, limits=_limits()) @@ -398,6 +402,12 @@ class ATSSource(Source): # its details again. _runner.stamp(self.state, "ats", target, run_marker, listing, [compound(i) for i in listing.ids]) + if not listing.filtered: + size = len(listing.ids) + if not listing.complete: + kept = self.state.get(f"ats:size:{target}", "") + size = max(size, int(kept) if kept.isdigit() else 0) + self.state[f"ats:size:{target}"] = str(size) if detail_sets is not None: for key, ids in zip(detail_keys, detail_sets, strict=True): self.state[key] = json.dumps(sorted(ids)) diff --git a/plugins/readers/_contract.py b/plugins/readers/_contract.py index 250d494..4f8305f 100644 --- a/plugins/readers/_contract.py +++ b/plugins/readers/_contract.py @@ -21,6 +21,11 @@ Signatures of a module that declares `READS`, with `Get` and `Page` from def detail(get: Get, slug: str, row: Row) -> dict[str, Any] # text="detail" def facets(get: Get, slug: str) -> dict[str, dict[str, str]] # an "ids" filter +`facets` maps each `ids` dimension to `{value: host id}`: an ISO code for +`country`, a place name for `place`, any word for `remote`. A dimension the +host has no facet for is left out, and the runner then reads it unfiltered; +a dimension present without a wanted value means the host has none of it. + A module without `READS` keeps the signatures without `ask` until it declares; `undeclared()` in the package lists those modules. """ @@ -146,11 +151,20 @@ class Ask(_Frozen): carries one value. With `ids`, `countries` and `places` hold the host's ids from `facets`, not codes or names. `params` are a search's saved params (and a resolved key under `"key"`). An empty `Ask()` is an - unfiltered read.""" + unfiltered read. + + `remote=True` asks for remote roles. A reach of countries and remote + means "in these countries, or remote", which no host can ask in one + request (two facets intersect), so the runner makes the remote roles a + series of their own: a reader never gets both `remote=True` and a + country or a place. With `remote="ids"`, `remote_ids` holds the host's + ids of its remote values (`facets(...)["remote"]`, any key), and + `remote` stays True.""" countries: tuple[str, ...] = () places: tuple[str, ...] = () remote: bool | None = None + remote_ids: tuple[str, ...] = () query: str = "" params: dict[str, Any] = {} diff --git a/plugins/readers/_countries.py b/plugins/readers/_countries.py new file mode 100644 index 0000000..2acea40 --- /dev/null +++ b/plugins/readers/_countries.py @@ -0,0 +1,93 @@ +"""Country codes and the English names hosts write, for the readers that +filter by a country's name (Eightfold, Phenom) or name a host id by it +(Workday, Oracle), and for SuccessFactors, which writes a place's country as +its code. + +`NAMES` is ISO 3166-1 alpha-2 to the English short name. `code_of` reads a +name back, with the spellings hosts use that are not the short name +("United States of America", "Korea, Republic of", "Czech Republic"). +""" + +from __future__ import annotations + +import unicodedata + +#: ISO 3166-1 alpha-2 to the English name, which is what `country_of` reads: +#: a bare "DE" or "SE" in a location line is taken for Delaware or nothing. +NAMES: dict[str, str] = dict(entry.strip().split(" ", 1) for entry in """ +AD Andorra | AE United Arab Emirates | AF Afghanistan | AG Antigua and Barbuda | AI +Anguilla | AL Albania | AM Armenia | AO Angola | AR Argentina | AS American Samoa | AT +Austria | AU Australia | AW Aruba | AX Åland Islands | AZ Azerbaijan | BA Bosnia and +Herzegovina | BB Barbados | BD Bangladesh | BE Belgium | BF Burkina Faso | BG Bulgaria +| BH Bahrain | BI Burundi | BJ Benin | BM Bermuda | BN Brunei | BO Bolivia | BQ +Caribbean Netherlands | BR Brazil | BS Bahamas | BT Bhutan | BW Botswana | BY Belarus +| BZ Belize | CA Canada | CD Democratic Republic of the Congo | CF Central African +Republic | CG Republic of the Congo | CH Switzerland | CI Côte d'Ivoire | CK Cook +Islands | CL Chile | CM Cameroon | CN China | CO Colombia | CR Costa Rica | CU Cuba | +CV Cabo Verde | CW Curaçao | CY Cyprus | CZ Czechia | DE Germany | DJ Djibouti | DK +Denmark | DM Dominica | DO Dominican Republic | DZ Algeria | EC Ecuador | EE Estonia | +EG Egypt | ER Eritrea | ES Spain | ET Ethiopia | FI Finland | FJ Fiji | FK Falkland +Islands | FM Micronesia | FO Faroe Islands | FR France | GA Gabon | GB United Kingdom +| GD Grenada | GE Georgia | GF French Guiana | GG Guernsey | GH Ghana | GI Gibraltar | +GL Greenland | GM Gambia | GN Guinea | GP Guadeloupe | GQ Equatorial Guinea | GR +Greece | GT Guatemala | GU Guam | GW Guinea-Bissau | GY Guyana | HK Hong Kong | HN +Honduras | HR Croatia | HT Haiti | HU Hungary | ID Indonesia | IE Ireland | IL Israel +| IM Isle of Man | IN India | IQ Iraq | IR Iran | IS Iceland | IT Italy | JE Jersey | +JM Jamaica | JO Jordan | JP Japan | KE Kenya | KG Kyrgyzstan | KH Cambodia | KI +Kiribati | KM Comoros | KN Saint Kitts and Nevis | KP North Korea | KR South Korea | +KW Kuwait | KY Cayman Islands | KZ Kazakhstan | LA Laos | LB Lebanon | LC Saint Lucia +| LI Liechtenstein | LK Sri Lanka | LR Liberia | LS Lesotho | LT Lithuania | LU +Luxembourg | LV Latvia | LY Libya | MA Morocco | MC Monaco | MD Moldova | ME +Montenegro | MG Madagascar | MH Marshall Islands | MK North Macedonia | ML Mali | MM +Myanmar | MN Mongolia | MO Macao | MP Northern Mariana Islands | MQ Martinique | MR +Mauritania | MS Montserrat | MT Malta | MU Mauritius | MV Maldives | MW Malawi | MX +Mexico | MY Malaysia | MZ Mozambique | NA Namibia | NC New Caledonia | NE Niger | NG +Nigeria | NI Nicaragua | NL Netherlands | NO Norway | NP Nepal | NR Nauru | NU Niue | +NZ New Zealand | OM Oman | PA Panama | PE Peru | PF French Polynesia | PG Papua New +Guinea | PH Philippines | PK Pakistan | PL Poland | PR Puerto Rico | PS Palestine | PT +Portugal | PW Palau | PY Paraguay | QA Qatar | RE Réunion | RO Romania | RS Serbia | +RU Russia | RW Rwanda | SA Saudi Arabia | SB Solomon Islands | SC Seychelles | SD +Sudan | SE Sweden | SG Singapore | SI Slovenia | SK Slovakia | SL Sierra Leone | SM +San Marino | SN Senegal | SO Somalia | SR Suriname | SS South Sudan | ST São Tomé and +Príncipe | SV El Salvador | SY Syria | SZ Eswatini | TC Turks and Caicos Islands | TD +Chad | TG Togo | TH Thailand | TJ Tajikistan | TL Timor-Leste | TM Turkmenistan | TN +Tunisia | TO Tonga | TR Turkey | TT Trinidad and Tobago | TV Tuvalu | TW Taiwan | TZ +Tanzania | UA Ukraine | UG Uganda | US United States | UY Uruguay | UZ Uzbekistan | VA +Vatican City | VC Saint Vincent and the Grenadines | VE Venezuela | VG British Virgin +Islands | VI U.S. Virgin Islands | VN Vietnam | VU Vanuatu | WS Samoa | XK Kosovo | YE +Yemen | ZA South Africa | ZM Zambia | ZW Zimbabwe +""".replace("\n", " ").split(" | ")) + +#: Other names hosts give a country: Workday's "United States of America" and +#: "Korea, Republic of", older and local spellings. +_OTHER = { + "united states of america": "US", "usa": "US", "u.s.a.": "US", "u.s.": "US", + "united kingdom of great britain and northern ireland": "GB", "great britain": "GB", + "uk": "GB", "england": "GB", "korea, republic of": "KR", "republic of korea": "KR", + "korea": "KR", "czech republic": "CZ", "turkiye": "TR", "viet nam": "VN", + "russian federation": "RU", "hong kong sar": "HK", "hong kong sar, china": "HK", + "taiwan, province of china": "TW", "macau": "MO", "the netherlands": "NL", + "holland": "NL", "slovak republic": "SK", "republic of ireland": "IE", + "ivory coast": "CI", "cote d'ivoire": "CI", "north macedonia, republic of": "MK", + "macedonia": "MK", "iran, islamic republic of": "IR", "moldova, republic of": "MD", + "tanzania, united republic of": "TZ", "united arab emirates (uae)": "AE", "uae": "AE", +} + + +def _fold(name: str) -> str: + decomposed = unicodedata.normalize("NFKD", name.strip().casefold()) + return " ".join("".join(c for c in decomposed if not unicodedata.combining(c)).split()) + + +_BY_NAME: dict[str, str] = {_fold(name): code for code, name in NAMES.items()} | _OTHER + + +def code_of(name: str) -> str: + """The ISO code of a country written by name, '' for a name not known. + Case and accents do not matter.""" + return _BY_NAME.get(_fold(name or ""), "") + + +def name_of(code: str) -> str: + """The English name of an ISO code, '' for a code not known.""" + return NAMES.get((code or "").strip().upper(), "") diff --git a/plugins/readers/_runner.py b/plugins/readers/_runner.py index 73bf67e..bcdb189 100644 --- a/plugins/readers/_runner.py +++ b/plugins/readers/_runner.py @@ -289,14 +289,22 @@ def resolve_key(key: Key, keys: Mapping[str, Any] | None) -> dict[str, str]: return out +#: The most series a reach may cost on a `one` filter: a reach of more +#: countries (a favourite's Europe) reads such a host unfiltered instead. +MAX_SERIES = 8 + + def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None = None, reach: Mapping[str, Any] | None = None, *, keys: Mapping[str, Any] | None = None) -> Ask: - """The `Ask` one target resolves to. `reach` (from the agents, package F) - and then the subscription's own `params` (a board's override, e.g. - `{"countries": ["DK"], "remote": true}`, or a search's saved params) set - the dimensions, params winning per dimension, narrowed to the ones the - reader declares in `READS.filters`. A search's params go whole into + """The `Ask` one target resolves to. `reach` (from the agents, see + `app/jobs/reach.py`) and then the subscription's own `params` (a board's + override, e.g. `{"countries": ["DK"], "remote": true}`, or a search's + saved params) set the dimensions, params winning per dimension, narrowed + to the ones the reader declares in `READS.filters`. `{"reach": "all"}` in + the params drops the reach, and so does a reach of more than + `MAX_SERIES` countries on a reader whose country filter is `one`. A + `remote` of False asks nothing. A search's params go whole into `Ask.params`; a reader with `READS.key` gets the resolved key under `Ask.params["key"]` (see `resolve_key`, which raises when it is missing). A module without `READS` gets `Ask()`.""" @@ -304,7 +312,12 @@ def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None = return Ask() reads: Reads = module.READS params = dict(subscription_params or {}) - wanted = _dimensions(module, reach or {}) | _dimensions(module, params) + reached = {} if params.get("reach") == "all" else _dimensions(module, reach or {}) + if reads.filters.country == "one" and len(reached.get("country", ())) > MAX_SERIES: + reached = {} + wanted = reached | _dimensions(module, params) + if wanted.get("remote") is False: + del wanted["remote"] fields: dict[str, Any] = {_ASK_FIELD[name]: value for name, value in wanted.items() if name in reads.filters.declared()} extra: dict[str, Any] = dict(params) if reads.scope == "search" else {} @@ -320,29 +333,61 @@ def filtered(ask: Ask) -> bool: def series(module: ModuleType, ask: Ask, facets: Mapping[str, Mapping[str, str]] | None = None) -> list[Ask]: - """The series of requests `ask` takes on this reader: one per value of a - dimension declared `one` (the product when two are), one with every - value for `many`, and host ids from `facets` for `ids` (values `facets` - does not know are dropped; an empty list when none is left, since the - host then has nothing in reach). `[ask]` for a module without `READS`.""" + """The series of requests `ask` takes on this reader. Remote with a + country or a place is two parts, "in these places, or remote": the + places without remote, and remote alone. Each part is one series per + value of a dimension declared `one` (the product when two are), one + with every value for `many`, and host ids from `facets` for `ids`: + values `facets` does not know are dropped, and a part with none left is + no series, since the host has nothing of it; a dimension `facets` does + not state at all is read unfiltered, and so is remote alone, while the + remote part beside places is then left out. `[ask]` for a module + without `READS`.""" if not declared(module): return [ask] + if ask.remote and (ask.countries or ask.places): + return [*_series(module, ask.model_copy(update={"remote": None}), facets), + *_series(module, ask.model_copy(update={"countries": (), "places": ()}), + facets, beside=True)] + return _series(module, ask, facets) + + +def _series(module: ModuleType, ask: Ask, facets: Mapping[str, Mapping[str, str]] | None, + beside: bool = False) -> list[Ask]: + """`series` for an ask that is places or remote, not both. `beside` marks + the remote part of an ask that also has places.""" reach = module.READS.filters + known_facets = facets or {} countries, places = ask.countries, ask.places for dimension in ("country", "place"): values = countries if dimension == "country" else places if getattr(reach, dimension) != "ids" or not values: continue - known = (facets or {}).get(dimension) or {} - mapped = tuple(known[v] for v in values if v in known) - if not mapped: - log.info("readers: %s knows none of %s %s", module.NAME, dimension, - ", ".join(values)) - return [] + if dimension not in known_facets: + log.info("readers: %s states no %s facet; read unfiltered", module.NAME, dimension) + mapped: tuple[str, ...] = () + else: + known = known_facets.get(dimension) or {} + mapped = tuple(dict.fromkeys(known[v] for v in values if v in known)) + if not mapped: + log.info("readers: %s knows none of %s %s", module.NAME, dimension, + ", ".join(values)) + return [] if dimension == "country": countries = mapped else: places = mapped + if reach.remote == "ids" and ask.remote: + if "remote" not in known_facets: + if beside: + log.info("readers: %s states no remote facet; remote left out", module.NAME) + return [] + ask = ask.model_copy(update={"remote": None}) + else: + ids = tuple(dict.fromkeys((known_facets.get("remote") or {}).values())) + if not ids: + return [] + ask = ask.model_copy(update={"remote_ids": ids}) by_country = [(c,) for c in countries] if reach.country == "one" and countries \ else [countries] by_place = [(p,) for p in places] if reach.place == "one" and places else [places] @@ -505,12 +550,13 @@ def _load(state: Mapping[str, str], key: str) -> dict[str, Any]: def facets_for(module: ModuleType, slug: str, get: Get, ask: Ask, state: MutableMapping[str, str] | None = None, key: str = "") -> dict[str, Any]: - """The reader's `facets` when `ask` sets a dimension it declares `ids`, + """The reader's `facets` when `ask` sets a dimension it declares `ids` + (a country, a place, or remote), kept in `state[key]` for `FACETS_TTL`; `{}` otherwise.""" if not declared(module) or not hasattr(module, "facets"): return {} ids = [name for name, reach in module.READS.filters.declared().items() if reach == "ids"] - if not any(getattr(ask, _ASK_FIELD[name]) for name in ids if name in ("country", "place")): + if not any(getattr(ask, _ASK_FIELD[name]) for name in ids if name != "query"): return {} now = datetime.now(UTC) if state is not None and key: diff --git a/plugins/readers/eightfold.py b/plugins/readers/eightfold.py index fe2364f..075a7c9 100644 --- a/plugins/readers/eightfold.py +++ b/plugins/readers/eightfold.py @@ -15,14 +15,17 @@ links. `/api/apply/v2/jobs` answers 403 "Not authorized for PCSX" on the sites tried, so it is not used. The search ignores `num` and answers ten roles a page, so a board costs one -request per ten roles, and `ats.MAX_PAGES` (100) stops a read at 1,000 roles: a -larger board (Infineon's 1,292, Microsoft's 2,319) is read incomplete every pull, -and an incomplete read never closes the roles it did not list. The rows are -sorted newest first (`sort_by=timestamp`) so that what the cap keeps is the -newest thousand and the pages do not shuffle between two requests, unlike the -default order, which puts hot roles first. The search does take a `location` -free text (Ericsson's "Sweden" found 157 of 637), which could bring a read -under the cap, but a target names no filter and none is built here. +request per ten roles, and `ats.MAX_PAGES` (100) stops a read at 1,000 roles. +The rows are sorted newest first (`sort_by=timestamp`) so that what the cap +keeps is the newest thousand and the pages do not shuffle between two +requests, unlike the default order, which puts hot roles first. + +A country is filtered on the server by `location=`, one a +request (`Filters(country="one")`): Ericsson's "Sweden" answered 154 of 618 on +2026-09-30, a role at several places is kept when one of them matches, and +the review measured Infineon at 163 of 1,288 and Microsoft at 60 of 2,320 for +Denmark and Sweden, which brings both under the cap. A code with no English +name in `_countries` is read unfiltered. Discovery names Eightfold from an `eightfold.ai` link, and the host and the domain from one that carries `domain=`. @@ -35,15 +38,17 @@ import json import re from collections.abc import Callable from typing import Any +from urllib.parse import quote from desk.sources import strip_html -from ._contract import Ask, Fields, Place, Reads, Row +from ._contract import Ask, Fields, Filters, Place, Reads, Row +from ._countries import name_of from ._shared import Get, Page, is_web_url, only, parse_dt READS = Reads( scope="board", listing="paged", page_size=10, text="detail", complete="stated", - places="many", + places="many", filters=Filters(country="one"), fields=Fields(posted="listed", remote="stated"), robots="read: robots.txt allows /careers (checked 2026-09-29)", ) @@ -95,12 +100,14 @@ def careers_url(slug: str) -> str: def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page: """One page of `/api/pcsx/search` -> `{"status": 200, "data": {"positions": - [...], "count": n}}`; the next starts at the offset after this page's - positions, and an empty page ends the read.""" + [...], "count": n}}`, in the one country `ask` names; the next starts at + the offset after this page's positions, and an empty page ends the read.""" host, domain = parts(slug) start = int(cursor or 0) + country = name_of(ask.countries[0]) if ask.countries else "" + where = f"&location={quote(country)}" if country else "" answer = json.loads(get(f"https://{host}/api/pcsx/search?domain={domain}" - f"&sort_by=timestamp&start={start}")) + f"&sort_by=timestamp&start={start}{where}")) data = answer.get("data") if isinstance(answer, dict) else None positions = data.get("positions") if isinstance(data, dict) else None if not isinstance(positions, list): diff --git a/plugins/readers/oracle.py b/plugins/readers/oracle.py index e75057a..c02826f 100644 --- a/plugins/readers/oracle.py +++ b/plugins/readers/oracle.py @@ -8,6 +8,18 @@ number is `CX_1001` or `CX_1` on the tenants read so far, and the site's own name (`SinchCareer`) is taken in its place. Written from live boards on 2026-09-29 (Milestone Systems, Nokia, Danske Bank, Sinch); jobhunt-hq did not read Oracle. + +Countries are filtered on the server by `selectedLocationsFacet`, the pod's +geography ids (`Filters(country="ids")`), which `facets` reads off +`locationsFacet`; several ids go in one request, joined with `;`, the way the +site's own search sends a multiple choice. That list holds only the pod's ten +biggest countries (fa-evmr's, 2026-09-30), so when it is full a country +missing from it may still have roles: `facets` then maps every other country +to `*`, and a series asking for one is read without a country filter. An id +the pod does not know answers the whole board (review). Remote is +`selectedWorkplaceTypesFacet=ORA_REMOTE` (`remote="one"`); two facets are an +intersection (Finland and remote answered 0 on 2026-09-30), which is why the +runner asks for remote in a series of its own. """ from __future__ import annotations @@ -21,7 +33,8 @@ from urllib.parse import quote from desk.sources import strip_html -from ._contract import Ask, Fields, Place, Reads +from ._contract import Ask, Fields, Filters, Place, Reads +from ._countries import NAMES, code_of from ._shared import Get, Page, Row, only, parse_dt NAME = "oracle" @@ -41,7 +54,7 @@ PATTERNS = ( READS = Reads( scope="board", listing="paged", page_size=200, text="detail", teaser=True, - complete="stated", places="many", + complete="stated", places="many", filters=Filters(country="ids", remote="one"), fields=Fields(posted="trusted", remote="stated", country="stated"), robots=("read: robots.txt allows /hcmUI/CandidateExperience/en/sites//jobs " "(checked 2026-09-29)"), @@ -103,7 +116,7 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page: data = json.loads(get( f"{_api(slug)}/recruitingCEJobRequisitions?onlyData=true" f"&expand=requisitionList.secondaryLocations" - f"&finder=findReqs;siteNumber={site},limit={PAGE_SIZE},offset={offset}")) + f"&finder=findReqs;siteNumber={site},{_selected(ask)}limit={PAGE_SIZE},offset={offset}")) search = (data.get("items") or [{}])[0] roles = search.get("requisitionList") or [] # `_paged` counts `id`, which Oracle spells `Id`. @@ -111,6 +124,42 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page: return search, results, search.get("TotalJobsCount"), offset + len(roles) if roles else None +#: How many countries `locationsFacet` lists at most; a list this long may +#: leave some out. +FACET_LIMIT = 10 + + +def _selected(ask: Ask) -> str: + """The finder's facet parameters for `ask`, each ending in a comma: the + location ids (none when one is `*`, a country the pod did not list) and + remote.""" + out = "" + ids = [i for i in ask.countries if re.fullmatch(r"\d+", i)] + if ids and "*" not in ask.countries: + out += f"selectedLocationsFacet={'%3B'.join(ids)}," + if ask.remote: + out += "selectedWorkplaceTypesFacet=ORA_REMOTE," + return out + + +def facets(get: Get, slug: str) -> dict[str, dict[str, str]]: + """The pod's geography id of each country `locationsFacet` names, read + with the facets and no roles. When the list is `FACET_LIMIT` long, every + country it leaves out maps to `*`: its roles may be there, and a read + for it goes unfiltered.""" + site = _parts(slug)[1] + data = json.loads(get( + f"{_api(slug)}/recruitingCEJobRequisitions?onlyData=true" + f"&finder=findReqs;siteNumber={site},facetsList=LOCATIONS,limit=1,offset=0")) + listed = (data.get("items") or [{}])[0].get("locationsFacet") or [] + country = {code: str(entry["Id"]) for entry in listed + if isinstance(entry, dict) and entry.get("Id") + and (code := code_of(str(entry.get("Name") or "")))} + if len(listed) >= FACET_LIMIT: + country = dict.fromkeys(NAMES, "*") | country + return {"country": country} + + def parse(body: bytes, slug: str) -> list[Row]: return rows((json.loads(body).get("items") or [{}])[0], slug) diff --git a/plugins/readers/phenom.py b/plugins/readers/phenom.py index aaeb916..caa623c 100644 --- a/plugins/readers/phenom.py +++ b/plugins/readers/phenom.py @@ -9,6 +9,14 @@ answered without a cookie, an `Origin` header, a `pageId` or the site's `country`, both read off the slug's path) and the paging. Nothing is taken from the page's script. Written from live sites on 2026-09-29 (King, Just Eat Takeaway, Philips); jobhunt-hq did not read Phenom. + +Countries are filtered on the server as a list of English names in +`selected_fields.country`, all in one request (`Filters(country="many")`); the +list is a union, and a role at several places is kept when one of them +matches. Philips answered 111 of 873 for Netherlands and Germany on +2026-09-30. There is no remote field to filter on. A code with no English +name in `_countries` is left out of the list, and a list left empty reads the +whole board. """ from __future__ import annotations @@ -21,7 +29,8 @@ from typing import Any from desk.sources import strip_html -from ._contract import Ask, Fields, Place, Reads +from ._contract import Ask, Fields, Filters, Place, Reads +from ._countries import name_of from ._shared import Get, Page, Row, only, parse_dt NAME = "phenom" @@ -37,7 +46,8 @@ PATTERNS = ( READS = Reads( scope="board", listing="paged", page_size=100, text="detail", teaser=True, - complete="stated", places="many", fields=Fields(posted="trusted", remote="stated"), + complete="stated", places="many", filters=Filters(country="many"), + fields=Fields(posted="trusted", remote="stated"), robots="read: robots.txt allows ///search-results (checked 2026-09-29)", ) @@ -77,19 +87,21 @@ def careers_url(slug: str) -> str: return f"{_root(slug)}/search-results" -def _body(slug: str, offset: int) -> dict[str, Any]: +def _body(slug: str, offset: int, countries: tuple[str, ...] = ()) -> dict[str, Any]: """What the site's search page sends for a listing of every role, newest - first, less the parts that only tell the page what to draw. `lang` is - `en_global` for `/global/en` and `en_us` for `/us/en`, the site's own - spelling on the three sites read.""" + first, less the parts that only tell the page what to draw, in the + `countries` (ISO codes) given. `lang` is `en_global` for `/global/en` and + `en_us` for `/us/en`, the site's own spelling on the three sites read.""" _host, country, lang = _parts(slug) + names = [name for code in countries if (name := name_of(code))] return { "lang": f"{lang}_{country}", "deviceType": "desktop", "country": country, "pageName": "search-results", "ddoKey": "refineSearch", "sortBy": "", "subsearch": "", "from": offset, "jobs": True, "counts": True, "all_fetch": False, "pageType": "landingPage", "size": PAGE_SIZE, "clearAll": False, "jdsource": "facets", "isSliderEnable": False, "siteType": "external", - "location": "", "keywords": "", "global": True, "selected_fields": {}, + "location": "", "keywords": "", "global": True, + "selected_fields": {"country": names} if names else {}, "sort": {"order": "desc", "field": "postedDate"}, "locationData": {}, } @@ -106,7 +118,7 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page: offset = int(cursor or 0) host = _parts(slug)[0] data = json.loads(get(f"https://{host}/widgets", method="POST", - payload=_body(slug, offset))) + payload=_body(slug, offset, ask.countries))) search = data.get("refineSearch") if not isinstance(search, dict) or search.get("status", 200) != 200: raise ValueError("not a Phenom search answer: no refineSearch") diff --git a/plugins/readers/smartrecruiters.py b/plugins/readers/smartrecruiters.py index 021cc0f..b58907f 100644 --- a/plugins/readers/smartrecruiters.py +++ b/plugins/readers/smartrecruiters.py @@ -3,16 +3,22 @@ with a stated total. The identifier is case-sensitive (`BoschGroup`). Written from one real board on 2026-09-23; jobhunt-hq did not read SmartRecruiters. + +A country is filtered on the server, `?country=dk`, one a request +(`Filters(country="one")`): Bosch's DE, SE and DK summed to what the app held +(review, 2026-09-29), and DK answered 2 of 4,830 on 2026-09-30. The API +ignores a remote parameter, so remote is not filtered. """ from __future__ import annotations import json import logging +import re from collections.abc import Callable from typing import Any -from ._contract import Ask, Fields, Place, Reads +from ._contract import Ask, Fields, Filters, Place, Reads from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok log = logging.getLogger(__name__) @@ -27,7 +33,7 @@ PATTERNS = ( #: An unknown identifier answers an empty list, which is `empty="suspect"`. READS = Reads( scope="board", listing="paged", page_size=100, text="none", complete="stated", - places="one", empty="suspect", + places="one", empty="suspect", filters=Filters(country="one"), fields=Fields(posted="trusted", remote="stated", country="stated", seniority="stated", employment="stated"), robots="api: no robots.txt check needed (public board API), 2026-09-30", @@ -73,9 +79,13 @@ def probe_ok(body: bytes) -> bool: def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page: + """One page, in the one country `ask` names, which the API takes in lower + case.""" offset = int(cursor or 0) + country = ask.countries[0].lower() if ask.countries else "" + where = f"&country={country}" if re.fullmatch(r"[a-z]{2}", country) else "" data = json.loads(get(f"https://{_API}/v1/companies/{slug}/postings" - f"?offset={offset}&limit={PAGE_SIZE}")) + f"?offset={offset}&limit={PAGE_SIZE}{where}")) content = data.get("content") or [] return data, content, data.get("totalFound"), offset + len(content) if content else None diff --git a/plugins/readers/successfactors.py b/plugins/readers/successfactors.py index c9feb57..0124020 100644 --- a/plugins/readers/successfactors.py +++ b/plugins/readers/successfactors.py @@ -30,8 +30,13 @@ module reads, and neither is a translation of a listed role. A path prefix names a brand of a shared site. `//search/` lists only that brand on the server (Universal Robots 30 of Teradyne's 486, MiR 1), so the read is as small as the company; `rows` still keeps only the roles -under `//job/`, in case a site does not scope. There is no server-side -country filter in this version. +under `//job/`, in case a site does not scope. + +A country is filtered on the server by the search's own facet, +`optionsFacetsDD_country=DK`, one country a request (`Filters(country="one")`); +it worked on the four tenants tried in the review and on Vestas (64 of 668, +2026-09-30). A role at several places is listed when one of them is in the +country. There is no remote facet. A role's own page carries microdata (`itemtype` schema.org JobPosting) with `datePosted` and the address; `detail` reads the description, the date and the @@ -52,7 +57,8 @@ from urllib.parse import urljoin, urlsplit from desk.sources import strip_html -from ._contract import Ask, Fields, Place, Reads +from ._contract import Ask, Fields, Filters, Place, Reads +from ._countries import NAMES from ._shared import Get, Page, Row, is_web_url, only, parse_dt NAME = "successfactors" @@ -79,6 +85,7 @@ KEEP_CASE = True READS = Reads( scope="board", listing="paged", text="detail", complete="stated", places="many", + filters=Filters(country="one"), fields=Fields(posted="trusted", country="stated"), robots="read: robots.txt allows /search/ and /job/ (checked 2026-09-29)", ) @@ -122,52 +129,10 @@ _COUNTRY_LABELS = ("country", "land") _MONTHS = {name: number for number, name in enumerate( ("jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"), 1)} -#: ISO 3166-1 alpha-2 to the English name, which is what `country_of` reads: -#: a bare "DE" or "SE" in a location line is taken for Delaware or nothing. A -#: code that is not here stays as the site wrote it. -_COUNTRIES = dict(entry.strip().split(" ", 1) for entry in """ -AD Andorra | AE United Arab Emirates | AF Afghanistan | AG Antigua and Barbuda | AI -Anguilla | AL Albania | AM Armenia | AO Angola | AR Argentina | AS American Samoa | AT -Austria | AU Australia | AW Aruba | AX Åland Islands | AZ Azerbaijan | BA Bosnia and -Herzegovina | BB Barbados | BD Bangladesh | BE Belgium | BF Burkina Faso | BG Bulgaria -| BH Bahrain | BI Burundi | BJ Benin | BM Bermuda | BN Brunei | BO Bolivia | BQ -Caribbean Netherlands | BR Brazil | BS Bahamas | BT Bhutan | BW Botswana | BY Belarus -| BZ Belize | CA Canada | CD Democratic Republic of the Congo | CF Central African -Republic | CG Republic of the Congo | CH Switzerland | CI Côte d'Ivoire | CK Cook -Islands | CL Chile | CM Cameroon | CN China | CO Colombia | CR Costa Rica | CU Cuba | -CV Cabo Verde | CW Curaçao | CY Cyprus | CZ Czechia | DE Germany | DJ Djibouti | DK -Denmark | DM Dominica | DO Dominican Republic | DZ Algeria | EC Ecuador | EE Estonia | -EG Egypt | ER Eritrea | ES Spain | ET Ethiopia | FI Finland | FJ Fiji | FK Falkland -Islands | FM Micronesia | FO Faroe Islands | FR France | GA Gabon | GB United Kingdom -| GD Grenada | GE Georgia | GF French Guiana | GG Guernsey | GH Ghana | GI Gibraltar | -GL Greenland | GM Gambia | GN Guinea | GP Guadeloupe | GQ Equatorial Guinea | GR -Greece | GT Guatemala | GU Guam | GW Guinea-Bissau | GY Guyana | HK Hong Kong | HN -Honduras | HR Croatia | HT Haiti | HU Hungary | ID Indonesia | IE Ireland | IL Israel -| IM Isle of Man | IN India | IQ Iraq | IR Iran | IS Iceland | IT Italy | JE Jersey | -JM Jamaica | JO Jordan | JP Japan | KE Kenya | KG Kyrgyzstan | KH Cambodia | KI -Kiribati | KM Comoros | KN Saint Kitts and Nevis | KP North Korea | KR South Korea | -KW Kuwait | KY Cayman Islands | KZ Kazakhstan | LA Laos | LB Lebanon | LC Saint Lucia -| LI Liechtenstein | LK Sri Lanka | LR Liberia | LS Lesotho | LT Lithuania | LU -Luxembourg | LV Latvia | LY Libya | MA Morocco | MC Monaco | MD Moldova | ME -Montenegro | MG Madagascar | MH Marshall Islands | MK North Macedonia | ML Mali | MM -Myanmar | MN Mongolia | MO Macao | MP Northern Mariana Islands | MQ Martinique | MR -Mauritania | MS Montserrat | MT Malta | MU Mauritius | MV Maldives | MW Malawi | MX -Mexico | MY Malaysia | MZ Mozambique | NA Namibia | NC New Caledonia | NE Niger | NG -Nigeria | NI Nicaragua | NL Netherlands | NO Norway | NP Nepal | NR Nauru | NU Niue | -NZ New Zealand | OM Oman | PA Panama | PE Peru | PF French Polynesia | PG Papua New -Guinea | PH Philippines | PK Pakistan | PL Poland | PR Puerto Rico | PS Palestine | PT -Portugal | PW Palau | PY Paraguay | QA Qatar | RE Réunion | RO Romania | RS Serbia | -RU Russia | RW Rwanda | SA Saudi Arabia | SB Solomon Islands | SC Seychelles | SD -Sudan | SE Sweden | SG Singapore | SI Slovenia | SK Slovakia | SL Sierra Leone | SM -San Marino | SN Senegal | SO Somalia | SR Suriname | SS South Sudan | ST São Tomé and -Príncipe | SV El Salvador | SY Syria | SZ Eswatini | TC Turks and Caicos Islands | TD -Chad | TG Togo | TH Thailand | TJ Tajikistan | TL Timor-Leste | TM Turkmenistan | TN -Tunisia | TO Tonga | TR Turkey | TT Trinidad and Tobago | TV Tuvalu | TW Taiwan | TZ -Tanzania | UA Ukraine | UG Uganda | US United States | UY Uruguay | UZ Uzbekistan | VA -Vatican City | VC Saint Vincent and the Grenadines | VE Venezuela | VG British Virgin -Islands | VI U.S. Virgin Islands | VN Vietnam | VU Vanuatu | WS Samoa | XK Kosovo | YE -Yemen | ZA South Africa | ZM Zambia | ZW Zimbabwe -""".replace("\n", " ").split(" | ")) +#: ISO 3166-1 alpha-2 to the English name (`_countries.NAMES`): a bare "DE" +#: or "SE" in a location line is taken for Delaware or nothing. A code that +#: is not there stays as the site wrote it. +_COUNTRIES = NAMES def _parts(slug: str) -> tuple[str, str]: @@ -190,9 +155,12 @@ def careers_url(slug: str) -> str: return f"https://{host}{prefix}/search/" -def _search_url(slug: str, start: int) -> str: +def _search_url(slug: str, start: int, country: str = "") -> str: + """The search page from `start`, in one country when `country` (an ISO + code) is given.""" host, prefix = _parts(slug) - return f"https://{host}{prefix}/search/?q=&locationsearch=&startrow={start}" + facet = f"&optionsFacetsDD_country={country}" if re.fullmatch(r"[A-Z]{2}", country) else "" + return f"https://{host}{prefix}/search/?q=&locationsearch={facet}&startrow={start}" # ------------------------------------------------------------------ listing @@ -296,12 +264,13 @@ def _entries(text: str) -> list[dict[str, Any]]: def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page: - """One search page. The cursor is `(start row, first role of the page + """One search page, in the one country `ask` names. The cursor is `(start row, first role of the page before)`: the next start row is what this page returned, and a page that starts with the same role as the last one is the site ignoring `startrow`, which ends the read (incomplete) instead of looping.""" start, previous = cursor or (0, None) - text = get(_search_url(slug, start)).decode("utf-8", "replace") + country = ask.countries[0] if ask.countries else "" + text = get(_search_url(slug, start, country)).decode("utf-8", "replace") if not _SEARCH_PAGE.search(text): raise ValueError("not a SuccessFactors search page") found = _entries(text) diff --git a/plugins/readers/workday.py b/plugins/readers/workday.py index b6f8738..fb9b25e 100644 --- a/plugins/readers/workday.py +++ b/plugins/readers/workday.py @@ -12,6 +12,17 @@ keeps the case the URL wrote (`KEEP_CASE`). Written from live career sites on 2026-09-29 (`docs/plan-hosts/workday.md`). The listing carries no description, so `detail` reads each role's own JSON. + +Countries and remote are filtered on the server by the tenant's own facets +(`Filters(country="ids", remote="ids")`): `appliedFacets` takes +`{: [, ...]}`, ids within one facet are a union, and two +facets are an intersection, which is why the runner asks for remote in a +series of its own. Each tenant names its facets its own way (LEGO's +`Location` lists countries, others nest `locationCountry` under +`locationMainGroup`), so `facets` reads them off the first page and picks the +one whose values are countries. A host id is `=`. LEGO +answered 97 of 455 for Denmark on 2026-09-30; a role at several places is +kept when one of them is in the country. """ from __future__ import annotations @@ -26,7 +37,8 @@ from typing import Any from desk.sources import strip_html -from ._contract import Ask, Fields, Place, Reads +from ._contract import Ask, Fields, Filters, Place, Reads +from ._countries import code_of from ._shared import Get, Page, Row, only, parse_dt log = logging.getLogger(__name__) @@ -37,7 +49,7 @@ KEEP_CASE = True READS = Reads( scope="board", listing="paged", page_size=20, ceiling=2000, text="detail", - complete="stated", places="many", + complete="stated", places="many", filters=Filters(country="ids", remote="ids"), fields=Fields(posted="trusted", remote="stated", country="stated", employment="stated"), robots="read: robots.txt allows // (checked 2026-09-29)", ) @@ -104,7 +116,8 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page: offset, total = cursor or (0, None) data = json.loads(get( f"https://{_host(slug)}/wday/cxs/{tenant}/{site}/jobs", method="POST", - payload={"appliedFacets": {}, "limit": PAGE_SIZE, "offset": offset, "searchText": ""})) + payload={"appliedFacets": _applied(ask), "limit": PAGE_SIZE, "offset": offset, + "searchText": ""})) if cursor is None: stated = data.get("total") total = stated if isinstance(stated, int) and not isinstance(stated, bool) else None @@ -118,6 +131,69 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page: return data, results, total, (following, total) if more else None +def _applied(ask: Ask) -> dict[str, list[str]]: + """`appliedFacets` for the host ids `ask` carries, `=` + each, grouped by facet.""" + out: dict[str, list[str]] = {} + for value in (*ask.countries, *ask.remote_ids): + facet, sep, value_id = value.partition("=") + if sep and facet and value_id: + out.setdefault(facet, []).append(value_id) + return out + + +#: A facet whose values are countries: at least this share of them name one. +_COUNTRY_SHARE = 0.8 + + +def _facet_lists(facets: Any) -> list[tuple[str, list[dict[str, Any]]]]: + """`(facetParameter, values)` of every facet of an answer, the facets nested + in a group (`locationMainGroup` holds `locationCountry` on some tenants) + included.""" + out: list[tuple[str, list[dict[str, Any]]]] = [] + for facet in facets if isinstance(facets, list) else []: + if not isinstance(facet, dict): + continue + values = [v for v in facet.get("values") or [] if isinstance(v, dict)] + nested = [v for v in values if "values" in v and "facetParameter" in v] + out += _facet_lists(nested) + plain = [v for v in values if v not in nested] + if plain and str(facet.get("facetParameter") or ""): + out.append((str(facet["facetParameter"]), plain)) + return out + + +def facets(get: Get, slug: str) -> dict[str, dict[str, str]]: + """The tenant's host ids, read off one page of one role: `country` maps + an ISO code to `=` in the facet whose values are + countries (most of them, `_COUNTRY_SHARE`; the one with most countries + when two are), and `remote` maps each facet value that `_REMOTE` reads as + remote ("Remote", "Fully Flexible"), in whatever facet, to its id. A + tenant with neither states no such dimension, and the runner then does + not filter on it.""" + tenant, _dc, site = _parts(slug) # type: ignore[misc] + data = json.loads(get( + f"https://{_host(slug)}/wday/cxs/{tenant}/{site}/jobs", method="POST", + payload={"appliedFacets": {}, "limit": 1, "offset": 0, "searchText": ""})) + country: dict[str, str] = {} + remote: dict[str, str] = {} + for name, values in _facet_lists(data.get("facets")): + found = {code: f"{name}={v['id']}" for v in values + if v.get("id") and (code := code_of(str(v.get("descriptor") or "")))} + if len(found) >= _COUNTRY_SHARE * len(values) and len(found) > len(country): + country = found + for v in values: + word = " ".join(str(v.get("descriptor") or "").split()) + if v.get("id") and _remote(word) == "remote": + remote[word.lower()] = f"{name}={v['id']}" + out: dict[str, dict[str, str]] = {} + if country: + out["country"] = country + if remote: + out["remote"] = remote + return out + + def _posting_id(posting: dict[str, Any], position: int) -> str: return str(posting.get("externalPath") or "".join(posting.get("bulletFields") or []) or f"#{position}") diff --git a/tests/fixtures/ats/oracle-facets.json b/tests/fixtures/ats/oracle-facets.json new file mode 100644 index 0000000..04532dc --- /dev/null +++ b/tests/fixtures/ats/oracle-facets.json @@ -0,0 +1,87 @@ +{ + "items": [ + { + "SearchId": 1, + "Facets": "LOCATIONS;WORKPLACE_TYPES", + "Offset": 0, + "Limit": 1, + "TotalJobsCount": 550, + "SiteNumber": "CX_1", + "locationsFacet": [ + { + "Id": 300000000480126, + "Name": "United States", + "TotalCount": 215 + }, + { + "Id": 300000000471745, + "Name": "India", + "TotalCount": 123 + }, + { + "Id": 300000000471544, + "Name": "Canada", + "TotalCount": 79 + }, + { + "Id": 300000000471982, + "Name": "Portugal", + "TotalCount": 38 + }, + { + "Id": 300000000471904, + "Name": "Malaysia", + "TotalCount": 31 + }, + { + "Id": 300000000471640, + "Name": "Finland", + "TotalCount": 23 + }, + { + "Id": 300000000471967, + "Name": "Poland", + "TotalCount": 21 + }, + { + "Id": 300000000480081, + "Name": "Thailand", + "TotalCount": 17 + }, + { + "Id": 300000000471760, + "Name": "Italy", + "TotalCount": 12 + }, + { + "Id": 300000000471601, + "Name": "Germany", + "TotalCount": 10 + } + ], + "workplaceTypesFacet": [ + { + "Id": "ORA_HYBRID", + "Name": "Hybrid", + "TotalCount": 317 + }, + { + "Id": "ORA_ON_SITE", + "Name": "Onsite", + "TotalCount": 230 + }, + { + "Id": "ORA_REMOTE", + "Name": "Remote", + "TotalCount": 3 + } + ], + "categoriesFacet": [], + "requisitionList": [] + } + ], + "count": 1, + "hasMore": false, + "limit": 25, + "offset": 0 +} \ No newline at end of file diff --git a/tests/fixtures/ats/workday-facets.json b/tests/fixtures/ats/workday-facets.json new file mode 100644 index 0000000..91e188b --- /dev/null +++ b/tests/fixtures/ats/workday-facets.json @@ -0,0 +1,109 @@ +{ + "total": 455, + "jobPostings": [], + "facets": [ + { + "facetParameter": "Location", + "descriptor": "Location", + "values": [ + { + "descriptor": "United States of America", + "id": "bc33aa3152ec42d4995f4791a106ed09", + "count": 173 + }, + { + "descriptor": "Denmark", + "id": "49ab063f422741e2aef271de00efeac8", + "count": 97 + }, + { + "descriptor": "China", + "id": "6cb77610a8a543aea2d6bc10457e35d4", + "count": 30 + }, + { + "descriptor": "United Kingdom", + "id": "29247e57dbaf46fb855b224e03170bc7", + "count": 29 + }, + { + "descriptor": "Czechia", + "id": "fc078443155c4ad294201ecf5a61a499", + "count": 25 + }, + { + "descriptor": "Hungary", + "id": "9db257f5937e4421b2fac64eec6832f8", + "count": 21 + }, + { + "descriptor": "Germany", + "id": "dcc5b7608d8644b3a93716604e78e995", + "count": 18 + }, + { + "descriptor": "Vietnam", + "id": "db69e8c8446c11de98360015c5e6daf6", + "count": 17 + } + ] + }, + { + "facetParameter": "timeType", + "descriptor": "Time Type", + "values": [ + { + "descriptor": "Full time", + "id": "07564020a44510063cb1bcb298e20001", + "count": 270 + }, + { + "descriptor": "Part time", + "id": "07564020a44510063cb1bcb298e20000", + "count": 162 + } + ] + }, + { + "facetParameter": "locationMainGroup", + "values": [ + { + "facetParameter": "locationRegionStateProvince", + "descriptor": "Region/State/Province", + "values": [ + { + "descriptor": "Alabama", + "id": "31475924e5494080a8a458bf4fa293ed", + "count": 2 + }, + { + "descriptor": "Alberta", + "id": "588eb24f950e4c5b93d5ad3f37aebad2", + "count": 3 + }, + { + "descriptor": "Arizona", + "id": "c7b20b0d4bc04711a00900569e9afabd", + "count": 4 + }, + { + "descriptor": "Arkansas", + "id": "cea6c5355e1b4983b5fd0640310385b5", + "count": 1 + }, + { + "descriptor": "Bavaria", + "id": "bd89fff5ffa6440a9abeaf2c5c1dc37c", + "count": 1 + }, + { + "descriptor": "Beijing", + "id": "4b194fc4ae3b4cbe95a1b379454e1775", + "count": 1 + } + ] + } + ] + } + ] +} \ No newline at end of file diff --git a/tests/test_ats_eightfold.py b/tests/test_ats_eightfold.py index ed5e5f0..ad70888 100644 --- a/tests/test_ats_eightfold.py +++ b/tests/test_ats_eightfold.py @@ -204,10 +204,10 @@ def test_discovery_reads_the_host_and_domain_from_a_link_that_has_both() -> None assert "eightfold" not in {h for h, _ in discover._UNREADABLE if h not in readers.HOSTS} -def test_it_declares_a_paged_read_of_ten_with_a_total_and_no_filter_yet() -> None: +def test_it_declares_a_paged_read_of_ten_with_a_total_and_one_country_a_request() -> None: reads = eightfold.READS assert (reads.listing, reads.page_size, reads.text, reads.complete, reads.places) == ( "paged", 10, "detail", "stated", "many") - assert reads.filters.declared() == {} and reads.fields.posted == "listed" + assert reads.filters.declared() == {"country": "one"} and reads.fields.posted == "listed" assert reads.robots.startswith("read: robots.txt allows /careers") assert not hasattr(eightfold, "robots_page") diff --git a/tests/test_reach.py b/tests/test_reach.py new file mode 100644 index 0000000..2b99a09 --- /dev/null +++ b/tests/test_reach.py @@ -0,0 +1,212 @@ +"""The reach of a pull (`app.jobs.reach`): from the enabled agents' gates and +favourites to each board's `reach`, and through the pull's batches to the +runner's `Ask`. No network: the runner's listing is stubbed.""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import ClassVar + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from pipeline import Tally # noqa: E402 +from app.jobs import pull, reach # noqa: E402 +from app.models import Gate # noqa: E402 +from app.storage import Store # noqa: E402 +from plugins.readers import Ask, _runner # noqa: E402 + +EUROPE = ["AT", "BE", "BG", "HR", "CY", "CZ", "DK", "EE", "FI", "FR", "DE", "GR", "HU", "IE", + "IT", "LV", "LT", "LU", "MT", "NL", "PL", "PT", "RO", "SK", "SI", "ES", "SE", "IS", + "LI", "NO", "GB", "CH", "EU"] + + +class FakeConfig: + sources: ClassVar[dict[str, object]] = {} + searches: ClassVar[list[object]] = [] + + +@pytest.fixture +def store(tmp_path: Path) -> Store: + return Store(tmp_path / "roledesk.db") + + +def _wolf(store: Store, **extra: object) -> int: + """The live agent 2 of 2026-09-30: Europe required, Denmark, Sweden, remote + or a town in Skåne or Denmark.""" + gates = [Gate(field="country", values=EUROPE, mode="require"), + Gate(field="country", values=["DK", "SE"], mode="any"), + Gate(field="remote", values=["remote"], mode="any"), + Gate(field="place", values=["Malmö", "Lund", "copenhagen", "odense"], mode="any"), + Gate(field="remote", values=["hybrid"], mode="prefer", weight=0.05)] + return store.save_agent("Wolf", gates=gates, **extra).id # type: ignore[arg-type] + + +def _board(store: Store, target: str, company: str, size: int | None = 1000, + params: dict[str, object] | None = None) -> None: + store.add_subscription("ats", target, label=company, company=company, + board=target.partition(":")[0], params=params or {}) + store.add_company(company) + if size is not None: + store.state[f"ats:size:{target}"] = str(size) + + +# ------------------------------------------------------------------ gates + +def test_the_base_reach_is_the_union_of_the_any_gates_with_towns_as_countries( + store: Store) -> None: + _wolf(store) + store.save_agent("Berlin", gates=[Gate(field="place", values=["Berlin"], mode="any")]) + plan = reach.plan(store) + assert plan.base == reach.Reach(frozenset({"DK", "SE", "DE"}), remote=True) + assert plan.base.params() == {"countries": ["DE", "DK", "SE"], "remote": True} + + +@pytest.mark.parametrize("gates", [ + [], + [Gate(field="country", values=["DK"], mode="any"), + Gate(field="function", values=["engineering"], mode="any")], + [Gate(field="remote", values=["hybrid"], mode="any")], + [Gate(field="place", values=["Nowhereville"], mode="any")], +], ids=["no any gates", "a function in the group", "hybrid anywhere", "an unknown town"]) +def test_an_agent_that_can_take_a_role_from_anywhere_makes_the_reach_everywhere( + store: Store, gates: list[Gate]) -> None: + _wolf(store) + store.save_agent("Other", gates=gates) + assert reach.plan(store).base == reach.EVERYWHERE + + +def test_an_agent_switched_off_does_not_widen_the_reach(store: Store) -> None: + _wolf(store) + store.save_agent("Off", gates=[], enabled=False) + assert reach.plan(store).base.countries == {"DK", "SE"} + + +def test_with_no_agent_switched_on_every_board_is_read_whole(store: Store) -> None: + _wolf(store, enabled=False) + assert reach.plan(store).base == reach.EVERYWHERE + + +def test_the_require_reach_intersects_and_expands_europe(store: Store) -> None: + gaz = store.gazetteer() + agent = store.agent(_wolf(store)) + assert agent is not None + europe = reach.require_reach(agent, gaz) + assert {"DK", "SE", "DE", "FR", "GB", "NO", "CH"} <= europe.countries + assert "US" not in europe.countries and "EU" not in europe.countries + narrower = agent.model_copy(update={"gates": [ + *agent.gates, Gate(field="country", values=["DK", "US"], mode="require")]}) + assert reach.require_reach(narrower, gaz) == reach.Reach(frozenset({"DK"})) + plain = agent.model_copy(update={"gates": []}) + assert reach.require_reach(plain, gaz) == reach.EVERYWHERE + + +# ------------------------------------------------------------- per board + +def test_a_favourite_is_read_across_the_require_reach(store: Store) -> None: + wolf = _wolf(store) + _board(store, "smartrecruiters:mistral", "Mistral AI") + _board(store, "smartrecruiters:acme", "Acme") + store.set_affinity(wolf, "mistral-ai", 2) + plan = reach.plan(store) + subs = {s.target: s for s in store.subscriptions(source="ats")} + favourite, why = plan.why(subs["smartrecruiters:mistral"]) + assert why == "favourite" and {"DE", "FR", "DK"} <= favourite.countries + assert plan.why(subs["smartrecruiters:acme"]) == (plan.base, "agents") + + +def test_a_company_starred_by_one_agent_keeps_the_others_reach(store: Store) -> None: + wolf = _wolf(store) + store.save_agent("Anywhere", gates=[Gate(field="country", values=["US"], mode="any")]) + _board(store, "smartrecruiters:mistral", "Mistral AI") + store.set_affinity(wolf, "mistral-ai", 2) + favourite = reach.plan(store).favourites["mistral-ai"] + assert "US" in favourite.countries and "FR" in favourite.countries + + +@pytest.mark.parametrize("size, params, why", [ + (299, {}, "small"), + (None, {}, "unknown"), + (5000, {"reach": "all"}, "override"), + (5000, {"countries": ["DE"]}, "override"), + (300, {}, "agents"), +]) +def test_small_unknown_and_overridden_boards_get_no_reach( + store: Store, size: int | None, params: dict[str, object], why: str) -> None: + _wolf(store) + _board(store, "workday:acme.wd3/Jobs", "Acme", size, params) + [sub] = store.subscriptions(source="ats") + found, said = reach.plan(store).why(sub) + assert said == why + assert (found.params() is None) == (why != "agents") + + +def test_a_board_read_before_sizes_were_kept_is_sized_by_its_open_roles(store: Store) -> None: + _board(store, "workday:acme.wd3/Jobs", "Acme", size=None) + [sub] = store.subscriptions(source="ats") + store.state["ats:ok:workday:acme.wd3/Jobs"] = "run-0" + assert reach.sizes(store, [sub]) == {"workday:acme.wd3/Jobs": 0} + # A filtered board's open roles are not its size. + store.state["ats:filtered:workday:acme.wd3/Jobs"] = "1" + assert reach.sizes(store, [sub]) == {} + + +def test_describe_is_short() -> None: + assert reach.Reach(frozenset({"SE", "DK"}), True).describe() == "DK, SE, remote" + assert reach.Reach(frozenset(EUROPE[:10])).describe() == "10 countries" + assert reach.EVERYWHERE.describe() == "everywhere" + + +# ---------------------------------------------------------------- the pull + +def _row(job_id: str) -> dict[str, object]: + return {"source_id": job_id, "title": f"Engineer {job_id}", "url": f"https://x.example/{job_id}", + "places": [{"text": "Copenhagen", "remote": None}], "posted": None, + "description": "", "remote": None, "salary_text": ""} + + +def _pull(store: Store) -> list[dict[str, object]]: + batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type] + results = [pull.run_batch(FakeConfig(), store, **batch) # type: ignore[arg-type] + for batch in batches] + pull.close_boards(FakeConfig(), store, Tally(results=tuple(results))) # type: ignore[arg-type] + return batches + + +def test_the_pull_hands_each_board_its_reach_and_the_runner_narrows_it( + store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + _wolf(store) + _board(store, "smartrecruiters:BoschGroup", "Bosch") + _board(store, "smartrecruiters:tiny", "Tiny", size=12) + asked: dict[str, Ask] = {} + + def fake(module: object, slug: str, get: object, ask: Ask, **kwargs: object + ) -> _runner.Listing: + asked[slug] = ask + return _runner.Listing([_row("1")], True, ["1"], _runner.filtered(ask)) # type: ignore[list-item] + + monkeypatch.setattr(_runner, "read_listing", fake) + [batch] = _pull(store) + assert batch["reach"] == {"smartrecruiters:BoschGroup": {"countries": ["DK", "SE"], + "remote": True}} + # SmartRecruiters filters countries one a request and cannot ask for remote. + assert asked["BoschGroup"] == Ask(countries=("DK", "SE")) + assert asked["tiny"] == Ask() + assert store.state["ats:filtered:smartrecruiters:BoschGroup"] == "1" + # The small board, read whole, keeps its size; the filtered one keeps its last. + assert store.state["ats:size:smartrecruiters:tiny"] == "1" + assert store.state["ats:size:smartrecruiters:BoschGroup"] == "1000" + + +def test_a_batch_queued_before_the_reach_existed_still_runs( + store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + _board(store, "smartrecruiters:acme", "Acme") + monkeypatch.setattr(_runner, "read_listing", lambda *a, **k: _runner.Listing( + [_row("1")], True, ["1"])) # type: ignore[list-item] + result = pull.run_batch(FakeConfig(), store, # type: ignore[arg-type] + targets=["smartrecruiters:acme"], run="r") + assert result["roles"] == 1 diff --git a/tests/test_readers_filters.py b/tests/test_readers_filters.py new file mode 100644 index 0000000..a33cca1 --- /dev/null +++ b/tests/test_readers_filters.py @@ -0,0 +1,268 @@ +"""Filtering at the source: what the six hosts that filter by country (and +Workday and Oracle by remote) send for an `Ask`, their `facets`, and the +runner's series for a reach of countries and remote. Against saved answers; +no network.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any +from urllib.parse import parse_qs, unquote, urlsplit + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from plugins.readers import ( # noqa: E402 + Ask, _countries, _runner, eightfold, oracle, phenom, smartrecruiters, successfactors, + workday, +) + +FIXTURES = ROOT / "tests" / "fixtures" / "ats" + + +class Recorder: + """A `get` that answers `body` to every request and keeps what was asked.""" + + def __init__(self, body: bytes) -> None: + self.body = body + self.asked: list[tuple[str, dict[str, Any] | None]] = [] + + def __call__(self, url: str, *, method: str = "GET", payload: dict[str, Any] | None = None, + headers: dict[str, str] | None = None) -> bytes: + self.asked.append((url, payload)) + return self.body + + +def _query(url: str) -> dict[str, list[str]]: + return parse_qs(urlsplit(url).query) + + +# ------------------------------------------------------------ declarations + +def test_the_six_hosts_declare_what_their_search_filters() -> None: + declared = {module.NAME: module.READS.filters.declared() for module in + (smartrecruiters, eightfold, workday, successfactors, phenom, oracle)} + assert declared == { + "smartrecruiters": {"country": "one"}, + "eightfold": {"country": "one"}, + "workday": {"country": "ids", "remote": "ids"}, + "successfactors": {"country": "one"}, + "phenom": {"country": "many"}, + "oracle": {"country": "ids", "remote": "one"}, + } + + +def test_country_names_read_both_ways_with_the_hosts_own_spellings() -> None: + assert _countries.name_of("dk") == "Denmark" and _countries.name_of("ZZ") == "" + assert _countries.code_of("United States of America") == "US" + assert _countries.code_of("Korea, Republic of") == "KR" + assert _countries.code_of(" czech republic ") == "CZ" + assert _countries.code_of("Türkiye") == "TR" and _countries.code_of("Turkey") == "TR" + assert _countries.code_of("Atlantis") == "" + assert successfactors._COUNTRIES["SE"] == "Sweden" + + +# -------------------------------------------------------- one a request + +def test_smartrecruiters_asks_one_country_in_lower_case() -> None: + get = Recorder(b'{"content": [], "totalFound": 0}') + smartrecruiters.page(get, "BoschGroup", None, Ask(countries=("DK",))) + smartrecruiters.page(get, "BoschGroup", 100, Ask()) + assert _query(get.asked[0][0])["country"] == ["dk"] + assert "country" not in _query(get.asked[1][0]) + + +def test_eightfold_asks_one_country_by_its_english_name() -> None: + get = Recorder(b'{"status": 200, "data": {"positions": [], "count": 0}}') + eightfold.page(get, "jobs.ericsson.com/ericsson.com", None, Ask(countries=("SE",))) + eightfold.page(get, "jobs.ericsson.com/ericsson.com", None, Ask(countries=("GB",))) + eightfold.page(get, "jobs.ericsson.com/ericsson.com", None, Ask(countries=("ZZ",))) + assert _query(get.asked[0][0])["location"] == ["Sweden"] + assert _query(get.asked[1][0])["location"] == ["United Kingdom"] + # A code with no name is read unfiltered: a superset, never too little. + assert "location" not in _query(get.asked[2][0]) + + +def test_successfactors_asks_one_country_by_its_facet() -> None: + body = (FIXTURES / "successfactors-table.html").read_bytes() + get = Recorder(body) + successfactors.page(get, "careers.vestas.com", None, Ask(countries=("DK",))) + successfactors.page(get, "jobs.teradyne.com/MiR", None, Ask(countries=("DE",))) + successfactors.page(get, "careers.vestas.com", None, Ask()) + assert get.asked[0][0] == ("https://careers.vestas.com/search/?q=&locationsearch=" + "&optionsFacetsDD_country=DK&startrow=0") + assert get.asked[1][0].startswith("https://jobs.teradyne.com/MiR/search/?") + assert _query(get.asked[1][0])["optionsFacetsDD_country"] == ["DE"] + assert "optionsFacetsDD_country" not in get.asked[2][0] + + +# ------------------------------------------------------------ a list + +def test_phenom_asks_every_country_by_name_in_one_request() -> None: + get = Recorder((FIXTURES / "phenom.json").read_bytes()) + phenom.page(get, "www.careers.philips.com/global/en", None, + Ask(countries=("NL", "DE", "ZZ"))) + phenom.page(get, "www.careers.philips.com/global/en", None, Ask()) + phenom.page(get, "www.careers.philips.com/global/en", None, Ask(countries=("ZZ",))) + assert get.asked[0][1]["selected_fields"] == {"country": ["Netherlands", "Germany"]} + assert get.asked[1][1]["selected_fields"] == {} + assert get.asked[2][1]["selected_fields"] == {} + + +# ------------------------------------------------------------ host ids + +LEGO = "lego.wd103/LEGO_External" + + +def _workday_facets(**extra: Any) -> dict[str, Any]: + saved = json.loads((FIXTURES / "workday-facets.json").read_text()) + saved["facets"] += extra.get("facets", []) + return saved + + +def test_workday_facets_find_the_facet_of_countries_and_the_remote_values() -> None: + remote = {"facetParameter": "remoteType", "descriptor": "Remote Type", "values": [ + {"descriptor": "Fully Flexible", "id": "ff1", "count": 30}, + {"descriptor": "Hybrid", "id": "hy1", "count": 20}, + {"descriptor": "Remote", "id": "rm1", "count": 3}]} + get = Recorder(json.dumps(_workday_facets(facets=[remote])).encode()) + found = workday.facets(get, LEGO) + assert found["country"]["DK"] == "Location=49ab063f422741e2aef271de00efeac8" + assert found["country"]["US"] == "Location=bc33aa3152ec42d4995f4791a106ed09" + # The regions nested under `locationMainGroup` and the time types are + # not countries. + assert all(v.startswith("Location=") for v in found["country"].values()) + assert found["remote"] == {"fully flexible": "remoteType=ff1", "remote": "remoteType=rm1"} + url, payload = get.asked[0] + assert url == "https://lego.wd103.myworkdayjobs.com/wday/cxs/lego/LEGO_External/jobs" + assert payload == {"appliedFacets": {}, "limit": 1, "offset": 0, "searchText": ""} + + +def test_workday_facets_of_a_tenant_without_remote_state_no_remote() -> None: + found = workday.facets(Recorder(json.dumps(_workday_facets()).encode()), LEGO) + assert set(found) == {"country"} + + +def test_workday_applies_the_host_ids_grouped_by_facet() -> None: + get = Recorder(b'{"total": 0, "jobPostings": []}') + workday.page(get, LEGO, None, Ask(countries=("Location=a", "Location=b"))) + workday.page(get, LEGO, None, Ask(remote=True, remote_ids=("remoteType=r", "Location=x"))) + workday.page(get, LEGO, None, Ask()) + assert get.asked[0][1]["appliedFacets"] == {"Location": ["a", "b"]} + assert get.asked[1][1]["appliedFacets"] == {"remoteType": ["r"], "Location": ["x"]} + assert get.asked[2][1]["appliedFacets"] == {} + + +POD = "fa-evmr-saasfaprod1.fa.ocs.oraclecloud.com/CX_1" + + +def test_oracle_facets_map_the_listed_countries_and_mark_the_rest_unknown() -> None: + get = Recorder((FIXTURES / "oracle-facets.json").read_bytes()) + found = oracle.facets(get, POD)["country"] + assert found["FI"] == "300000000471640" and found["US"] == "300000000480126" + # Ten listed: the pod lists only its biggest, so Denmark may be there. + assert found["DK"] == "*" + assert "facetsList=LOCATIONS" in get.asked[0][0] and "expand" not in get.asked[0][0] + + +def test_oracle_facets_of_a_short_list_leave_the_rest_out() -> None: + saved = json.loads((FIXTURES / "oracle-facets.json").read_text()) + saved["items"][0]["locationsFacet"] = saved["items"][0]["locationsFacet"][:3] + found = oracle.facets(Recorder(json.dumps(saved).encode()), POD)["country"] + assert set(found) == {"US", "IN", "CA"} + + +def test_oracle_selects_locations_and_remote_in_the_finder() -> None: + get = Recorder(b'{"items": [{"requisitionList": [], "TotalJobsCount": 0}]}') + oracle.page(get, POD, None, Ask(countries=("300000000471640", "300000000480126"))) + oracle.page(get, POD, None, Ask(remote=True)) + oracle.page(get, POD, None, Ask(countries=("300000000471640", "*"))) + finders = [unquote(_query(url)["finder"][0]) for url, _ in get.asked] + assert finders[0] == ("findReqs;siteNumber=CX_1,selectedLocationsFacet=" + "300000000471640;300000000480126,limit=200,offset=0") + assert finders[1] == ("findReqs;siteNumber=CX_1,selectedWorkplaceTypesFacet=ORA_REMOTE," + "limit=200,offset=0") + # A country the pod did not list: the series reads the whole board. + assert finders[2] == "findReqs;siteNumber=CX_1,limit=200,offset=0" + + +# --------------------------------------------------------------- series + +def test_countries_and_remote_are_two_series_on_a_host_that_filters_both() -> None: + ask = _runner.ask_for(workday, {}, {"countries": ["DK", "SE"], "remote": True}) + facets = {"country": {"DK": "Location=dk", "SE": "Location=se"}, + "remote": {"remote": "remoteType=r"}} + asks = _runner.series(workday, ask, facets) + assert [(a.countries, a.remote, a.remote_ids) for a in asks] == [ + (("Location=dk", "Location=se"), None, ()), + ((), True, ("remoteType=r",))] + + +def test_remote_is_left_out_beside_countries_where_the_host_has_no_remote_facet() -> None: + ask = Ask(countries=("DK",), remote=True) + asks = _runner.series(workday, ask, {"country": {"DK": "Location=dk"}}) + assert [(a.countries, a.remote) for a in asks] == [(("Location=dk",), None)] + # Remote alone on such a host is the whole board. + [alone] = _runner.series(workday, Ask(remote=True), {"country": {}}) + assert (alone.countries, alone.remote, alone.remote_ids) == ((), None, ()) + + +def test_a_host_with_no_country_facet_is_read_unfiltered_not_emptied() -> None: + [whole] = _runner.series(workday, Ask(countries=("DK",)), {}) + assert whole.countries == () + + +def test_oracle_reads_countries_and_remote_as_two_series() -> None: + ask = _runner.ask_for(oracle, {}, {"countries": ["FI"], "remote": True}) + asks = _runner.series(oracle, ask, {"country": {"FI": "300000000471640"}}) + assert [(a.countries, a.remote) for a in asks] == [ + (("300000000471640",), None), ((), True)] + + +def test_a_one_host_gets_a_series_a_country_and_no_remote() -> None: + ask = _runner.ask_for(smartrecruiters, {}, {"countries": ["DK", "SE"], "remote": True}) + assert ask.remote is None + assert [a.countries for a in _runner.series(smartrecruiters, ask)] == [("DK",), ("SE",)] + + +def test_a_reach_too_wide_for_a_one_host_reads_it_unfiltered() -> None: + europe = [c for c in _countries.NAMES if c in { + "AT", "BE", "BG", "HR", "CY", "CZ", "DK", "EE", "FI", "FR", "DE", "GR", "HU", "IE"}] + assert len(europe) > _runner.MAX_SERIES + assert _runner.ask_for(eightfold, {}, {"countries": europe}) == Ask() + # A list in one request takes it whole. + assert _runner.ask_for(phenom, {}, {"countries": europe}).countries == tuple(europe) + # A board's own params are kept however many. + assert len(_runner.ask_for(eightfold, {"countries": europe}, None).countries) == len(europe) + + +@pytest.mark.parametrize("params, expected", [ + ({"reach": "all"}, Ask()), + ({"countries": ["DE"], "remote": False}, Ask(countries=("DE",))), + ({"remote": True}, Ask(countries=("DK",), remote=True)), +]) +def test_a_boards_params_win_over_the_reach(params: dict[str, Any], expected: Ask) -> None: + reach = {"countries": ["DK"], "remote": True} + assert _runner.ask_for(oracle, params, reach) == expected + + +def test_facets_are_asked_for_remote_alone_on_a_host_that_names_it_by_id() -> None: + calls: list[str] = [] + + def facets(get: Any, slug: str) -> dict[str, dict[str, str]]: + calls.append(slug) + return {"remote": {"remote": "remoteType=r"}} + + state: dict[str, str] = {} + original = workday.facets + try: + workday.facets = facets # type: ignore[assignment] + found = _runner.facets_for(workday, LEGO, Recorder(b""), Ask(remote=True), state, "k") + finally: + workday.facets = original # type: ignore[assignment] + assert found == {"remote": {"remote": "remoteType=r"}} and calls == [LEGO]