Read big boards within the agents' reach: countries and remote from the enabled agents, Europe for favourites, filtered at the source by six hosts

app/jobs/reach.py derives each board's reach once per pull and the pull hands it to the runner through the batch payload. SmartRecruiters, Eightfold and SuccessFactors filter one country a request, Phenom a list, Workday and Oracle by host ids from facets. Remote beside countries is a series of its own, since two facets intersect.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 09:55:18 +02:00
1 parent 3adafd8950
commit 3462a4abe7
17 files changed
+1324 -116

No files matched your search

+13 -3
View File
@@ -17,8 +17,12 @@ A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
`lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what
its slug is. Every board is read independently; one that fails is logged,
recorded as failing under `ats:failing:<target>`, and skipped. A board's
subscription `params` (`{"countries": ["DK"], "remote": true}`) are its
filter, narrowed to what its reader declares (`_runner.ask_for`).
filter is the `reach` option the pull hands it (the agents', see
`app/jobs/reach.py`), over which its subscription `params` (`{"countries":
["DK"], "remote": true}`, `{"reach": "all"}`) win, narrowed to what its
reader declares (`_runner.ask_for`). An unfiltered read records the board's
size under `ats:size:<target>`, which the reach reads to leave small boards
whole.
The limits below are read when a board is read, so a test that replaces one
on this module reaches the runner.
@@ -349,7 +353,7 @@ class ATSSource(Source):
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
module = readers.HOSTS[host]
get = _getter(client, host, slug)
ask = _runner.ask_for(module, self.params_for(target),
ask = _runner.ask_for(module, self.params_for(target), self.options.get("reach"),
keys=self.options.get("keys"))
listing = _runner.read(self.state, "ats", target, module, slug, get, ask,
limits=_limits())
@@ -398,6 +402,12 @@ class ATSSource(Source):
# its details again.
_runner.stamp(self.state, "ats", target, run_marker, listing,
[compound(i) for i in listing.ids])
if not listing.filtered:
size = len(listing.ids)
if not listing.complete:
kept = self.state.get(f"ats:size:{target}", "")
size = max(size, int(kept) if kept.isdigit() else 0)
self.state[f"ats:size:{target}"] = str(size)
if detail_sets is not None:
for key, ids in zip(detail_keys, detail_sets, strict=True):
self.state[key] = json.dumps(sorted(ids))
+15 -1
View File
@@ -21,6 +21,11 @@ Signatures of a module that declares `READS`, with `Get` and `Page` from
def detail(get: Get, slug: str, row: Row) -> dict[str, Any] # text="detail"
def facets(get: Get, slug: str) -> dict[str, dict[str, str]] # an "ids" filter
`facets` maps each `ids` dimension to `{value: host id}`: an ISO code for
`country`, a place name for `place`, any word for `remote`. A dimension the
host has no facet for is left out, and the runner then reads it unfiltered;
a dimension present without a wanted value means the host has none of it.
A module without `READS` keeps the signatures without `ask` until it
declares; `undeclared()` in the package lists those modules.
"""
@@ -146,11 +151,20 @@ class Ask(_Frozen):
carries one value. With `ids`, `countries` and `places` hold the host's
ids from `facets`, not codes or names. `params` are a search's saved
params (and a resolved key under `"key"`). An empty `Ask()` is an
unfiltered read."""
unfiltered read.
`remote=True` asks for remote roles. A reach of countries and remote
means "in these countries, or remote", which no host can ask in one
request (two facets intersect), so the runner makes the remote roles a
series of their own: a reader never gets both `remote=True` and a
country or a place. With `remote="ids"`, `remote_ids` holds the host's
ids of its remote values (`facets(...)["remote"]`, any key), and
`remote` stays True."""
countries: tuple[str, ...] = ()
places: tuple[str, ...] = ()
remote: bool | None = None
remote_ids: tuple[str, ...] = ()
query: str = ""
params: dict[str, Any] = {}
+93
View File
@@ -0,0 +1,93 @@
"""Country codes and the English names hosts write, for the readers that
filter by a country's name (Eightfold, Phenom) or name a host id by it
(Workday, Oracle), and for SuccessFactors, which writes a place's country as
its code.
`NAMES` is ISO 3166-1 alpha-2 to the English short name. `code_of` reads a
name back, with the spellings hosts use that are not the short name
("United States of America", "Korea, Republic of", "Czech Republic").
"""
from __future__ import annotations
import unicodedata
#: ISO 3166-1 alpha-2 to the English name, which is what `country_of` reads:
#: a bare "DE" or "SE" in a location line is taken for Delaware or nothing.
NAMES: dict[str, str] = dict(entry.strip().split(" ", 1) for entry in """
AD Andorra | AE United Arab Emirates | AF Afghanistan | AG Antigua and Barbuda | AI
Anguilla | AL Albania | AM Armenia | AO Angola | AR Argentina | AS American Samoa | AT
Austria | AU Australia | AW Aruba | AX Åland Islands | AZ Azerbaijan | BA Bosnia and
Herzegovina | BB Barbados | BD Bangladesh | BE Belgium | BF Burkina Faso | BG Bulgaria
| BH Bahrain | BI Burundi | BJ Benin | BM Bermuda | BN Brunei | BO Bolivia | BQ
Caribbean Netherlands | BR Brazil | BS Bahamas | BT Bhutan | BW Botswana | BY Belarus
| BZ Belize | CA Canada | CD Democratic Republic of the Congo | CF Central African
Republic | CG Republic of the Congo | CH Switzerland | CI Côte d'Ivoire | CK Cook
Islands | CL Chile | CM Cameroon | CN China | CO Colombia | CR Costa Rica | CU Cuba |
CV Cabo Verde | CW Curaçao | CY Cyprus | CZ Czechia | DE Germany | DJ Djibouti | DK
Denmark | DM Dominica | DO Dominican Republic | DZ Algeria | EC Ecuador | EE Estonia |
EG Egypt | ER Eritrea | ES Spain | ET Ethiopia | FI Finland | FJ Fiji | FK Falkland
Islands | FM Micronesia | FO Faroe Islands | FR France | GA Gabon | GB United Kingdom
| GD Grenada | GE Georgia | GF French Guiana | GG Guernsey | GH Ghana | GI Gibraltar |
GL Greenland | GM Gambia | GN Guinea | GP Guadeloupe | GQ Equatorial Guinea | GR
Greece | GT Guatemala | GU Guam | GW Guinea-Bissau | GY Guyana | HK Hong Kong | HN
Honduras | HR Croatia | HT Haiti | HU Hungary | ID Indonesia | IE Ireland | IL Israel
| IM Isle of Man | IN India | IQ Iraq | IR Iran | IS Iceland | IT Italy | JE Jersey |
JM Jamaica | JO Jordan | JP Japan | KE Kenya | KG Kyrgyzstan | KH Cambodia | KI
Kiribati | KM Comoros | KN Saint Kitts and Nevis | KP North Korea | KR South Korea |
KW Kuwait | KY Cayman Islands | KZ Kazakhstan | LA Laos | LB Lebanon | LC Saint Lucia
| LI Liechtenstein | LK Sri Lanka | LR Liberia | LS Lesotho | LT Lithuania | LU
Luxembourg | LV Latvia | LY Libya | MA Morocco | MC Monaco | MD Moldova | ME
Montenegro | MG Madagascar | MH Marshall Islands | MK North Macedonia | ML Mali | MM
Myanmar | MN Mongolia | MO Macao | MP Northern Mariana Islands | MQ Martinique | MR
Mauritania | MS Montserrat | MT Malta | MU Mauritius | MV Maldives | MW Malawi | MX
Mexico | MY Malaysia | MZ Mozambique | NA Namibia | NC New Caledonia | NE Niger | NG
Nigeria | NI Nicaragua | NL Netherlands | NO Norway | NP Nepal | NR Nauru | NU Niue |
NZ New Zealand | OM Oman | PA Panama | PE Peru | PF French Polynesia | PG Papua New
Guinea | PH Philippines | PK Pakistan | PL Poland | PR Puerto Rico | PS Palestine | PT
Portugal | PW Palau | PY Paraguay | QA Qatar | RE Réunion | RO Romania | RS Serbia |
RU Russia | RW Rwanda | SA Saudi Arabia | SB Solomon Islands | SC Seychelles | SD
Sudan | SE Sweden | SG Singapore | SI Slovenia | SK Slovakia | SL Sierra Leone | SM
San Marino | SN Senegal | SO Somalia | SR Suriname | SS South Sudan | ST São Tomé and
Príncipe | SV El Salvador | SY Syria | SZ Eswatini | TC Turks and Caicos Islands | TD
Chad | TG Togo | TH Thailand | TJ Tajikistan | TL Timor-Leste | TM Turkmenistan | TN
Tunisia | TO Tonga | TR Turkey | TT Trinidad and Tobago | TV Tuvalu | TW Taiwan | TZ
Tanzania | UA Ukraine | UG Uganda | US United States | UY Uruguay | UZ Uzbekistan | VA
Vatican City | VC Saint Vincent and the Grenadines | VE Venezuela | VG British Virgin
Islands | VI U.S. Virgin Islands | VN Vietnam | VU Vanuatu | WS Samoa | XK Kosovo | YE
Yemen | ZA South Africa | ZM Zambia | ZW Zimbabwe
""".replace("\n", " ").split(" | "))
#: Other names hosts give a country: Workday's "United States of America" and
#: "Korea, Republic of", older and local spellings.
_OTHER = {
"united states of america": "US", "usa": "US", "u.s.a.": "US", "u.s.": "US",
"united kingdom of great britain and northern ireland": "GB", "great britain": "GB",
"uk": "GB", "england": "GB", "korea, republic of": "KR", "republic of korea": "KR",
"korea": "KR", "czech republic": "CZ", "turkiye": "TR", "viet nam": "VN",
"russian federation": "RU", "hong kong sar": "HK", "hong kong sar, china": "HK",
"taiwan, province of china": "TW", "macau": "MO", "the netherlands": "NL",
"holland": "NL", "slovak republic": "SK", "republic of ireland": "IE",
"ivory coast": "CI", "cote d'ivoire": "CI", "north macedonia, republic of": "MK",
"macedonia": "MK", "iran, islamic republic of": "IR", "moldova, republic of": "MD",
"tanzania, united republic of": "TZ", "united arab emirates (uae)": "AE", "uae": "AE",
}
def _fold(name: str) -> str:
decomposed = unicodedata.normalize("NFKD", name.strip().casefold())
return " ".join("".join(c for c in decomposed if not unicodedata.combining(c)).split())
_BY_NAME: dict[str, str] = {_fold(name): code for code, name in NAMES.items()} | _OTHER
def code_of(name: str) -> str:
"""The ISO code of a country written by name, '' for a name not known.
Case and accents do not matter."""
return _BY_NAME.get(_fold(name or ""), "")
def name_of(code: str) -> str:
"""The English name of an ISO code, '' for a code not known."""
return NAMES.get((code or "").strip().upper(), "")
+65 -19
View File
@@ -289,14 +289,22 @@ def resolve_key(key: Key, keys: Mapping[str, Any] | None) -> dict[str, str]:
return out
#: The most series a reach may cost on a `one` filter: a reach of more
#: countries (a favourite's Europe) reads such a host unfiltered instead.
MAX_SERIES = 8
def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None = None,
reach: Mapping[str, Any] | None = None, *,
keys: Mapping[str, Any] | None = None) -> Ask:
"""The `Ask` one target resolves to. `reach` (from the agents, package F)
and then the subscription's own `params` (a board's override, e.g.
`{"countries": ["DK"], "remote": true}`, or a search's saved params) set
the dimensions, params winning per dimension, narrowed to the ones the
reader declares in `READS.filters`. A search's params go whole into
"""The `Ask` one target resolves to. `reach` (from the agents, see
`app/jobs/reach.py`) and then the subscription's own `params` (a board's
override, e.g. `{"countries": ["DK"], "remote": true}`, or a search's
saved params) set the dimensions, params winning per dimension, narrowed
to the ones the reader declares in `READS.filters`. `{"reach": "all"}` in
the params drops the reach, and so does a reach of more than
`MAX_SERIES` countries on a reader whose country filter is `one`. A
`remote` of False asks nothing. A search's params go whole into
`Ask.params`; a reader with `READS.key` gets the resolved key under
`Ask.params["key"]` (see `resolve_key`, which raises when it is missing).
A module without `READS` gets `Ask()`."""
@@ -304,7 +312,12 @@ def ask_for(module: ModuleType, subscription_params: Mapping[str, Any] | None =
return Ask()
reads: Reads = module.READS
params = dict(subscription_params or {})
wanted = _dimensions(module, reach or {}) | _dimensions(module, params)
reached = {} if params.get("reach") == "all" else _dimensions(module, reach or {})
if reads.filters.country == "one" and len(reached.get("country", ())) > MAX_SERIES:
reached = {}
wanted = reached | _dimensions(module, params)
if wanted.get("remote") is False:
del wanted["remote"]
fields: dict[str, Any] = {_ASK_FIELD[name]: value for name, value in wanted.items()
if name in reads.filters.declared()}
extra: dict[str, Any] = dict(params) if reads.scope == "search" else {}
@@ -320,29 +333,61 @@ def filtered(ask: Ask) -> bool:
def series(module: ModuleType, ask: Ask,
facets: Mapping[str, Mapping[str, str]] | None = None) -> list[Ask]:
"""The series of requests `ask` takes on this reader: one per value of a
dimension declared `one` (the product when two are), one with every
value for `many`, and host ids from `facets` for `ids` (values `facets`
does not know are dropped; an empty list when none is left, since the
host then has nothing in reach). `[ask]` for a module without `READS`."""
"""The series of requests `ask` takes on this reader. Remote with a
country or a place is two parts, "in these places, or remote": the
places without remote, and remote alone. Each part is one series per
value of a dimension declared `one` (the product when two are), one
with every value for `many`, and host ids from `facets` for `ids`:
values `facets` does not know are dropped, and a part with none left is
no series, since the host has nothing of it; a dimension `facets` does
not state at all is read unfiltered, and so is remote alone, while the
remote part beside places is then left out. `[ask]` for a module
without `READS`."""
if not declared(module):
return [ask]
if ask.remote and (ask.countries or ask.places):
return [*_series(module, ask.model_copy(update={"remote": None}), facets),
*_series(module, ask.model_copy(update={"countries": (), "places": ()}),
facets, beside=True)]
return _series(module, ask, facets)
def _series(module: ModuleType, ask: Ask, facets: Mapping[str, Mapping[str, str]] | None,
beside: bool = False) -> list[Ask]:
"""`series` for an ask that is places or remote, not both. `beside` marks
the remote part of an ask that also has places."""
reach = module.READS.filters
known_facets = facets or {}
countries, places = ask.countries, ask.places
for dimension in ("country", "place"):
values = countries if dimension == "country" else places
if getattr(reach, dimension) != "ids" or not values:
continue
known = (facets or {}).get(dimension) or {}
mapped = tuple(known[v] for v in values if v in known)
if not mapped:
log.info("readers: %s knows none of %s %s", module.NAME, dimension,
", ".join(values))
return []
if dimension not in known_facets:
log.info("readers: %s states no %s facet; read unfiltered", module.NAME, dimension)
mapped: tuple[str, ...] = ()
else:
known = known_facets.get(dimension) or {}
mapped = tuple(dict.fromkeys(known[v] for v in values if v in known))
if not mapped:
log.info("readers: %s knows none of %s %s", module.NAME, dimension,
", ".join(values))
return []
if dimension == "country":
countries = mapped
else:
places = mapped
if reach.remote == "ids" and ask.remote:
if "remote" not in known_facets:
if beside:
log.info("readers: %s states no remote facet; remote left out", module.NAME)
return []
ask = ask.model_copy(update={"remote": None})
else:
ids = tuple(dict.fromkeys((known_facets.get("remote") or {}).values()))
if not ids:
return []
ask = ask.model_copy(update={"remote_ids": ids})
by_country = [(c,) for c in countries] if reach.country == "one" and countries \
else [countries]
by_place = [(p,) for p in places] if reach.place == "one" and places else [places]
@@ -505,12 +550,13 @@ def _load(state: Mapping[str, str], key: str) -> dict[str, Any]:
def facets_for(module: ModuleType, slug: str, get: Get, ask: Ask,
state: MutableMapping[str, str] | None = None, key: str = "") -> dict[str, Any]:
"""The reader's `facets` when `ask` sets a dimension it declares `ids`,
"""The reader's `facets` when `ask` sets a dimension it declares `ids`
(a country, a place, or remote),
kept in `state[key]` for `FACETS_TTL`; `{}` otherwise."""
if not declared(module) or not hasattr(module, "facets"):
return {}
ids = [name for name, reach in module.READS.filters.declared().items() if reach == "ids"]
if not any(getattr(ask, _ASK_FIELD[name]) for name in ids if name in ("country", "place")):
if not any(getattr(ask, _ASK_FIELD[name]) for name in ids if name != "query"):
return {}
now = datetime.now(UTC)
if state is not None and key:
+20 -13
View File
@@ -15,14 +15,17 @@ links. `/api/apply/v2/jobs` answers 403 "Not authorized for PCSX" on the sites
tried, so it is not used.
The search ignores `num` and answers ten roles a page, so a board costs one
request per ten roles, and `ats.MAX_PAGES` (100) stops a read at 1,000 roles: a
larger board (Infineon's 1,292, Microsoft's 2,319) is read incomplete every pull,
and an incomplete read never closes the roles it did not list. The rows are
sorted newest first (`sort_by=timestamp`) so that what the cap keeps is the
newest thousand and the pages do not shuffle between two requests, unlike the
default order, which puts hot roles first. The search does take a `location`
free text (Ericsson's "Sweden" found 157 of 637), which could bring a read
under the cap, but a target names no filter and none is built here.
request per ten roles, and `ats.MAX_PAGES` (100) stops a read at 1,000 roles.
The rows are sorted newest first (`sort_by=timestamp`) so that what the cap
keeps is the newest thousand and the pages do not shuffle between two
requests, unlike the default order, which puts hot roles first.
A country is filtered on the server by `location=<country name>`, one a
request (`Filters(country="one")`): Ericsson's "Sweden" answered 154 of 618 on
2026-09-30, a role at several places is kept when one of them matches, and
the review measured Infineon at 163 of 1,288 and Microsoft at 60 of 2,320 for
Denmark and Sweden, which brings both under the cap. A code with no English
name in `_countries` is read unfiltered.
Discovery names Eightfold from an `eightfold.ai` link, and the host and the
domain from one that carries `domain=`.
@@ -35,15 +38,17 @@ import json
import re
from collections.abc import Callable
from typing import Any
from urllib.parse import quote
from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads, Row
from ._contract import Ask, Fields, Filters, Place, Reads, Row
from ._countries import name_of
from ._shared import Get, Page, is_web_url, only, parse_dt
READS = Reads(
scope="board", listing="paged", page_size=10, text="detail", complete="stated",
places="many",
places="many", filters=Filters(country="one"),
fields=Fields(posted="listed", remote="stated"),
robots="read: robots.txt allows /careers (checked 2026-09-29)",
)
@@ -95,12 +100,14 @@ def careers_url(slug: str) -> str:
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
"""One page of `/api/pcsx/search` -> `{"status": 200, "data": {"positions":
[...], "count": n}}`; the next starts at the offset after this page's
positions, and an empty page ends the read."""
[...], "count": n}}`, in the one country `ask` names; the next starts at
the offset after this page's positions, and an empty page ends the read."""
host, domain = parts(slug)
start = int(cursor or 0)
country = name_of(ask.countries[0]) if ask.countries else ""
where = f"&location={quote(country)}" if country else ""
answer = json.loads(get(f"https://{host}/api/pcsx/search?domain={domain}"
f"&sort_by=timestamp&start={start}"))
f"&sort_by=timestamp&start={start}{where}"))
data = answer.get("data") if isinstance(answer, dict) else None
positions = data.get("positions") if isinstance(data, dict) else None
if not isinstance(positions, list):
+52 -3
View File
@@ -8,6 +8,18 @@ number is `CX_1001` or `CX_1` on the tenants read so far, and the site's own
name (`SinchCareer`) is taken in its place. Written from live boards on
2026-09-29 (Milestone Systems, Nokia, Danske Bank, Sinch); jobhunt-hq did not
read Oracle.
Countries are filtered on the server by `selectedLocationsFacet`, the pod's
geography ids (`Filters(country="ids")`), which `facets` reads off
`locationsFacet`; several ids go in one request, joined with `;`, the way the
site's own search sends a multiple choice. That list holds only the pod's ten
biggest countries (fa-evmr's, 2026-09-30), so when it is full a country
missing from it may still have roles: `facets` then maps every other country
to `*`, and a series asking for one is read without a country filter. An id
the pod does not know answers the whole board (review). Remote is
`selectedWorkplaceTypesFacet=ORA_REMOTE` (`remote="one"`); two facets are an
intersection (Finland and remote answered 0 on 2026-09-30), which is why the
runner asks for remote in a series of its own.
"""
from __future__ import annotations
@@ -21,7 +33,8 @@ from urllib.parse import quote
from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads
from ._contract import Ask, Fields, Filters, Place, Reads
from ._countries import NAMES, code_of
from ._shared import Get, Page, Row, only, parse_dt
NAME = "oracle"
@@ -41,7 +54,7 @@ PATTERNS = (
READS = Reads(
scope="board", listing="paged", page_size=200, text="detail", teaser=True,
complete="stated", places="many",
complete="stated", places="many", filters=Filters(country="ids", remote="one"),
fields=Fields(posted="trusted", remote="stated", country="stated"),
robots=("read: robots.txt allows /hcmUI/CandidateExperience/en/sites/<site>/jobs "
"(checked 2026-09-29)"),
@@ -103,7 +116,7 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
data = json.loads(get(
f"{_api(slug)}/recruitingCEJobRequisitions?onlyData=true"
f"&expand=requisitionList.secondaryLocations"
f"&finder=findReqs;siteNumber={site},limit={PAGE_SIZE},offset={offset}"))
f"&finder=findReqs;siteNumber={site},{_selected(ask)}limit={PAGE_SIZE},offset={offset}"))
search = (data.get("items") or [{}])[0]
roles = search.get("requisitionList") or []
# `_paged` counts `id`, which Oracle spells `Id`.
@@ -111,6 +124,42 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
return search, results, search.get("TotalJobsCount"), offset + len(roles) if roles else None
#: How many countries `locationsFacet` lists at most; a list this long may
#: leave some out.
FACET_LIMIT = 10
def _selected(ask: Ask) -> str:
"""The finder's facet parameters for `ask`, each ending in a comma: the
location ids (none when one is `*`, a country the pod did not list) and
remote."""
out = ""
ids = [i for i in ask.countries if re.fullmatch(r"\d+", i)]
if ids and "*" not in ask.countries:
out += f"selectedLocationsFacet={'%3B'.join(ids)},"
if ask.remote:
out += "selectedWorkplaceTypesFacet=ORA_REMOTE,"
return out
def facets(get: Get, slug: str) -> dict[str, dict[str, str]]:
"""The pod's geography id of each country `locationsFacet` names, read
with the facets and no roles. When the list is `FACET_LIMIT` long, every
country it leaves out maps to `*`: its roles may be there, and a read
for it goes unfiltered."""
site = _parts(slug)[1]
data = json.loads(get(
f"{_api(slug)}/recruitingCEJobRequisitions?onlyData=true"
f"&finder=findReqs;siteNumber={site},facetsList=LOCATIONS,limit=1,offset=0"))
listed = (data.get("items") or [{}])[0].get("locationsFacet") or []
country = {code: str(entry["Id"]) for entry in listed
if isinstance(entry, dict) and entry.get("Id")
and (code := code_of(str(entry.get("Name") or "")))}
if len(listed) >= FACET_LIMIT:
country = dict.fromkeys(NAMES, "*") | country
return {"country": country}
def parse(body: bytes, slug: str) -> list[Row]:
return rows((json.loads(body).get("items") or [{}])[0], slug)
+20 -8
View File
@@ -9,6 +9,14 @@ answered without a cookie, an `Origin` header, a `pageId` or the site's
`country`, both read off the slug's path) and the paging. Nothing is taken
from the page's script. Written from live sites on 2026-09-29 (King, Just Eat
Takeaway, Philips); jobhunt-hq did not read Phenom.
Countries are filtered on the server as a list of English names in
`selected_fields.country`, all in one request (`Filters(country="many")`); the
list is a union, and a role at several places is kept when one of them
matches. Philips answered 111 of 873 for Netherlands and Germany on
2026-09-30. There is no remote field to filter on. A code with no English
name in `_countries` is left out of the list, and a list left empty reads the
whole board.
"""
from __future__ import annotations
@@ -21,7 +29,8 @@ from typing import Any
from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads
from ._contract import Ask, Fields, Filters, Place, Reads
from ._countries import name_of
from ._shared import Get, Page, Row, only, parse_dt
NAME = "phenom"
@@ -37,7 +46,8 @@ PATTERNS = (
READS = Reads(
scope="board", listing="paged", page_size=100, text="detail", teaser=True,
complete="stated", places="many", fields=Fields(posted="trusted", remote="stated"),
complete="stated", places="many", filters=Filters(country="many"),
fields=Fields(posted="trusted", remote="stated"),
robots="read: robots.txt allows /<country>/<lang>/search-results (checked 2026-09-29)",
)
@@ -77,19 +87,21 @@ def careers_url(slug: str) -> str:
return f"{_root(slug)}/search-results"
def _body(slug: str, offset: int) -> dict[str, Any]:
def _body(slug: str, offset: int, countries: tuple[str, ...] = ()) -> dict[str, Any]:
"""What the site's search page sends for a listing of every role, newest
first, less the parts that only tell the page what to draw. `lang` is
`en_global` for `/global/en` and `en_us` for `/us/en`, the site's own
spelling on the three sites read."""
first, less the parts that only tell the page what to draw, in the
`countries` (ISO codes) given. `lang` is `en_global` for `/global/en` and
`en_us` for `/us/en`, the site's own spelling on the three sites read."""
_host, country, lang = _parts(slug)
names = [name for code in countries if (name := name_of(code))]
return {
"lang": f"{lang}_{country}", "deviceType": "desktop", "country": country,
"pageName": "search-results", "ddoKey": "refineSearch", "sortBy": "",
"subsearch": "", "from": offset, "jobs": True, "counts": True, "all_fetch": False,
"pageType": "landingPage", "size": PAGE_SIZE, "clearAll": False,
"jdsource": "facets", "isSliderEnable": False, "siteType": "external",
"location": "", "keywords": "", "global": True, "selected_fields": {},
"location": "", "keywords": "", "global": True,
"selected_fields": {"country": names} if names else {},
"sort": {"order": "desc", "field": "postedDate"}, "locationData": {},
}
@@ -106,7 +118,7 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
offset = int(cursor or 0)
host = _parts(slug)[0]
data = json.loads(get(f"https://{host}/widgets", method="POST",
payload=_body(slug, offset)))
payload=_body(slug, offset, ask.countries)))
search = data.get("refineSearch")
if not isinstance(search, dict) or search.get("status", 200) != 200:
raise ValueError("not a Phenom search answer: no refineSearch")
+13 -3
View File
@@ -3,16 +3,22 @@ with a stated total. The identifier is case-sensitive (`BoschGroup`).
Written from one real board on 2026-09-23; jobhunt-hq did not read
SmartRecruiters.
A country is filtered on the server, `?country=dk`, one a request
(`Filters(country="one")`): Bosch's DE, SE and DK summed to what the app held
(review, 2026-09-29), and DK answered 2 of 4,830 on 2026-09-30. The API
ignores a remote parameter, so remote is not filtered.
"""
from __future__ import annotations
import json
import logging
import re
from collections.abc import Callable
from typing import Any
from ._contract import Ask, Fields, Place, Reads
from ._contract import Ask, Fields, Filters, Place, Reads
from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok
log = logging.getLogger(__name__)
@@ -27,7 +33,7 @@ PATTERNS = (
#: An unknown identifier answers an empty list, which is `empty="suspect"`.
READS = Reads(
scope="board", listing="paged", page_size=100, text="none", complete="stated",
places="one", empty="suspect",
places="one", empty="suspect", filters=Filters(country="one"),
fields=Fields(posted="trusted", remote="stated", country="stated", seniority="stated",
employment="stated"),
robots="api: no robots.txt check needed (public board API), 2026-09-30",
@@ -73,9 +79,13 @@ def probe_ok(body: bytes) -> bool:
def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
"""One page, in the one country `ask` names, which the API takes in lower
case."""
offset = int(cursor or 0)
country = ask.countries[0].lower() if ask.countries else ""
where = f"&country={country}" if re.fullmatch(r"[a-z]{2}", country) else ""
data = json.loads(get(f"https://{_API}/v1/companies/{slug}/postings"
f"?offset={offset}&limit={PAGE_SIZE}"))
f"?offset={offset}&limit={PAGE_SIZE}{where}"))
content = data.get("content") or []
return data, content, data.get("totalFound"), offset + len(content) if content else None
+22 -53
View File
@@ -30,8 +30,13 @@ module reads, and neither is a translation of a listed role.
A path prefix names a brand of a shared site. `<host>/<prefix>/search/` lists
only that brand on the server (Universal Robots 30 of Teradyne's 486, MiR 1),
so the read is as small as the company; `rows` still keeps only the roles
under `/<prefix>/job/`, in case a site does not scope. There is no server-side
country filter in this version.
under `/<prefix>/job/`, in case a site does not scope.
A country is filtered on the server by the search's own facet,
`optionsFacetsDD_country=DK`, one country a request (`Filters(country="one")`);
it worked on the four tenants tried in the review and on Vestas (64 of 668,
2026-09-30). A role at several places is listed when one of them is in the
country. There is no remote facet.
A role's own page carries microdata (`itemtype` schema.org JobPosting) with
`datePosted` and the address; `detail` reads the description, the date and the
@@ -52,7 +57,8 @@ from urllib.parse import urljoin, urlsplit
from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads
from ._contract import Ask, Fields, Filters, Place, Reads
from ._countries import NAMES
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
NAME = "successfactors"
@@ -79,6 +85,7 @@ KEEP_CASE = True
READS = Reads(
scope="board", listing="paged", text="detail", complete="stated", places="many",
filters=Filters(country="one"),
fields=Fields(posted="trusted", country="stated"),
robots="read: robots.txt allows /search/ and /job/ (checked 2026-09-29)",
)
@@ -122,52 +129,10 @@ _COUNTRY_LABELS = ("country", "land")
_MONTHS = {name: number for number, name in enumerate(
("jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"), 1)}
#: ISO 3166-1 alpha-2 to the English name, which is what `country_of` reads:
#: a bare "DE" or "SE" in a location line is taken for Delaware or nothing. A
#: code that is not here stays as the site wrote it.
_COUNTRIES = dict(entry.strip().split(" ", 1) for entry in """
AD Andorra | AE United Arab Emirates | AF Afghanistan | AG Antigua and Barbuda | AI
Anguilla | AL Albania | AM Armenia | AO Angola | AR Argentina | AS American Samoa | AT
Austria | AU Australia | AW Aruba | AX Åland Islands | AZ Azerbaijan | BA Bosnia and
Herzegovina | BB Barbados | BD Bangladesh | BE Belgium | BF Burkina Faso | BG Bulgaria
| BH Bahrain | BI Burundi | BJ Benin | BM Bermuda | BN Brunei | BO Bolivia | BQ
Caribbean Netherlands | BR Brazil | BS Bahamas | BT Bhutan | BW Botswana | BY Belarus
| BZ Belize | CA Canada | CD Democratic Republic of the Congo | CF Central African
Republic | CG Republic of the Congo | CH Switzerland | CI Côte d'Ivoire | CK Cook
Islands | CL Chile | CM Cameroon | CN China | CO Colombia | CR Costa Rica | CU Cuba |
CV Cabo Verde | CW Curaçao | CY Cyprus | CZ Czechia | DE Germany | DJ Djibouti | DK
Denmark | DM Dominica | DO Dominican Republic | DZ Algeria | EC Ecuador | EE Estonia |
EG Egypt | ER Eritrea | ES Spain | ET Ethiopia | FI Finland | FJ Fiji | FK Falkland
Islands | FM Micronesia | FO Faroe Islands | FR France | GA Gabon | GB United Kingdom
| GD Grenada | GE Georgia | GF French Guiana | GG Guernsey | GH Ghana | GI Gibraltar |
GL Greenland | GM Gambia | GN Guinea | GP Guadeloupe | GQ Equatorial Guinea | GR
Greece | GT Guatemala | GU Guam | GW Guinea-Bissau | GY Guyana | HK Hong Kong | HN
Honduras | HR Croatia | HT Haiti | HU Hungary | ID Indonesia | IE Ireland | IL Israel
| IM Isle of Man | IN India | IQ Iraq | IR Iran | IS Iceland | IT Italy | JE Jersey |
JM Jamaica | JO Jordan | JP Japan | KE Kenya | KG Kyrgyzstan | KH Cambodia | KI
Kiribati | KM Comoros | KN Saint Kitts and Nevis | KP North Korea | KR South Korea |
KW Kuwait | KY Cayman Islands | KZ Kazakhstan | LA Laos | LB Lebanon | LC Saint Lucia
| LI Liechtenstein | LK Sri Lanka | LR Liberia | LS Lesotho | LT Lithuania | LU
Luxembourg | LV Latvia | LY Libya | MA Morocco | MC Monaco | MD Moldova | ME
Montenegro | MG Madagascar | MH Marshall Islands | MK North Macedonia | ML Mali | MM
Myanmar | MN Mongolia | MO Macao | MP Northern Mariana Islands | MQ Martinique | MR
Mauritania | MS Montserrat | MT Malta | MU Mauritius | MV Maldives | MW Malawi | MX
Mexico | MY Malaysia | MZ Mozambique | NA Namibia | NC New Caledonia | NE Niger | NG
Nigeria | NI Nicaragua | NL Netherlands | NO Norway | NP Nepal | NR Nauru | NU Niue |
NZ New Zealand | OM Oman | PA Panama | PE Peru | PF French Polynesia | PG Papua New
Guinea | PH Philippines | PK Pakistan | PL Poland | PR Puerto Rico | PS Palestine | PT
Portugal | PW Palau | PY Paraguay | QA Qatar | RE Réunion | RO Romania | RS Serbia |
RU Russia | RW Rwanda | SA Saudi Arabia | SB Solomon Islands | SC Seychelles | SD
Sudan | SE Sweden | SG Singapore | SI Slovenia | SK Slovakia | SL Sierra Leone | SM
San Marino | SN Senegal | SO Somalia | SR Suriname | SS South Sudan | ST São Tomé and
Príncipe | SV El Salvador | SY Syria | SZ Eswatini | TC Turks and Caicos Islands | TD
Chad | TG Togo | TH Thailand | TJ Tajikistan | TL Timor-Leste | TM Turkmenistan | TN
Tunisia | TO Tonga | TR Turkey | TT Trinidad and Tobago | TV Tuvalu | TW Taiwan | TZ
Tanzania | UA Ukraine | UG Uganda | US United States | UY Uruguay | UZ Uzbekistan | VA
Vatican City | VC Saint Vincent and the Grenadines | VE Venezuela | VG British Virgin
Islands | VI U.S. Virgin Islands | VN Vietnam | VU Vanuatu | WS Samoa | XK Kosovo | YE
Yemen | ZA South Africa | ZM Zambia | ZW Zimbabwe
""".replace("\n", " ").split(" | "))
#: ISO 3166-1 alpha-2 to the English name (`_countries.NAMES`): a bare "DE"
#: or "SE" in a location line is taken for Delaware or nothing. A code that
#: is not there stays as the site wrote it.
_COUNTRIES = NAMES
def _parts(slug: str) -> tuple[str, str]:
@@ -190,9 +155,12 @@ def careers_url(slug: str) -> str:
return f"https://{host}{prefix}/search/"
def _search_url(slug: str, start: int) -> str:
def _search_url(slug: str, start: int, country: str = "") -> str:
"""The search page from `start`, in one country when `country` (an ISO
code) is given."""
host, prefix = _parts(slug)
return f"https://{host}{prefix}/search/?q=&locationsearch=&startrow={start}"
facet = f"&optionsFacetsDD_country={country}" if re.fullmatch(r"[A-Z]{2}", country) else ""
return f"https://{host}{prefix}/search/?q=&locationsearch={facet}&startrow={start}"
# ------------------------------------------------------------------ listing
@@ -296,12 +264,13 @@ def _entries(text: str) -> list[dict[str, Any]]:
def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
"""One search page. The cursor is `(start row, first role of the page
"""One search page, in the one country `ask` names. The cursor is `(start row, first role of the page
before)`: the next start row is what this page returned, and a page that
starts with the same role as the last one is the site ignoring `startrow`,
which ends the read (incomplete) instead of looping."""
start, previous = cursor or (0, None)
text = get(_search_url(slug, start)).decode("utf-8", "replace")
country = ask.countries[0] if ask.countries else ""
text = get(_search_url(slug, start, country)).decode("utf-8", "replace")
if not _SEARCH_PAGE.search(text):
raise ValueError("not a SuccessFactors search page")
found = _entries(text)
+79 -3
View File
@@ -12,6 +12,17 @@ keeps the case the URL wrote (`KEEP_CASE`).
Written from live career sites on 2026-09-29 (`docs/plan-hosts/workday.md`).
The listing carries no description, so `detail` reads each role's own JSON.
Countries and remote are filtered on the server by the tenant's own facets
(`Filters(country="ids", remote="ids")`): `appliedFacets` takes
`{<facetParameter>: [<id>, ...]}`, ids within one facet are a union, and two
facets are an intersection, which is why the runner asks for remote in a
series of its own. Each tenant names its facets its own way (LEGO's
`Location` lists countries, others nest `locationCountry` under
`locationMainGroup`), so `facets` reads them off the first page and picks the
one whose values are countries. A host id is `<facetParameter>=<id>`. LEGO
answered 97 of 455 for Denmark on 2026-09-30; a role at several places is
kept when one of them is in the country.
"""
from __future__ import annotations
@@ -26,7 +37,8 @@ from typing import Any
from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads
from ._contract import Ask, Fields, Filters, Place, Reads
from ._countries import code_of
from ._shared import Get, Page, Row, only, parse_dt
log = logging.getLogger(__name__)
@@ -37,7 +49,7 @@ KEEP_CASE = True
READS = Reads(
scope="board", listing="paged", page_size=20, ceiling=2000, text="detail",
complete="stated", places="many",
complete="stated", places="many", filters=Filters(country="ids", remote="ids"),
fields=Fields(posted="trusted", remote="stated", country="stated", employment="stated"),
robots="read: robots.txt allows /<Site>/ (checked 2026-09-29)",
)
@@ -104,7 +116,8 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
offset, total = cursor or (0, None)
data = json.loads(get(
f"https://{_host(slug)}/wday/cxs/{tenant}/{site}/jobs", method="POST",
payload={"appliedFacets": {}, "limit": PAGE_SIZE, "offset": offset, "searchText": ""}))
payload={"appliedFacets": _applied(ask), "limit": PAGE_SIZE, "offset": offset,
"searchText": ""}))
if cursor is None:
stated = data.get("total")
total = stated if isinstance(stated, int) and not isinstance(stated, bool) else None
@@ -118,6 +131,69 @@ def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
return data, results, total, (following, total) if more else None
def _applied(ask: Ask) -> dict[str, list[str]]:
"""`appliedFacets` for the host ids `ask` carries, `<facetParameter>=<id>`
each, grouped by facet."""
out: dict[str, list[str]] = {}
for value in (*ask.countries, *ask.remote_ids):
facet, sep, value_id = value.partition("=")
if sep and facet and value_id:
out.setdefault(facet, []).append(value_id)
return out
#: A facet whose values are countries: at least this share of them name one.
_COUNTRY_SHARE = 0.8
def _facet_lists(facets: Any) -> list[tuple[str, list[dict[str, Any]]]]:
"""`(facetParameter, values)` of every facet of an answer, the facets nested
in a group (`locationMainGroup` holds `locationCountry` on some tenants)
included."""
out: list[tuple[str, list[dict[str, Any]]]] = []
for facet in facets if isinstance(facets, list) else []:
if not isinstance(facet, dict):
continue
values = [v for v in facet.get("values") or [] if isinstance(v, dict)]
nested = [v for v in values if "values" in v and "facetParameter" in v]
out += _facet_lists(nested)
plain = [v for v in values if v not in nested]
if plain and str(facet.get("facetParameter") or ""):
out.append((str(facet["facetParameter"]), plain))
return out
def facets(get: Get, slug: str) -> dict[str, dict[str, str]]:
"""The tenant's host ids, read off one page of one role: `country` maps
an ISO code to `<facetParameter>=<id>` in the facet whose values are
countries (most of them, `_COUNTRY_SHARE`; the one with most countries
when two are), and `remote` maps each facet value that `_REMOTE` reads as
remote ("Remote", "Fully Flexible"), in whatever facet, to its id. A
tenant with neither states no such dimension, and the runner then does
not filter on it."""
tenant, _dc, site = _parts(slug) # type: ignore[misc]
data = json.loads(get(
f"https://{_host(slug)}/wday/cxs/{tenant}/{site}/jobs", method="POST",
payload={"appliedFacets": {}, "limit": 1, "offset": 0, "searchText": ""}))
country: dict[str, str] = {}
remote: dict[str, str] = {}
for name, values in _facet_lists(data.get("facets")):
found = {code: f"{name}={v['id']}" for v in values
if v.get("id") and (code := code_of(str(v.get("descriptor") or "")))}
if len(found) >= _COUNTRY_SHARE * len(values) and len(found) > len(country):
country = found
for v in values:
word = " ".join(str(v.get("descriptor") or "").split())
if v.get("id") and _remote(word) == "remote":
remote[word.lower()] = f"{name}={v['id']}"
out: dict[str, dict[str, str]] = {}
if country:
out["country"] = country
if remote:
out["remote"] = remote
return out
def _posting_id(posting: dict[str, Any], position: int) -> str:
return str(posting.get("externalPath") or "".join(posting.get("bulletFields") or [])
or f"#{position}")