249 lines
11 KiB
Python
249 lines
11 KiB
Python
"""Phenom: `phenom:<host>/<country>/<lang>`, the career site a company runs
|
|
on its own domain, at the locale root it uses: `phenom:careers.king.com/us/en`,
|
|
`phenom:www.careers.philips.com/global/en`. A JSON search that pages by offset
|
|
and states its total.
|
|
|
|
The listing is the `POST /widgets` call the site's own search page makes. It
|
|
answered without a cookie, an `Origin` header, a `pageId` or the site's
|
|
`refNum`, so the body carries none of them: only the locale (`lang`,
|
|
`country`, both read off the slug's path) and the paging. Nothing is taken
|
|
from the page's script. Written from live sites on 2026-09-29 (King, Just Eat
|
|
Takeaway, Philips); jobhunt-hq did not read Phenom.
|
|
|
|
Countries are filtered on the server as a list of English names in
|
|
`selected_fields.country`, all in one request (`Filters(country="many")`); the
|
|
list is a union, and a role at several places is kept when one of them
|
|
matches. Philips answered 111 of 873 for Netherlands and Germany on
|
|
2026-09-30. There is no remote field to filter on. A code with no English
|
|
name in `_countries` is left out of the list, and a list left empty reads the
|
|
whole board.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import html
|
|
import json
|
|
import re
|
|
from collections.abc import Callable
|
|
from typing import Any
|
|
|
|
from desk.sources import strip_html
|
|
|
|
from ._contract import Ask, Fields, Filters, Place, Reads
|
|
from ._countries import name_of
|
|
from ._shared import Get, Page, Row, add_place, only, parse_dt
|
|
|
|
NAME = "phenom"
|
|
LABEL = "Phenom"
|
|
PATTERNS = (
|
|
# A link to the site's search page, the one URL of a Phenom site that carries
|
|
# both the host and the locale root and is named the same on every one.
|
|
r"https?://([a-z0-9-]+(?:\.[a-z0-9-]+)*\.[a-z][a-z0-9-]*)/([a-z]{2,8})/([a-z]{2})/search-results",
|
|
# No slug: a page that loads Phenom's own assets, on a site whose locale
|
|
# root discovery has not seen. The host is known, the board is not.
|
|
r"phenompeople\.com|phenom\.com/careers",
|
|
)
|
|
|
|
READS = Reads(
|
|
scope="board", listing="paged", page_size=100, text="detail", teaser=True,
|
|
complete="stated", places="many", filters=Filters(country="many"),
|
|
fields=Fields(posted="trusted", remote="stated"),
|
|
robots="read: robots.txt allows /<country>/<lang>/search-results (checked 2026-09-29)",
|
|
)
|
|
|
|
#: The page size asked for. 100 is what the site's own search sends; `size`
|
|
#: 200 was answered in full by Just Eat Takeaway and 500 by Philips (of 865),
|
|
#: but a page is 340 KB at 100 roles already.
|
|
PAGE_SIZE = 100
|
|
|
|
#: A DNS name with a dot and a letter first in its last label, so an address
|
|
#: (`127.0.0.1`) or a single word (`localhost`) never names a site the fetch
|
|
#: could be pointed at; `ats._get` still refuses local names and every host
|
|
#: but this one, redirects included. Then the locale root: `country/lang`,
|
|
#: which every Phenom site read so far writes as `global/en`, `na/en`, `us/en`.
|
|
_SLUG = re.compile(r"[a-z0-9-]+(?:\.[a-z0-9-]+)*\.[a-z][a-z0-9-]*/[a-z]{2,8}/[a-z]{2}")
|
|
|
|
|
|
def _parts(slug: str) -> tuple[str, str, str]:
|
|
"""`(host, country, lang)` of a slug that passed `slug_ok`."""
|
|
host, country, lang = slug.split("/")
|
|
return host, country, lang
|
|
|
|
|
|
def slug_ok(slug: str) -> bool:
|
|
return bool(_SLUG.fullmatch(slug))
|
|
|
|
|
|
def allowed(slug: str) -> Callable[[str], bool]:
|
|
return only(_parts(slug)[0])
|
|
|
|
|
|
def _root(slug: str) -> str:
|
|
host, country, lang = _parts(slug)
|
|
return f"https://{host}/{country}/{lang}"
|
|
|
|
|
|
def careers_url(slug: str) -> str:
|
|
return f"{_root(slug)}/search-results"
|
|
|
|
|
|
def _body(slug: str, offset: int, countries: tuple[str, ...] = ()) -> dict[str, Any]:
|
|
"""What the site's search page sends for a listing of every role, newest
|
|
first, less the parts that only tell the page what to draw, in the
|
|
`countries` (ISO codes) given. `lang` is `en_global` for `/global/en` and
|
|
`en_us` for `/us/en`, the site's own spelling on the three sites read."""
|
|
_host, country, lang = _parts(slug)
|
|
names = [name for code in countries if (name := name_of(code))]
|
|
return {
|
|
"lang": f"{lang}_{country}", "deviceType": "desktop", "country": country,
|
|
"pageName": "search-results", "ddoKey": "refineSearch", "sortBy": "",
|
|
"subsearch": "", "from": offset, "jobs": True, "counts": True, "all_fetch": False,
|
|
"pageType": "landingPage", "size": PAGE_SIZE, "clearAll": False,
|
|
"jdsource": "facets", "isSliderEnable": False, "siteType": "external",
|
|
"location": "", "keywords": "", "global": True,
|
|
"selected_fields": {"country": names} if names else {},
|
|
"sort": {"order": "desc", "field": "postedDate"}, "locationData": {},
|
|
}
|
|
|
|
|
|
def _key(job: dict[str, Any]) -> str:
|
|
"""The role's id on the board: `jobSeqNo`, which carries the site and the
|
|
language and is the one of the three ids (`jobSeqNo`, `jobId`, `reqId`)
|
|
that the site's job page takes on every site read. `reqId` is not unique
|
|
across languages."""
|
|
return str(job.get("jobSeqNo") or job.get("jobId") or job.get("reqId") or "").strip()
|
|
|
|
|
|
def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
|
|
offset = int(cursor or 0)
|
|
host = _parts(slug)[0]
|
|
data = json.loads(get(f"https://{host}/widgets", method="POST",
|
|
payload=_body(slug, offset, ask.countries)))
|
|
search = data.get("refineSearch")
|
|
if not isinstance(search, dict) or search.get("status", 200) != 200:
|
|
raise ValueError("not a Phenom search answer: no refineSearch")
|
|
jobs = (search.get("data") or {}).get("jobs") or []
|
|
results = [{"id": _key(job)} for job in jobs]
|
|
return search, results, search.get("totalHits"), offset + len(jobs) if jobs else None
|
|
|
|
|
|
def parse(body: bytes, slug: str) -> list[Row]:
|
|
return rows(json.loads(body).get("refineSearch") or {}, slug)
|
|
|
|
|
|
def _text(value: Any) -> str:
|
|
return value.strip() if isinstance(value, str) else ""
|
|
|
|
|
|
def _plain(value: Any) -> str:
|
|
return html.unescape(strip_html(_text(value))).replace("\xa0", " ").strip()
|
|
|
|
|
|
def _slug_of(title: str) -> str:
|
|
"""The decorative last part of a role's URL, its title with runs of
|
|
anything but letters and digits as one dash; it matches the site's own
|
|
(checked on all 32 roles of King's sitemap). The page is found by the id."""
|
|
return re.sub(r"[^A-Za-z0-9]+", "-", title).strip("-")
|
|
|
|
|
|
def rows(search: dict[str, Any], slug: str) -> list[Row]:
|
|
"""The roles of one page, `refineSearch.data.jobs`. A role has the title,
|
|
`jobSeqNo`, `dateCreated` (the posting) beside `postedDate` (a date at
|
|
midnight that the site moves when it refreshes a role: King's "12mth FTC"
|
|
role, created in June, says 28 September), `category`, the locations as
|
|
`multi_location` (a list, one per place) beside `location` (the first),
|
|
and `descriptionTeaser`, the first paragraph of the text, kept as `teaser`;
|
|
the full text is on the role's page, see `detail`. `applyUrl` points at the apply system
|
|
(Workday for Just Eat Takeaway and Philips), so the role's URL is its page
|
|
on the site. No remote flag and no pay are in the list."""
|
|
out = []
|
|
for job in search.get("data", {}).get("jobs") or []:
|
|
key = _key(job)
|
|
title = _text(job.get("title"))
|
|
if not title or not key:
|
|
continue
|
|
places: list[Place] = []
|
|
for text in (job.get("location"), *(job.get("multi_location") or [])):
|
|
if isinstance(text, str):
|
|
add_place(places, {"text": text.strip(), "remote": None})
|
|
tail = _slug_of(title)
|
|
out.append({
|
|
"source_id": key,
|
|
"title": title,
|
|
"places": places,
|
|
"location": "; ".join(p["text"] for p in places),
|
|
"url": f"{_root(slug)}/job/{key}" + (f"/{tail}" if tail else ""),
|
|
"posted": parse_dt(job.get("dateCreated") or job.get("postedDate")),
|
|
"department": _text(job.get("category")),
|
|
"description": "",
|
|
"teaser": _plain(job.get("descriptionTeaser")),
|
|
"remote": None, # the list does not say
|
|
"salary_text": "", # nor does it state pay
|
|
})
|
|
return out
|
|
|
|
|
|
#: The JSON-LD blocks of a page; a Phenom job page has one `JobPosting` and,
|
|
#: on Philips's, a `BreadcrumbList` and more.
|
|
_JSON_LD = re.compile(r"<script[^>]+application/ld\+json[^>]*>(.*?)</script>", re.S | re.I)
|
|
|
|
|
|
#: Where a page assigns its state, and the words its `checkRemote` says that
|
|
#: `store._remote` reads (King's pages say "On-site"; no other was seen).
|
|
_STATE = "phApp.ddo = "
|
|
_REMOTE = {"remote": "remote", "hybrid": "hybrid", "on-site": "onsite", "onsite": "onsite"}
|
|
|
|
|
|
def _job_posting(body: str) -> dict[str, Any]:
|
|
"""The `JobPosting` among a page's JSON-LD blocks, or {}."""
|
|
for block in _JSON_LD.findall(body):
|
|
try:
|
|
found = json.loads(block)
|
|
except ValueError:
|
|
continue
|
|
for candidate in found if isinstance(found, list) else [found]:
|
|
if isinstance(candidate, dict) and candidate.get("@type") == "JobPosting":
|
|
return candidate
|
|
return {}
|
|
|
|
|
|
def _page_job(body: str) -> dict[str, Any]:
|
|
"""The role as the page's own state holds it, `phApp.ddo.jobDetail.data.job`,
|
|
or {}. King's pages of roles posted the same day had no `JobPosting`, and
|
|
this had the full description. Only that one object is read: the state
|
|
around it holds the site's session values, which nothing here uses."""
|
|
start = body.find(_STATE)
|
|
if start < 0:
|
|
return {}
|
|
try:
|
|
state, _end = json.JSONDecoder().raw_decode(body, start + len(_STATE))
|
|
job = state["jobDetail"]["data"]["job"]
|
|
except (ValueError, KeyError, TypeError):
|
|
return {}
|
|
return job if isinstance(job, dict) else {}
|
|
|
|
|
|
def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
|
"""The full description from the role's page: the `JobPosting` in its
|
|
JSON-LD, whose `description` is HTML with its entities escaped (`<p>`
|
|
and `&` inside), so it is unescaped, stripped and unescaped again; or,
|
|
where the page has none, the page state's own `description`, which is
|
|
plain HTML. A page with neither raises, so the role is asked again on a
|
|
later pull instead of counting as read with no text. `datePosted` is taken
|
|
as `posted`: it can be earlier than the list's `postedDate` (Philips's list
|
|
said the day of the read for a role whose page said 2026-09-08), and a role
|
|
keeps the earliest date it ever had. King's page state says `checkRemote`."""
|
|
body = get(row["url"], headers={"Accept": "text/html"}).decode("utf-8", "replace")
|
|
posting, job = _job_posting(body), _page_job(body)
|
|
out: dict[str, Any] = {}
|
|
if description := (_plain(html.unescape(_text(posting.get("description"))))
|
|
or _plain(job.get("description"))):
|
|
out["description"] = description
|
|
else:
|
|
raise ValueError("no description on the role's page")
|
|
if posted := parse_dt(posting.get("datePosted")):
|
|
out["posted"] = posted
|
|
if remote := _REMOTE.get(_text(job.get("checkRemote")).lower()):
|
|
out["remote"] = remote
|
|
return out
|