Read Phenom career sites through their search widget, with the description from each role page's JSON-LD
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
2b1dc08714
commit
8ee6f9b546
4 files changed
+532
No files matched your search
@@ -0,0 +1,197 @@
|
||||
"""Phenom: `phenom:<host>/<country>/<lang>`, the career site a company runs
|
||||
on its own domain, at the locale root it uses: `phenom:careers.king.com/us/en`,
|
||||
`phenom:www.careers.philips.com/global/en`. A JSON search that pages by offset
|
||||
and states its total.
|
||||
|
||||
The listing is the `POST /widgets` call the site's own search page makes. It
|
||||
answered without a cookie, an `Origin` header, a `pageId` or the site's
|
||||
`refNum`, so the body carries none of them: only the locale (`lang`,
|
||||
`country`, both read off the slug's path) and the paging. Nothing is taken
|
||||
from the page's script. Written from live sites on 2026-09-29 (King, Just Eat
|
||||
Takeaway, Philips); jobhunt-hq did not read Phenom.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._shared import Get, Page, Row, only, parse_dt
|
||||
|
||||
NAME = "phenom"
|
||||
LABEL = "Phenom"
|
||||
PATTERNS = (
|
||||
# A link to the site's search page, the one URL of a Phenom site that carries
|
||||
# both the host and the locale root and is named the same on every one.
|
||||
r"https?://([a-z0-9-]+(?:\.[a-z0-9-]+)*\.[a-z][a-z0-9-]*)/([a-z]{2,8})/([a-z]{2})/search-results",
|
||||
# No slug: a page that loads Phenom's own assets, on a site whose locale
|
||||
# root discovery has not seen. The host is known, the board is not.
|
||||
r"phenompeople\.com|phenom\.com/careers",
|
||||
)
|
||||
|
||||
#: The page size asked for. 100 is what the site's own search sends; `size`
|
||||
#: 200 was answered in full by Just Eat Takeaway and 500 by Philips (of 865),
|
||||
#: but a page is 340 KB at 100 roles already.
|
||||
PAGE_SIZE = 100
|
||||
|
||||
#: A DNS name with a dot and a letter first in its last label, so an address
|
||||
#: (`127.0.0.1`) or a single word (`localhost`) never names a site the fetch
|
||||
#: could be pointed at; `ats._get` still refuses local names and every host
|
||||
#: but this one, redirects included. Then the locale root: `country/lang`,
|
||||
#: which every Phenom site read so far writes as `global/en`, `na/en`, `us/en`.
|
||||
_SLUG = re.compile(r"[a-z0-9-]+(?:\.[a-z0-9-]+)*\.[a-z][a-z0-9-]*/[a-z]{2,8}/[a-z]{2}")
|
||||
|
||||
|
||||
def _parts(slug: str) -> tuple[str, str, str]:
|
||||
"""`(host, country, lang)` of a slug that passed `slug_ok`."""
|
||||
host, country, lang = slug.split("/")
|
||||
return host, country, lang
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
return bool(_SLUG.fullmatch(slug))
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return only(_parts(slug)[0])
|
||||
|
||||
|
||||
def _root(slug: str) -> str:
|
||||
host, country, lang = _parts(slug)
|
||||
return f"https://{host}/{country}/{lang}"
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
return f"{_root(slug)}/search-results"
|
||||
|
||||
|
||||
def robots_page(slug: str) -> str:
|
||||
return careers_url(slug)
|
||||
|
||||
|
||||
def _body(slug: str, offset: int) -> dict[str, Any]:
|
||||
"""What the site's search page sends for a listing of every role, newest
|
||||
first, less the parts that only tell the page what to draw. `lang` is
|
||||
`en_global` for `/global/en` and `en_us` for `/us/en`, the site's own
|
||||
spelling on the three sites read."""
|
||||
_host, country, lang = _parts(slug)
|
||||
return {
|
||||
"lang": f"{lang}_{country}", "deviceType": "desktop", "country": country,
|
||||
"pageName": "search-results", "ddoKey": "refineSearch", "sortBy": "",
|
||||
"subsearch": "", "from": offset, "jobs": True, "counts": True, "all_fetch": False,
|
||||
"pageType": "landingPage", "size": PAGE_SIZE, "clearAll": False,
|
||||
"jdsource": "facets", "isSliderEnable": False, "siteType": "external",
|
||||
"location": "", "keywords": "", "global": True, "selected_fields": {},
|
||||
"sort": {"order": "desc", "field": "postedDate"}, "locationData": {},
|
||||
}
|
||||
|
||||
|
||||
def _key(job: dict[str, Any]) -> str:
|
||||
"""The role's id on the board: `jobSeqNo`, which carries the site and the
|
||||
language and is the one of the three ids (`jobSeqNo`, `jobId`, `reqId`)
|
||||
that the site's job page takes on every site read. `reqId` is not unique
|
||||
across languages."""
|
||||
return str(job.get("jobSeqNo") or job.get("jobId") or job.get("reqId") or "").strip()
|
||||
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any) -> Page:
|
||||
offset = int(cursor or 0)
|
||||
host = _parts(slug)[0]
|
||||
data = json.loads(get(f"https://{host}/widgets", method="POST",
|
||||
payload=_body(slug, offset)))
|
||||
search = data.get("refineSearch")
|
||||
if not isinstance(search, dict) or search.get("status", 200) != 200:
|
||||
raise ValueError("not a Phenom search answer: no refineSearch")
|
||||
jobs = (search.get("data") or {}).get("jobs") or []
|
||||
results = [{"id": _key(job)} for job in jobs]
|
||||
return search, results, search.get("totalHits"), offset + len(jobs) if jobs else None
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
return rows(json.loads(body).get("refineSearch") or {}, slug)
|
||||
|
||||
|
||||
def _text(value: Any) -> str:
|
||||
return value.strip() if isinstance(value, str) else ""
|
||||
|
||||
|
||||
def _plain(value: Any) -> str:
|
||||
return html.unescape(strip_html(_text(value))).replace("\xa0", " ").strip()
|
||||
|
||||
|
||||
def _slug_of(title: str) -> str:
|
||||
"""The decorative last part of a role's URL, its title with runs of
|
||||
anything but letters and digits as one dash; it matches the site's own
|
||||
(checked on all 32 roles of King's sitemap). The page is found by the id."""
|
||||
return re.sub(r"[^A-Za-z0-9]+", "-", title).strip("-")
|
||||
|
||||
|
||||
def rows(search: dict[str, Any], slug: str) -> list[Row]:
|
||||
"""The roles of one page, `refineSearch.data.jobs`. A role has the title,
|
||||
`jobSeqNo`, `postedDate` (a date at midnight), `category`, the locations
|
||||
as `multi_location` (a list, one per place) beside `location` (the first),
|
||||
and `descriptionTeaser`, the first paragraph of the text; the full text is
|
||||
on the role's page, see `detail`. `applyUrl` points at the apply system
|
||||
(Workday for Just Eat Takeaway and Philips), so the role's URL is its page
|
||||
on the site. No remote flag and no pay are in the list."""
|
||||
out = []
|
||||
for job in search.get("data", {}).get("jobs") or []:
|
||||
key = _key(job)
|
||||
title = _text(job.get("title"))
|
||||
if not title or not key:
|
||||
continue
|
||||
places: list[str] = []
|
||||
for place in (*(job.get("multi_location") or []), job.get("location")):
|
||||
if isinstance(place, str) and place.strip() and place.strip() not in places:
|
||||
places.append(place.strip())
|
||||
tail = _slug_of(title)
|
||||
out.append({
|
||||
"source_id": key,
|
||||
"title": title,
|
||||
"location": "; ".join(places),
|
||||
"url": f"{_root(slug)}/job/{key}" + (f"/{tail}" if tail else ""),
|
||||
"posted": parse_dt(job.get("postedDate")),
|
||||
"department": _text(job.get("category")),
|
||||
"description": _plain(job.get("descriptionTeaser")),
|
||||
"remote": None, # the list does not say
|
||||
"salary_text": "", # nor does it state pay
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
#: The JSON-LD blocks of a page; a Phenom job page has one `JobPosting` and,
|
||||
#: on Philips's, a `BreadcrumbList` and more.
|
||||
_JSON_LD = re.compile(r"<script[^>]+application/ld\+json[^>]*>(.*?)</script>", re.S | re.I)
|
||||
|
||||
|
||||
def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
"""The full description from the role's page: the `JobPosting` in its
|
||||
JSON-LD, whose `description` is HTML with its entities escaped (`<p>`
|
||||
and `&` inside), so it is unescaped, stripped and unescaped again.
|
||||
`datePosted` is taken as `posted`: it can be earlier than the list's
|
||||
`postedDate` (Philips's list said the day of the read for a role whose page
|
||||
said 2026-09-08), and a role keeps the earliest date it ever had."""
|
||||
body = get(row["url"], headers={"Accept": "text/html"}).decode("utf-8", "replace")
|
||||
posting: dict[str, Any] = {}
|
||||
for block in _JSON_LD.findall(body):
|
||||
try:
|
||||
found = json.loads(block)
|
||||
except ValueError:
|
||||
continue
|
||||
for candidate in found if isinstance(found, list) else [found]:
|
||||
if isinstance(candidate, dict) and candidate.get("@type") == "JobPosting":
|
||||
posting = candidate
|
||||
break
|
||||
if posting:
|
||||
break
|
||||
out: dict[str, Any] = {}
|
||||
if description := _plain(html.unescape(_text(posting.get("description")))):
|
||||
out["description"] = description
|
||||
if posted := parse_dt(posting.get("datePosted")):
|
||||
out["posted"] = posted
|
||||
return out
|
||||
Reference in new issue
Block a user