Read a Phenom role's description from the page state when the page has no JobPosting, and fail a page with neither

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Sonnet 5.5 committed 2026-09-29 20:10:52 +02:00
1 parent 8f0db38393
commit cf182ad921
3 files changed
+77 -17

No files matched your search

+46 -14
View File
@@ -169,15 +169,14 @@ def rows(search: dict[str, Any], slug: str) -> list[Row]:
_JSON_LD = re.compile(r"<script[^>]+application/ld\+json[^>]*>(.*?)</script>", re.S | re.I)
def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
"""The full description from the role's page: the `JobPosting` in its
JSON-LD, whose `description` is HTML with its entities escaped (`&lt;p&gt;`
and `&amp;` inside), so it is unescaped, stripped and unescaped again.
`datePosted` is taken as `posted`: it can be earlier than the list's
`postedDate` (Philips's list said the day of the read for a role whose page
said 2026-09-08), and a role keeps the earliest date it ever had."""
body = get(row["url"], headers={"Accept": "text/html"}).decode("utf-8", "replace")
posting: dict[str, Any] = {}
#: Where a page assigns its state, and the words its `checkRemote` says that
#: `store._remote` reads (King's pages say "On-site"; no other was seen).
_STATE = "phApp.ddo = "
_REMOTE = {"remote": "remote", "hybrid": "hybrid", "on-site": "onsite", "onsite": "onsite"}
def _job_posting(body: str) -> dict[str, Any]:
"""The `JobPosting` among a page's JSON-LD blocks, or {}."""
for block in _JSON_LD.findall(body):
try:
found = json.loads(block)
@@ -185,13 +184,46 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
continue
for candidate in found if isinstance(found, list) else [found]:
if isinstance(candidate, dict) and candidate.get("@type") == "JobPosting":
posting = candidate
break
if posting:
break
return candidate
return {}
def _page_job(body: str) -> dict[str, Any]:
"""The role as the page's own state holds it, `phApp.ddo.jobDetail.data.job`,
or {}. King's pages of roles posted the same day had no `JobPosting`, and
this had the full description. Only that one object is read: the state
around it holds the site's session values, which nothing here uses."""
start = body.find(_STATE)
if start < 0:
return {}
try:
state, _end = json.JSONDecoder().raw_decode(body, start + len(_STATE))
job = state["jobDetail"]["data"]["job"]
except (ValueError, KeyError, TypeError):
return {}
return job if isinstance(job, dict) else {}
def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
"""The full description from the role's page: the `JobPosting` in its
JSON-LD, whose `description` is HTML with its entities escaped (`&lt;p&gt;`
and `&amp;` inside), so it is unescaped, stripped and unescaped again; or,
where the page has none, the page state's own `description`, which is
plain HTML. A page with neither raises, so the role is asked again on a
later pull instead of counting as read with no text. `datePosted` is taken
as `posted`: it can be earlier than the list's `postedDate` (Philips's list
said the day of the read for a role whose page said 2026-09-08), and a role
keeps the earliest date it ever had. King's page state says `checkRemote`."""
body = get(row["url"], headers={"Accept": "text/html"}).decode("utf-8", "replace")
posting, job = _job_posting(body), _page_job(body)
out: dict[str, Any] = {}
if description := _plain(html.unescape(_text(posting.get("description")))):
if description := (_plain(html.unescape(_text(posting.get("description"))))
or _plain(job.get("description"))):
out["description"] = description
else:
raise ValueError("no description on the role's page")
if posted := parse_dt(posting.get("datePosted")):
out["posted"] = posted
if remote := _REMOTE.get(_text(job.get("checkRemote")).lower()):
out["remote"] = remote
return out