Rename plugins/atshosts to plugins/readers
Every import, test and live doc follows the new path; behaviour is unchanged. tests/test_atshosts.py becomes tests/test_readers.py. plan-readers.md and review-ingest.md keep the old name, since they describe the state before. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
af337ef558
commit
73dec9028e
48 files changed
+104
-104
No files matched your search
@@ -0,0 +1,233 @@
|
||||
"""Workday: `workday:<tenant>.<wdN>/<Site>`, for example
|
||||
`workday:lego.wd103/LEGO_External` (the career site
|
||||
https://lego.wd103.myworkdayjobs.com/LEGO_External), paged by offset with a
|
||||
total that only the first page states.
|
||||
|
||||
The tenant is what the company was named when Workday was set up and is not
|
||||
always the company (`unitytech`, `novonesis`); the data centre is not
|
||||
derivable either, and a tenant has several sites, so all three are stored
|
||||
as they are found. The site is case-insensitive for the API (Red Hat's
|
||||
`jobs` and `Jobs` both work) but its robots.txt paths are not, so the slug
|
||||
keeps the case the URL wrote (`KEEP_CASE`).
|
||||
|
||||
Written from live career sites on 2026-09-29 (`docs/plan-hosts/workday.md`).
|
||||
The listing carries no description, so `detail` reads each role's own JSON.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._shared import Get, Page, Row, only, parse_dt
|
||||
|
||||
NAME = "workday"
|
||||
LABEL = "Workday"
|
||||
KEEP_CASE = True
|
||||
#: A tenant answers `/jobs` and `/Jobs` alike, so a rule in either case
|
||||
#: forbids the site.
|
||||
ROBOTS_ANY_CASE = True
|
||||
|
||||
#: One group for `<tenant>.<wdN>` and one for the site, which discovery joins
|
||||
#: into the slug with "/". A locale segment (`/en-US/`, `/de-DE/`) may come
|
||||
#: between; it is matched case-sensitively, and a bare `/en-US` that is no
|
||||
#: site at all is refused, so a site is never mistaken for a locale or the
|
||||
#: other way round. `wday/` is the API path, read by the second pattern. The
|
||||
#: tenant is lower case only: a slug with capitals in it is no slug.
|
||||
_TENANT = r"(?-i:[a-z0-9-]+)"
|
||||
_LOCALE = r"(?-i:[a-z]{2}-[A-Z]{2})"
|
||||
PATTERNS = (
|
||||
rf"(?<![A-Za-z0-9.-])({_TENANT}\.wd\d+)\.myworkdayjobs\.com/(?:{_LOCALE}/)?"
|
||||
rf"(?!wday/)(?!{_LOCALE}(?![A-Za-z0-9_-]))([A-Za-z0-9_-]+)",
|
||||
rf"(?<![A-Za-z0-9.-])({_TENANT}\.wd\d+)\.myworkdayjobs\.com/wday/cxs/{_TENANT}/"
|
||||
r"([A-Za-z0-9_-]+)",
|
||||
)
|
||||
|
||||
#: The page size asked for, which is also the only one the host serves: 21 or
|
||||
#: more answers 400 on every tenant tried.
|
||||
PAGE_SIZE = 20
|
||||
|
||||
#: Matched whole, so a trailing newline is no slug.
|
||||
_SLUG = re.compile(r"([a-z0-9-]+)\.(wd\d+)/([A-Za-z0-9_-]+)")
|
||||
|
||||
#: The path of one role on its site, `externalPath` in the listing
|
||||
#: (`/job/...`): a path, and nothing that adds a query or a fragment.
|
||||
_ROLE_PATH = re.compile(r"/[^\s?#]+")
|
||||
|
||||
|
||||
def _parts(slug: str) -> tuple[str, str, str] | None:
|
||||
"""`(tenant, data centre, site)`, or None for a slug that is none."""
|
||||
match = _SLUG.fullmatch(slug)
|
||||
return match.groups() if match else None # type: ignore[return-value]
|
||||
|
||||
|
||||
def _host(slug: str) -> str:
|
||||
tenant, dc, _site = _parts(slug) or ("", "", "")
|
||||
return f"{tenant}.{dc}.myworkdayjobs.com"
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
return _parts(slug) is not None
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
"""Only the tenant's own host. A slug that is none admits nothing, rather
|
||||
than a host the company does not own."""
|
||||
return only(_host(slug)) if _parts(slug) else (lambda host: False)
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
return f"https://{_host(slug)}/{_parts(slug)[2]}" # type: ignore[index]
|
||||
|
||||
|
||||
def robots_page(slug: str) -> str:
|
||||
"""With the trailing slash, since the rules the tenants write are
|
||||
`Disallow: /<Site>/` and read as prefixes: `/<Site>` alone would slip
|
||||
past the very rule that forbids the site."""
|
||||
return careers_url(slug) + "/"
|
||||
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any) -> Page:
|
||||
"""`cursor` is `(offset, total)`. `total` is read on the first page only:
|
||||
on the later ones it is 0 on most tenants and the real figure on a few,
|
||||
so it is carried. An offset at or past the end does not answer an empty
|
||||
list but the first page again, so the read stops on `offset >= total`
|
||||
rather than on an empty page."""
|
||||
tenant, _dc, site = _parts(slug) # type: ignore[misc]
|
||||
offset, total = cursor or (0, None)
|
||||
data = json.loads(get(
|
||||
f"https://{_host(slug)}/wday/cxs/{tenant}/{site}/jobs", method="POST",
|
||||
payload={"appliedFacets": {}, "limit": PAGE_SIZE, "offset": offset, "searchText": ""}))
|
||||
if cursor is None:
|
||||
stated = data.get("total")
|
||||
total = stated if isinstance(stated, int) and not isinstance(stated, bool) else None
|
||||
postings = data.get("jobPostings") or []
|
||||
# Every posting is counted, a path-less one (LEGO lists one that has only
|
||||
# `bulletFields`) by its position, so the parser dropping it does not
|
||||
# leave the board short of its total.
|
||||
results = [{"id": _posting_id(p, offset + i)} for i, p in enumerate(postings)]
|
||||
following = offset + len(postings)
|
||||
more = bool(postings) and (total is None or following < total)
|
||||
return data, results, total, (following, total) if more else None
|
||||
|
||||
|
||||
def _posting_id(posting: dict[str, Any], position: int) -> str:
|
||||
return str(posting.get("externalPath") or "".join(posting.get("bulletFields") or [])
|
||||
or f"#{position}")
|
||||
|
||||
|
||||
def _posted(text: str) -> datetime | None:
|
||||
"""The day of "Posted Today", "Posted Yesterday" and "Posted 5 Days Ago"
|
||||
at midnight UTC; None for "Posted 30+ Days Ago", which says only that it
|
||||
is older, and for a text in another language. Workday counts the days in
|
||||
its own time zone, so the day can be one off; `detail` states the date
|
||||
itself and replaces this one."""
|
||||
match = re.fullmatch(r"Posted (?:(Today)|(Yesterday)|(\d+) Days? Ago)", (text or "").strip())
|
||||
if not match:
|
||||
return None
|
||||
days = 0 if match.group(1) else 1 if match.group(2) else int(match.group(3))
|
||||
return _today() - timedelta(days=days)
|
||||
|
||||
|
||||
def _today() -> datetime:
|
||||
return datetime.now(UTC).replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
|
||||
|
||||
#: `remoteType` -> `store._remote`'s words. Only some tenants send it, and
|
||||
#: each words it its own way (Zendesk's "Fully Flexible", Axis' "On-Site"); a
|
||||
#: word not listed, a bare "No" among them, says nothing and stays unknown.
|
||||
_REMOTE = {"remote": "remote", "fully flexible": "remote", "hybrid": "hybrid",
|
||||
"in office": "onsite", "on-site": "onsite", "onsite": "onsite"}
|
||||
|
||||
|
||||
def _requisition(posting: dict[str, Any], path: str) -> str:
|
||||
"""The requisition id, which `bulletFields` lead with, if the path ends in
|
||||
it (`..._R-059566`, and `..._R-059566-2` for a repost of it). Where a
|
||||
tenant puts something else first, a place, the path is the id."""
|
||||
first = next(iter(posting.get("bulletFields") or []), None)
|
||||
name = path.rsplit("/", 1)[-1]
|
||||
return first if isinstance(first, str) and re.fullmatch(
|
||||
rf".*_{re.escape(first)}(?:-\d+)?", name) else ""
|
||||
|
||||
|
||||
def _remote(value: Any) -> str | None:
|
||||
return _REMOTE.get(str(value or "").strip().lower())
|
||||
|
||||
|
||||
def rows(decoded: dict[str, Any], slug: str) -> list[Row]:
|
||||
"""One page of `.../wday/cxs/<tenant>/<Site>/jobs` -> `{"total",
|
||||
"jobPostings": [...], "facets": [...]}`. A posting has `title`,
|
||||
`externalPath` (`/job/<place>/<title>_<requisition id>`), `locationsText`
|
||||
(a place, or "2 Locations"), `postedOn` (relative text) and
|
||||
`bulletFields`, which lead with the requisition id, and on some tenants
|
||||
`remoteType`. No description, no date and no country: `detail` has
|
||||
those. The id is the requisition id where the path carries it, and the
|
||||
path where not; a posting with no title or no path is skipped."""
|
||||
out = []
|
||||
for p in decoded.get("jobPostings") or []:
|
||||
title = (p.get("title") or "").strip()
|
||||
path = (p.get("externalPath") or "").strip()
|
||||
if not title or not _ROLE_PATH.fullmatch(path):
|
||||
continue
|
||||
out.append({
|
||||
"source_id": _requisition(p, path) or path,
|
||||
"title": title,
|
||||
"location": (p.get("locationsText") or "").strip(),
|
||||
"url": f"https://{_host(slug)}/{_parts(slug)[2]}{path}", # type: ignore[index]
|
||||
"posted": _posted(p.get("postedOn") or ""),
|
||||
"department": "",
|
||||
"description": "",
|
||||
"remote": _remote(p.get("remoteType")),
|
||||
"salary_text": "", # no field for pay, only the description's text
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
"""The role's own JSON, `.../wday/cxs/<tenant>/<Site><externalPath>`. The
|
||||
path is taken back off the row's URL, which was built from it. Pay is in
|
||||
the description text when it is anywhere: there is no field for it on
|
||||
any tenant. `startDate` is the posting date (it matched the "Posted N
|
||||
Days Ago" text in every sample), not a start of employment."""
|
||||
tenant, _dc, site = _parts(slug) # type: ignore[misc]
|
||||
prefix = f"https://{_host(slug)}/{site}"
|
||||
path = str(row.get("url") or "").removeprefix(prefix)
|
||||
if path == row.get("url") or not _ROLE_PATH.fullmatch(path) or ".." in path.split("/"):
|
||||
raise ValueError(f"{row.get('url')!r} is not a role of {slug}")
|
||||
info = json.loads(get(f"https://{_host(slug)}/wday/cxs/{tenant}/{site}{path}"))
|
||||
posting = info.get("jobPostingInfo")
|
||||
if not isinstance(posting, dict):
|
||||
raise ValueError("not a Workday role: no jobPostingInfo")
|
||||
out: dict[str, Any] = {}
|
||||
if description := " ".join(html.unescape(strip_html(posting.get("jobDescription") or ""))
|
||||
.split()):
|
||||
out["description"] = description
|
||||
if location := _location(posting):
|
||||
out["location"] = location
|
||||
if posted := parse_dt(posting.get("startDate")):
|
||||
out["posted"] = posted
|
||||
if remote := _remote(posting.get("remoteType")):
|
||||
out["remote"] = remote
|
||||
return out
|
||||
|
||||
|
||||
def _location(posting: dict[str, Any]) -> str:
|
||||
"""The main location and the others, joined by "; " as Teamtailor does, so
|
||||
that `country_of` reads them one at a time and the listing's "2 Locations"
|
||||
is replaced by places. A single location that does not name its country
|
||||
(LEGO's "Billund") gets it from the role's `country`."""
|
||||
places: list[str] = []
|
||||
for place in [posting.get("location"), *(posting.get("additionalLocations") or [])]:
|
||||
place = str(place or "").strip()
|
||||
if place and place not in places:
|
||||
places.append(place)
|
||||
country = str((posting.get("country") or {}).get("descriptor") or "").strip()
|
||||
if len(places) == 1 and country and country.lower() not in places[0].lower():
|
||||
places[0] += f", {country}"
|
||||
return "; ".join(places)
|
||||
Reference in new issue
Block a user