Files
jobboard/plugins/readers/successfactors.py
T
WolfandClaude Opus 5.5 c497fe965d Keep a description's headings, bullets and paragraphs, and draw them in the reader
The readers flattened a posting's HTML into one line with strip_html, so
sections and bullets ran together. _shared.html_text gives a line per
paragraph, '• ' before a list item and '## ' before a heading; every
reader's description goes through it, sections read per role get their
own headings, and the reader pane draws the lines as h3, ul and p.
Excerpts leave the marks out.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 21:40:12 +02:00

402 lines
19 KiB
Python

"""SAP SuccessFactors career sites (Recruiting Marketing): `successfactors:<host>`
for the career site's own host name (`careers.vestas.com`), or
`successfactors:<host>/<path prefix>` for the part of a shared site that is the
company (`jobs.teradyne.com/MiR`, Universal Robots inside Teradyne). Paged HTML
with a stated total.
Written from live sites on 2026-09-29 (Vestas, Demant, Scania, SAP, Novo
Nordisk, Systematic and Teradyne); jobhunt-hq did not read SuccessFactors.
Only the career site's front end is read, and only the paths its robots.txt
allows: `/search/` and `/job/`. Never `/services/` (the RSS feed, disallowed),
and never the `career*.successfactors.*` or `*.sapsf.*` back end, whose
robots.txt disallows everything; `allowed` admits the site's own host and no
other, so a slug cannot point a fetch there. A company that has only the back
end (Bang & Olufsen) is not readable. On these sites any unknown `*.xml` path
answers with a feed of several megabytes, so no path but those two is asked.
The listing is the search page, `/search/?q=&locationsearch=&startrow=N`,
which states a total and lists title, location and, on most sites, a date. It
comes in two layouts, a table and tiles, both parsed by `rows`. The page size
is the site's own (10, 15, 25 or 100) and cannot be set, so the next start row
is the rows the last page returned. `sitemap.xml` is not used: it lists every
locale, so a role written in two languages is there twice, and it carries no
title or date.
**The search lists the default locale only.** A role that exists only in
another locale (Carlsberg's French brand, 37 of 105) is not on the board this
module reads, and neither is a translation of a listed role.
A path prefix names a brand of a shared site. `<host>/<prefix>/search/` lists
only that brand on the server (Universal Robots 30 of Teradyne's 486, MiR 1),
so the read is as small as the company; `rows` still keeps only the roles
under `/<prefix>/job/`, in case a site does not scope.
A country is filtered on the server by the search's own facet,
`optionsFacetsDD_country=DK`, one country a request (`Filters(country="one")`);
it worked on the four tenants tried in the review and on Vestas (64 of 668,
2026-09-30). A role at several places is listed when one of them is in the
country. There is no remote facet.
A role's own page carries microdata (`itemtype` schema.org JobPosting) with
`datePosted` and the address; `detail` reads the description, the date and the
places with their country from it; a role at several places repeats the
`jobLocation` block, once per place. Novo Nordisk's pages have none: the
description is still in `itemprop="description"`, and the listing's date and
location stand.
A fetch term goes as the search form's `q=engineer`, one a request
(`Filters(query="one")`); with `optionsFacetsDD_country=DK` it is an AND
(survey, 2026-09-30: 40 of Vestas' 64 Danish roles).
"""
from __future__ import annotations
import html
import re
from collections.abc import Callable
from datetime import UTC, datetime
from typing import Any
from urllib.parse import quote, urljoin, urlsplit
from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads
from ._countries import NAMES
from ._shared import Get, Page, Row, add_place, html_text, is_web_url, only, parse_dt
NAME = "successfactors"
LABEL = "SuccessFactors"
# A career site is on the company's own domain and names no SuccessFactors
# host, so discovery can read a board only off an absolute link to one of its
# own paths that no other site has: a role, `/job/<title>/<numeric id>/`, a
# landing page, `/go/<name>/<numeric id>/`, or a search with the query keys the
# sites' own forms send (`createNewAlert`, `locationsearch`, `optionsFacetsDD_`).
# The host and a brand segment before the path are the slug. A page that only
# links the back end (`career5.successfactors.eu`, as Bang & Olufsen's does) or
# loads the sites' image CDN names the host with no slug, which keeps the
# company on the list of hosts the app names but has no board for.
PATTERNS = (
r"https?://([a-z0-9-]+(?:\.[a-z0-9-]+)+)(?:/([A-Za-z0-9_-]+))?"
r"/(?:job/[^/\s\"'<>?#]+/\d{9,10}/|go/[^/\s\"'<>?#]+/\d{6,}/"
r"|search/\?[^\s\"'<>]*(?:createNewAlert|locationsearch|optionsFacetsDD_))",
r"(?:career|jobs|performancemanager)\d*\.successfactors\.(?:eu|com)|\.sapsf\.(?:eu|com)"
r"|rmkcdn\.successfactors\.com",
)
# The prefix is case-sensitive (`MiR`, `Universal-Robots`), and so are the
# robots.txt rules that name it.
KEEP_CASE = True
READS = Reads(
scope="board", listing="paged", text="detail", complete="stated", places="many",
filters=Filters(country="one", query="one"),
fields=Fields(posted="trusted"),
robots="read: robots.txt allows /search/ and /job/ (checked 2026-09-29)",
)
#: A public DNS name in lower case with a letter-only last label, so an address
#: (`127.0.0.1`) or a single word (`localhost`) never names a site, and an
#: optional prefix of one or more path segments. `ats._get` refuses the local
#: names again at every request.
_SLUG = re.compile(r"((?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z]{2,})((?:/[A-Za-z0-9_-]+)*)")
_LOCAL = (".local", ".localhost", ".internal", ".lan", ".home.arpa", ".arpa", ".test",
".example", ".invalid")
#: The search pages' own layouts. A page that has none of these is not a
#: search page (a bot wall or an error page answered with 200), which must not
#: read as a board with no roles.
_SEARCH_PAGE = re.compile(
r'paginationLabel|tile-search-results-label|id="searchresults"|job-tile-list|class="data-row"')
#: The table layout: a row is `<tr class="data-row">`, with the job link
#: (twice: once for the phone), `span.jobLocation`, `span.jobDate` and, on some
#: sites, `span.jobDepartment`. The tile layout: `<li class="job-tile job-id-N">` and one labelled field per column.
_ROW = re.compile(r'<tr\b[^>]*\bclass="[^"]*\bdata-row\b')
_TILE = re.compile(r'<li\b[^>]*\bclass="job-tile[ "]')
_LINK = re.compile(
r'<a\b[^>]*\bhref="((?:https?://[^"/]+)?(?:/[^"/]+)*/job/[^"]*?/(\d+)/?)"[^>]*>(.*?)</a>', re.S)
_SPAN = r'<span\b[^>]*\bclass="(?:[^"]* )?{}(?: [^"]*)?"[^>]*>(.*?)</span>'
_TILE_VALUE = re.compile(r'id="job-\d+-desktop-section-([A-Za-z0-9]+)-value"[^>]*>(.*?)</div>', re.S)
_TILE_LABEL = re.compile(r'id="job-\d+-desktop-section-([A-Za-z0-9]+)-label"[^>]*>(.*?)</span>', re.S)
#: The total: `Results <b>1 to 10</b> of <b>661</b>` (table) or `Showing 1 to 10 of
#: 308 Jobs` (tiles); the last number of the label either way.
_TABLE_TOTAL = re.compile(r'paginationLabel[^>]*>(.*?)</span>', re.S)
_TILE_TOTAL = re.compile(r'id="tile-search-results-label"[^>]*>(.*?)</span>', re.S)
#: What a tile's labels call the place and its country, lower case. Demant
#: gives the city as "Location" and the country name as "Country"; Scania and
#: Carlsberg give a "Location" that ends in a country code.
_PLACE_LABELS = ("location", "city", "town")
_COUNTRY_LABELS = ("country", "land")
_MONTHS = {name: number for number, name in enumerate(
("jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"), 1)}
#: ISO 3166-1 alpha-2 to the English name (`_countries.NAMES`): a bare "DE"
#: or "SE" in a location line is taken for Delaware or nothing. A code that
#: is not there stays as the site wrote it.
_COUNTRIES = NAMES
def _parts(slug: str) -> tuple[str, str]:
"""`(host, "/prefix" or "")` of a slug."""
match = _SLUG.fullmatch(slug)
return (match.group(1), match.group(2)) if match else ("", "")
def slug_ok(slug: str) -> bool:
host, _prefix = _parts(slug)
return bool(host) and not host.endswith(_LOCAL)
def allowed(slug: str) -> Callable[[str], bool]:
return only(_parts(slug)[0])
def careers_url(slug: str) -> str:
host, prefix = _parts(slug)
return f"https://{host}{prefix}/search/"
def _search_url(slug: str, start: int, country: str = "", query: str = "") -> str:
"""The search page from `start`, in one country when `country` (an ISO
code) is given, matching `query` (the form's `q`) when given."""
host, prefix = _parts(slug)
facet = f"&optionsFacetsDD_country={country}" if re.fullmatch(r"[A-Z]{2}", country) else ""
return (f"https://{host}{prefix}/search/?q={quote(query, safe='')}&locationsearch="
f"{facet}&startrow={start}")
# ------------------------------------------------------------------ listing
def _text(fragment: str) -> str:
return html.unescape(strip_html(fragment))
def _date(text: str) -> datetime | None:
"""A listing's date as the site writes it in its language: "29 Sept 2026",
"Sep 29, 2026", "2026/09/29" or "29.09.2026". A form that could be day or
month first ("09/29/2026") is not guessed at and reads as no date."""
text = _text(text)
if match := re.search(r"(\d{1,2})\.?\s+([A-Za-z]{3})[A-Za-z]*\.?,?\s+(\d{4})", text):
day, month, year = match.groups()
elif match := re.search(r"([A-Za-z]{3})[A-Za-z]*\.?\s+(\d{1,2}),?\s+(\d{4})", text):
month, day, year = match.groups()
elif match := re.search(r"(\d{4})[-/.](\d{1,2})[-/.](\d{1,2})", text):
year, month, day = match.groups()
elif match := re.search(r"(\d{1,2})\.(\d{1,2})\.(\d{4})", text):
day, month, year = match.groups()
else:
return None
number = _MONTHS.get(month.lower()[:3]) or (int(month) if month.isdigit() else 0)
try:
return datetime(int(year), number, int(day), tzinfo=UTC)
except ValueError:
return None
def _place(text: str) -> str:
"""A site's location, "City, Region, CC, zip", as "City, Region, Country":
the two-letter token near the end is the country and what follows it the
postcode, which is dropped. The last such token wins, since a region can be
one too ("Boston, MA, US"). A line without one is left as it is."""
text = re.sub(r"\s+", " ", text).strip()
parts = [part.strip() for part in text.split(",")]
for i in range(len(parts) - 1, 0, -1):
if len(parts) - i <= 2 and re.fullmatch(r"[A-Z]{2}", parts[i]) and parts[i] in _COUNTRIES:
return ", ".join([*parts[:i], _COUNTRIES[parts[i]]])
return text
def _total(text: str) -> int | None:
for pattern in (_TABLE_TOTAL, _TILE_TOTAL):
if match := pattern.search(text):
numbers = re.findall(r"\d[\d.,\u00a0 ]*", _text(match.group(1)))
if numbers and (digits := re.sub(r"\D", "", numbers[-1])):
return int(digits)
return None
def _table_fields(chunk: str) -> tuple[str, str, str]:
"""`(place, date, department)` of a table row, from the spans the site marks
them with."""
def cell(name: str) -> str:
found = re.search(_SPAN.format(name), chunk, re.S)
return _text(found.group(1)) if found else ""
return cell("jobLocation"), cell("jobDate"), cell("jobDepartment")
def _tile_fields(chunk: str) -> tuple[str, str, str]:
"""`(place, date, department)` of a tile. The fields differ per company, so
the place and the country are found by their label and the date and the
department by the field's own name, which is `date` and `department` on
every site seen (Scania's "Work Area" is `department`)."""
keyed = dict(_TILE_VALUE.findall(chunk))
labelled = {_text(label).lower(): _text(keyed[key])
for key, label in _TILE_LABEL.findall(chunk) if key in keyed}
place = next((labelled[label] for label in _PLACE_LABELS if labelled.get(label)),
_text(keyed.get("location", "")))
country = next((labelled[label] for label in _COUNTRY_LABELS if labelled.get(label)), "")
place = _place(place)
if country and country.lower() not in place.lower():
place = f"{place}, {country}" if place else country
return place, _text(keyed.get("date", "")), _text(keyed.get("department", ""))
def _entries(text: str) -> list[dict[str, Any]]:
"""The roles of one search page, in the order shown, whichever layout it is
in and before the prefix filter: `id`, `href` (as written, relative),
`title`, `place`, `date` and `department`. A role is listed once, and a row
without a link to a role is not one."""
table = _ROW.split(text)[1:]
chunks = table or _TILE.split(text)[1:]
out: list[dict[str, Any]] = []
seen: set[str] = set()
for chunk in chunks:
if table:
chunk = chunk.split("</tr>", 1)[0]
if not (link := _LINK.search(chunk)):
continue
href, job_id, title = html.unescape(link.group(1)), link.group(2), _text(link.group(3))
if job_id in seen:
continue
seen.add(job_id)
place, date, department = (_table_fields if table else _tile_fields)(chunk)
out.append({"id": job_id, "href": href, "title": title, "place": _place(place),
"date": date, "department": department})
return out
def page(get: Get, slug: str, cursor: Any, ask: Ask = Ask()) -> Page:
"""One search page, in the one country `ask` names, matching its `query`. The cursor is `(start row, first role of the page
before)`: the next start row is what this page returned, and a page that
starts with the same role as the last one is the site ignoring `startrow`,
which ends the read (incomplete) instead of looping."""
start, previous = cursor or (0, None)
country = ask.countries[0] if ask.countries else ""
text = get(_search_url(slug, start, country, ask.query)).decode("utf-8", "replace")
if not _SEARCH_PAGE.search(text):
raise ValueError("not a SuccessFactors search page")
found = _entries(text)
total = _total(text)
if found and found[0]["id"] == previous:
return text, [], total, None
more = bool(found) and (total is None or start + len(found) < total)
return (text, [{"id": entry["id"]} for entry in found], total,
(start + len(found), found[0]["id"]) if more else None)
def rows(decoded: str, slug: str) -> list[Row]:
"""One search page's roles. `href` is on the site's own host, under the
slug's prefix when it has one; the id in it is the requisition's `source_id`.
The table and the tile layout give a title, a location, a date and, where
the site has the column, a department; nothing else about a role is on
the list, so its description and pay are empty."""
host, prefix = _parts(slug)
base = f"https://{host}/"
out = []
for entry in _entries(decoded):
url = urljoin(base, entry["href"])
if urlsplit(url).hostname != host or not is_web_url(url):
continue
if prefix and not urlsplit(url).path.lower().startswith(f"{prefix.lower()}/job/"):
continue
if not entry["title"]:
continue
out.append({
"source_id": entry["id"],
"title": entry["title"],
"places": [{"text": entry["place"], "remote": None}] if entry["place"] else [],
"location": entry["place"],
"url": url,
"posted": _date(entry["date"]),
"department": entry["department"],
"description": "",
"remote": None, # the listing does not say
"salary_text": "", # nor does it state pay
})
return out
# ------------------------------------------------------------------- detail
_META = re.compile(r'<meta\b[^>]*\bitemprop="([A-Za-z]+)"[^>]*\bcontent="([^"]*)"', re.I)
_JAVA_DATE = re.compile(r"[A-Za-z]{3} ([A-Za-z]{3}) (\d{1,2}) (\d\d:\d\d:\d\d) (?:UTC|GMT) (\d{4})")
def _element(text: str, start: int) -> str:
"""The inner HTML of the element whose start tag begins at `start`, up to
its matching close tag: nested elements of the same name are counted."""
open_end = text.index(">", start) + 1
name = re.match(r"<([A-Za-z0-9]+)", text[start:]).group(1) # type: ignore[union-attr]
depth = 1
for tag in re.finditer(rf"<(/?){name}\b[^>]*>", text[open_end:], re.I):
depth += -1 if tag.group(1) else 0 if tag.group(0).endswith("/>") else 1
if not depth:
return text[open_end:open_end + tag.start()]
return text[open_end:]
def _posted(value: str) -> datetime | None:
"""`datePosted` is a Java `Date.toString()`, "Tue Sep 29 02:00:00 UTC 2026",
on every site seen; an ISO date is read as well."""
if match := _JAVA_DATE.fullmatch(value.strip()):
month, day, clock, year = match.groups()
return parse_dt(f"{year}-{_MONTHS[month.lower()]:02d}-{int(day):02d} {clock}")
return parse_dt(value)
def _location(meta: dict[str, str]) -> str:
"""The address as "City, Country". Vestas and Demant state
`addressLocality` and `addressCountry` (a code), the other sites one
`streetAddress`, "City, CC, zip". The region is left out: the sites cut it
to a few letters ("Seou" for Seoul-teukpyolsi)."""
country = meta.get("addressCountry", "").strip()
country = _COUNTRIES.get(country.upper(), country)
if city := meta.get("addressLocality", "").strip():
return f"{city}, {country}" if country else city
if street := meta.get("streetAddress", "").strip():
return _place(street)
return country
def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
"""A role's own page: `description` from the block marked
`itemprop="description"`, which Novo Nordisk's pages have without the
microdata around it, and, from a page that has the `JobPosting` microdata,
`posted` (`datePosted`) and every place. A page that
is not a role's (the site answers 200 for a closed one with its search) has
no description block and gives an empty answer."""
text = get(row["url"]).decode("utf-8", "replace")
out: dict[str, Any] = {}
match = re.search(r'<(?:span|div)\b[^>]*\bitemprop="description"', text)
if match:
# An employer's own layout can put a style block in the description.
block = re.sub(r"<(style|script)\b.*?</\1>", " ", _element(text, match.start()),
flags=re.S | re.I)
if description := html_text(block):
out["description"] = description
if re.search(r'itemtype="https?://schema\.org/JobPosting"', text):
meta = {name: html.unescape(value) for name, value in _META.findall(text)}
if posted := _posted(meta.get("datePosted", "")):
out["posted"] = posted
addresses = _addresses(text) or [meta]
places: list[Place] = []
for address in addresses:
add_place(places, {"text": _location(address), "remote": None})
if places:
out["places"] = places
out["location"] = "; ".join(p["text"] for p in places)
return out
def _addresses(text: str) -> list[dict[str, str]]:
"""The microdata of each `jobLocation` block, one dict of `itemprop` ->
content per place. A role at several places repeats the block."""
found = []
for match in re.finditer(r'<(?:span|div)\b[^>]*\bitemprop="jobLocation"', text):
block = _element(text, match.start())
found.append({name: html.unescape(value) for name, value in _META.findall(block)})
return found