From 254645c29773b61378974c087b82e0c2998828c5 Mon Sep 17 00:00:00 2001 From: Wolf Date: Tue, 29 Sep 2026 20:10:11 +0200 Subject: [PATCH] Read SuccessFactors career sites by their search pages and role microdata Co-Authored-By: Claude Sonnet 5.5 --- plugins/atshosts/successfactors.py | 403 +++++++++++++++++++++++++++++ 1 file changed, 403 insertions(+) create mode 100644 plugins/atshosts/successfactors.py diff --git a/plugins/atshosts/successfactors.py b/plugins/atshosts/successfactors.py new file mode 100644 index 0000000..e63b199 --- /dev/null +++ b/plugins/atshosts/successfactors.py @@ -0,0 +1,403 @@ +"""SAP SuccessFactors career sites (Recruiting Marketing): `successfactors:` +for the career site's own host name (`careers.vestas.com`), or +`successfactors:/` for the part of a shared site that is the +company (`jobs.teradyne.com/MiR`, Universal Robots inside Teradyne). Paged HTML +with a stated total. + +Written from live sites on 2026-09-29 (Vestas, Demant, Scania, SAP, Novo +Nordisk, Systematic and Teradyne); jobhunt-hq did not read SuccessFactors. + +Only the career site's front end is read, and only the paths its robots.txt +allows: `/search/` and `/job/`. Never `/services/` (the RSS feed, disallowed), +and never the `career*.successfactors.*` or `*.sapsf.*` back end, whose +robots.txt disallows everything; `allowed` admits the site's own host and no +other, so a slug cannot point a fetch there. A company that has only the back +end (Bang & Olufsen) is not readable. On these sites any unknown `*.xml` path +answers with a feed of several megabytes, so no path but those two is asked. + +The listing is the search page, `/search/?q=&locationsearch=&startrow=N`, +which states a total and lists title, location and, on most sites, a date. It +comes in two layouts, a table and tiles, both parsed by `rows`. The page size +is the site's own (10, 15, 25 or 100) and cannot be set, so the next start row +is the rows the last page returned. `sitemap.xml` is not used: it lists every +locale, so a role written in two languages is there twice, and it carries no +title or date. + +**The search lists the default locale only.** A role that exists only in +another locale (Carlsberg's French brand, 37 of 105) is not on the board this +module reads, and neither is a translation of a listed role. + +A path prefix names a brand of a shared site. `//search/` lists +only that brand on the server (Universal Robots 30 of Teradyne's 486, MiR 1), +so the read is as small as the company; `rows` still keeps only the roles +under `//job/`, in case a site does not scope. There is no server-side +country filter in this version. + +A role's own page carries microdata (`itemtype` schema.org JobPosting) with +`datePosted` and the address; `detail` reads the description, the date and the +location with its country from it. Novo Nordisk's pages have none: the +description is still in `itemprop="description"`, and the listing's date and +location stand. +""" + +from __future__ import annotations + +import html +import re +from collections.abc import Callable +from datetime import UTC, datetime +from typing import Any +from urllib.parse import urljoin, urlsplit + +from desk.sources import strip_html + +from ._shared import Get, Page, Row, is_web_url, only, parse_dt + +NAME = "successfactors" +LABEL = "SuccessFactors" +# A career site is on the company's own domain and names no SuccessFactors +# host. Discovery can therefore read a board only off a page that links a +# role, `/job//<id>` with the numeric requisition id, and it takes the +# host and a brand segment from that link. A page that only links the back end +# (`career5.successfactors.eu`, as Bang & Olufsen's does) or loads the sites' +# image CDN names the host with no slug, which keeps the company on the list +# of hosts the app names but has no board for. +PATTERNS = ( + r"https?://([a-z0-9-]+(?:\.[a-z0-9-]+)+)(?:/([A-Za-z0-9_-]+))?/job/[^/\s\"'<>?#]+/\d{9,10}/", + r"(?:career|jobs|performancemanager)\d*\.successfactors\.(?:eu|com)|\.sapsf\.(?:eu|com)" + r"|rmkcdn\.successfactors\.com", +) +# The prefix is case-sensitive (`MiR`, `Universal-Robots`), and so are the +# robots.txt rules that name it. +KEEP_CASE = True + +#: A public DNS name in lower case with a letter-only last label, so an address +#: (`127.0.0.1`) or a single word (`localhost`) never names a site, and an +#: optional prefix of one or more path segments. `ats._get` refuses the local +#: names again at every request. +_SLUG = re.compile(r"((?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z]{2,})((?:/[A-Za-z0-9_-]+)*)") +_LOCAL = (".local", ".localhost", ".internal", ".lan", ".home.arpa", ".arpa", ".test", + ".example", ".invalid") + +#: The search pages' own layouts. A page that has none of these is not a +#: search page (a bot wall or an error page answered with 200), which must not +#: read as a board with no roles. +_SEARCH_PAGE = re.compile( + r'paginationLabel|tile-search-results-label|id="searchresults"|job-tile-list|class="data-row"') + +#: The table layout: a row is `<tr class="data-row">`, with the job link +#: (twice: once for the phone), `span.jobLocation`, `span.jobDate` and, on some +#: sites, `span.jobDepartment`. The tile layout: `<li class="job-tile job-id-N">` and one labelled field per column. +_ROW = re.compile(r'<tr\b[^>]*\bclass="[^"]*\bdata-row\b') +_TILE = re.compile(r'<li\b[^>]*\bclass="job-tile[ "]') +_LINK = re.compile( + r'<a\b[^>]*\bhref="((?:https?://[^"/]+)?(?:/[^"/]+)*/job/[^"]*?/(\d+)/?)"[^>]*>(.*?)</a>', re.S) +_SPAN = r'<span\b[^>]*\bclass="(?:[^"]* )?{}(?: [^"]*)?"[^>]*>(.*?)</span>' +_TILE_VALUE = re.compile(r'id="job-\d+-desktop-section-([A-Za-z0-9]+)-value"[^>]*>(.*?)</div>', re.S) +_TILE_LABEL = re.compile(r'id="job-\d+-desktop-section-([A-Za-z0-9]+)-label"[^>]*>(.*?)</span>', re.S) + +#: The total: `Results <b>1 to 10</b> of <b>661</b>` (table) or `Showing 1 to 10 of +#: 308 Jobs` (tiles); the last number of the label either way. +_TABLE_TOTAL = re.compile(r'paginationLabel[^>]*>(.*?)</span>', re.S) +_TILE_TOTAL = re.compile(r'id="tile-search-results-label"[^>]*>(.*?)</span>', re.S) + +#: What a tile's labels call the place and its country, lower case. Demant +#: gives the city as "Location" and the country name as "Country"; Scania and +#: Carlsberg give a "Location" that ends in a country code. +_PLACE_LABELS = ("location", "city", "town") +_COUNTRY_LABELS = ("country", "land") + +_MONTHS = {name: number for number, name in enumerate( + ("jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"), 1)} + +#: ISO 3166-1 alpha-2 to the English name, which is what `country_of` reads: +#: a bare "DE" or "SE" in a location line is taken for Delaware or nothing. A +#: code that is not here stays as the site wrote it. +_COUNTRIES = dict(entry.strip().split(" ", 1) for entry in """ +AD Andorra | AE United Arab Emirates | AF Afghanistan | AG Antigua and Barbuda | AI +Anguilla | AL Albania | AM Armenia | AO Angola | AR Argentina | AS American Samoa | AT +Austria | AU Australia | AW Aruba | AX Åland Islands | AZ Azerbaijan | BA Bosnia and +Herzegovina | BB Barbados | BD Bangladesh | BE Belgium | BF Burkina Faso | BG Bulgaria +| BH Bahrain | BI Burundi | BJ Benin | BM Bermuda | BN Brunei | BO Bolivia | BQ +Caribbean Netherlands | BR Brazil | BS Bahamas | BT Bhutan | BW Botswana | BY Belarus +| BZ Belize | CA Canada | CD Democratic Republic of the Congo | CF Central African +Republic | CG Republic of the Congo | CH Switzerland | CI Côte d'Ivoire | CK Cook +Islands | CL Chile | CM Cameroon | CN China | CO Colombia | CR Costa Rica | CU Cuba | +CV Cabo Verde | CW Curaçao | CY Cyprus | CZ Czechia | DE Germany | DJ Djibouti | DK +Denmark | DM Dominica | DO Dominican Republic | DZ Algeria | EC Ecuador | EE Estonia | +EG Egypt | ER Eritrea | ES Spain | ET Ethiopia | FI Finland | FJ Fiji | FK Falkland +Islands | FM Micronesia | FO Faroe Islands | FR France | GA Gabon | GB United Kingdom +| GD Grenada | GE Georgia | GF French Guiana | GG Guernsey | GH Ghana | GI Gibraltar | +GL Greenland | GM Gambia | GN Guinea | GP Guadeloupe | GQ Equatorial Guinea | GR +Greece | GT Guatemala | GU Guam | GW Guinea-Bissau | GY Guyana | HK Hong Kong | HN +Honduras | HR Croatia | HT Haiti | HU Hungary | ID Indonesia | IE Ireland | IL Israel +| IM Isle of Man | IN India | IQ Iraq | IR Iran | IS Iceland | IT Italy | JE Jersey | +JM Jamaica | JO Jordan | JP Japan | KE Kenya | KG Kyrgyzstan | KH Cambodia | KI +Kiribati | KM Comoros | KN Saint Kitts and Nevis | KP North Korea | KR South Korea | +KW Kuwait | KY Cayman Islands | KZ Kazakhstan | LA Laos | LB Lebanon | LC Saint Lucia +| LI Liechtenstein | LK Sri Lanka | LR Liberia | LS Lesotho | LT Lithuania | LU +Luxembourg | LV Latvia | LY Libya | MA Morocco | MC Monaco | MD Moldova | ME +Montenegro | MG Madagascar | MH Marshall Islands | MK North Macedonia | ML Mali | MM +Myanmar | MN Mongolia | MO Macao | MP Northern Mariana Islands | MQ Martinique | MR +Mauritania | MS Montserrat | MT Malta | MU Mauritius | MV Maldives | MW Malawi | MX +Mexico | MY Malaysia | MZ Mozambique | NA Namibia | NC New Caledonia | NE Niger | NG +Nigeria | NI Nicaragua | NL Netherlands | NO Norway | NP Nepal | NR Nauru | NU Niue | +NZ New Zealand | OM Oman | PA Panama | PE Peru | PF French Polynesia | PG Papua New +Guinea | PH Philippines | PK Pakistan | PL Poland | PR Puerto Rico | PS Palestine | PT +Portugal | PW Palau | PY Paraguay | QA Qatar | RE Réunion | RO Romania | RS Serbia | +RU Russia | RW Rwanda | SA Saudi Arabia | SB Solomon Islands | SC Seychelles | SD +Sudan | SE Sweden | SG Singapore | SI Slovenia | SK Slovakia | SL Sierra Leone | SM +San Marino | SN Senegal | SO Somalia | SR Suriname | SS South Sudan | ST São Tomé and +Príncipe | SV El Salvador | SY Syria | SZ Eswatini | TC Turks and Caicos Islands | TD +Chad | TG Togo | TH Thailand | TJ Tajikistan | TL Timor-Leste | TM Turkmenistan | TN +Tunisia | TO Tonga | TR Turkey | TT Trinidad and Tobago | TV Tuvalu | TW Taiwan | TZ +Tanzania | UA Ukraine | UG Uganda | US United States | UY Uruguay | UZ Uzbekistan | VA +Vatican City | VC Saint Vincent and the Grenadines | VE Venezuela | VG British Virgin +Islands | VI U.S. Virgin Islands | VN Vietnam | VU Vanuatu | WS Samoa | XK Kosovo | YE +Yemen | ZA South Africa | ZM Zambia | ZW Zimbabwe +""".replace("\n", " ").split(" | ")) + + +def _parts(slug: str) -> tuple[str, str]: + """`(host, "/prefix" or "")` of a slug.""" + match = _SLUG.fullmatch(slug) + return (match.group(1), match.group(2)) if match else ("", "") + + +def slug_ok(slug: str) -> bool: + host, _prefix = _parts(slug) + return bool(host) and not host.endswith(_LOCAL) + + +def allowed(slug: str) -> Callable[[str], bool]: + return only(_parts(slug)[0]) + + +def careers_url(slug: str) -> str: + host, prefix = _parts(slug) + return f"https://{host}{prefix}/search/" + + +def robots_page(slug: str) -> str: + return careers_url(slug) + + +def _search_url(slug: str, start: int) -> str: + host, prefix = _parts(slug) + return f"https://{host}{prefix}/search/?q=&locationsearch=&startrow={start}" + + +# ------------------------------------------------------------------ listing + +def _text(fragment: str) -> str: + return html.unescape(strip_html(fragment)) + + +def _date(text: str) -> datetime | None: + """A listing's date as the site writes it in its language: "29 Sept 2026", + "Sep 29, 2026", "2026/09/29" or "29.09.2026". A form that could be day or + month first ("09/29/2026") is not guessed at and reads as no date.""" + text = _text(text) + if match := re.search(r"(\d{1,2})\.?\s+([A-Za-z]{3})[A-Za-z]*\.?,?\s+(\d{4})", text): + day, month, year = match.groups() + elif match := re.search(r"([A-Za-z]{3})[A-Za-z]*\.?\s+(\d{1,2}),?\s+(\d{4})", text): + month, day, year = match.groups() + elif match := re.search(r"(\d{4})[-/.](\d{1,2})[-/.](\d{1,2})", text): + year, month, day = match.groups() + elif match := re.search(r"(\d{1,2})\.(\d{1,2})\.(\d{4})", text): + day, month, year = match.groups() + else: + return None + number = _MONTHS.get(month.lower()[:3]) or (int(month) if month.isdigit() else 0) + try: + return datetime(int(year), number, int(day), tzinfo=UTC) + except ValueError: + return None + + +def _place(text: str) -> str: + """A site's location, "City, Region, CC, zip", as "City, Region, Country": + the two-letter token near the end is the country and what follows it the + postcode, which is dropped. The last such token wins, since a region can be + one too ("Boston, MA, US"). A line without one is left as it is.""" + text = re.sub(r"\s+", " ", text).strip() + parts = [part.strip() for part in text.split(",")] + for i in range(len(parts) - 1, 0, -1): + if len(parts) - i <= 2 and re.fullmatch(r"[A-Z]{2}", parts[i]) and parts[i] in _COUNTRIES: + return ", ".join([*parts[:i], _COUNTRIES[parts[i]]]) + return text + + +def _total(text: str) -> int | None: + for pattern in (_TABLE_TOTAL, _TILE_TOTAL): + if match := pattern.search(text): + numbers = re.findall(r"\d[\d.,\u00a0 ]*", _text(match.group(1))) + if numbers and (digits := re.sub(r"\D", "", numbers[-1])): + return int(digits) + return None + + +def _table_fields(chunk: str) -> tuple[str, str, str]: + """`(place, date, department)` of a table row, from the spans the site marks + them with.""" + def cell(name: str) -> str: + found = re.search(_SPAN.format(name), chunk, re.S) + return _text(found.group(1)) if found else "" + return cell("jobLocation"), cell("jobDate"), cell("jobDepartment") + + +def _tile_fields(chunk: str) -> tuple[str, str, str]: + """`(place, date, department)` of a tile. The fields differ per company, so + the place and the country are found by their label and the date and the + department by the field's own name, which is `date` and `department` on + every site seen (Scania's "Work Area" is `department`).""" + keyed = dict(_TILE_VALUE.findall(chunk)) + labelled = {_text(label).lower(): _text(keyed[key]) + for key, label in _TILE_LABEL.findall(chunk) if key in keyed} + place = next((labelled[label] for label in _PLACE_LABELS if labelled.get(label)), + _text(keyed.get("location", ""))) + country = next((labelled[label] for label in _COUNTRY_LABELS if labelled.get(label)), "") + place = _place(place) + if country and country.lower() not in place.lower(): + place = f"{place}, {country}" if place else country + return place, _text(keyed.get("date", "")), _text(keyed.get("department", "")) + + +def _entries(text: str) -> list[dict[str, Any]]: + """The roles of one search page, in the order shown, whichever layout it is + in and before the prefix filter: `id`, `href` (as written, relative), + `title`, `place`, `date` and `department`. A role is listed once, and a row + without a link to a role is not one.""" + table = _ROW.split(text)[1:] + chunks = table or _TILE.split(text)[1:] + out: list[dict[str, Any]] = [] + seen: set[str] = set() + for chunk in chunks: + if table: + chunk = chunk.split("</tr>", 1)[0] + if not (link := _LINK.search(chunk)): + continue + href, job_id, title = html.unescape(link.group(1)), link.group(2), _text(link.group(3)) + if job_id in seen: + continue + seen.add(job_id) + place, date, department = (_table_fields if table else _tile_fields)(chunk) + out.append({"id": job_id, "href": href, "title": title, "place": _place(place), + "date": date, "department": department}) + return out + + +def page(get: Get, slug: str, cursor: Any) -> Page: + """One search page. The cursor is `(start row, first role of the page + before)`: the next start row is what this page returned, and a page that + starts with the same role as the last one is the site ignoring `startrow`, + which ends the read (incomplete) instead of looping.""" + start, previous = cursor or (0, None) + text = get(_search_url(slug, start)).decode("utf-8", "replace") + if not _SEARCH_PAGE.search(text): + raise ValueError("not a SuccessFactors search page") + found = _entries(text) + total = _total(text) + if found and found[0]["id"] == previous: + return text, [], total, None + more = bool(found) and (total is None or start + len(found) < total) + return (text, [{"id": entry["id"]} for entry in found], total, + (start + len(found), found[0]["id"]) if more else None) + + +def rows(decoded: str, slug: str) -> list[Row]: + """One search page's roles. `href` is on the site's own host, under the + slug's prefix when it has one; the id in it is the requisition's `source_id`. + The table and the tile layout give a title, a location, a date and, where + the site has the column, a department; nothing else about a role is on + the list, so its description and pay are empty.""" + host, prefix = _parts(slug) + base = f"https://{host}/" + out = [] + for entry in _entries(decoded): + url = urljoin(base, entry["href"]) + if urlsplit(url).hostname != host or not is_web_url(url): + continue + if prefix and not urlsplit(url).path.lower().startswith(f"{prefix.lower()}/job/"): + continue + if not entry["title"]: + continue + out.append({ + "source_id": entry["id"], + "title": entry["title"], + "location": entry["place"], + "url": url, + "posted": _date(entry["date"]), + "department": entry["department"], + "description": "", + "remote": None, # the listing does not say + "salary_text": "", # nor does it state pay + }) + return out + + +# ------------------------------------------------------------------- detail + +_META = re.compile(r'<meta\b[^>]*\bitemprop="([A-Za-z]+)"[^>]*\bcontent="([^"]*)"', re.I) +_JAVA_DATE = re.compile(r"[A-Za-z]{3} ([A-Za-z]{3}) (\d{1,2}) (\d\d:\d\d:\d\d) (?:UTC|GMT) (\d{4})") + + +def _element(text: str, start: int) -> str: + """The inner HTML of the element whose start tag begins at `start`, up to + its matching close tag: nested elements of the same name are counted.""" + open_end = text.index(">", start) + 1 + name = re.match(r"<([A-Za-z0-9]+)", text[start:]).group(1) # type: ignore[union-attr] + depth = 1 + for tag in re.finditer(rf"<(/?){name}\b[^>]*>", text[open_end:], re.I): + depth += -1 if tag.group(1) else 0 if tag.group(0).endswith("/>") else 1 + if not depth: + return text[open_end:open_end + tag.start()] + return text[open_end:] + + +def _posted(value: str) -> datetime | None: + """`datePosted` is a Java `Date.toString()`, "Tue Sep 29 02:00:00 UTC 2026", + on every site seen; an ISO date is read as well.""" + if match := _JAVA_DATE.fullmatch(value.strip()): + month, day, clock, year = match.groups() + return parse_dt(f"{year}-{_MONTHS[month.lower()]:02d}-{int(day):02d} {clock}") + return parse_dt(value) + + +def _location(meta: dict[str, str]) -> str: + """The address as "City, Country". Vestas and Demant state + `addressLocality` and `addressCountry` (a code), the other sites one + `streetAddress`, "City, CC, zip". The region is left out: the sites cut it + to a few letters ("Seou" for Seoul-teukpyolsi).""" + country = meta.get("addressCountry", "").strip() + country = _COUNTRIES.get(country.upper(), country) + if city := meta.get("addressLocality", "").strip(): + return f"{city}, {country}" if country else city + if street := meta.get("streetAddress", "").strip(): + return _place(street) + return country + + +def detail(get: Get, slug: str, row: Row) -> dict[str, Any]: + """A role's own page: `description` from the block marked + `itemprop="description"`, which Novo Nordisk's pages have without the + microdata around it, and, from a page that has the `JobPosting` microdata, + `posted` (`datePosted`) and `location` with its country. A page that + is not a role's (the site answers 200 for a closed one with its search) has + no description block and gives an empty answer.""" + text = get(row["url"]).decode("utf-8", "replace") + out: dict[str, Any] = {} + match = re.search(r'<(?:span|div)\b[^>]*\bitemprop="description"', text) + if match: + # An employer's own layout can put a style block in the description. + block = re.sub(r"<(style|script)\b.*?</\1>", " ", _element(text, match.start()), + flags=re.S | re.I) + if description := _text(block): + out["description"] = description + if re.search(r'itemtype="https?://schema\.org/JobPosting"', text): + meta = {name: html.unescape(value) for name, value in _META.findall(text)} + if posted := _posted(meta.get("datePosted", "")): + out["posted"] = posted + if place := _location(meta): + out["location"] = place + return out