Rename plugins/atshosts to plugins/readers
Every import, test and live doc follows the new path; behaviour is unchanged. tests/test_atshosts.py becomes tests/test_readers.py. plan-readers.md and review-ingest.md keep the old name, since they describe the state before. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
af337ef558
commit
73dec9028e
48 files changed
+104
-104
No files matched your search
@@ -0,0 +1,217 @@
|
||||
"""The hosts the `ats` plugin reads, one module each.
|
||||
|
||||
A host registers by existing: every module in this package whose name does
|
||||
not start with `_` is loaded when the package is imported, checked against
|
||||
the contract below, and listed in `HOSTS` in order of `(RANK, NAME)`. A
|
||||
module that breaks the contract fails the import with an ImportError naming
|
||||
the module and what it lacks, rather than a pull that fails board by board.
|
||||
`plugins/ats.py` builds its allow-list, feeds, pages, the Sources page's host
|
||||
select and its description from `HOSTS`, and `app/jobs/discover.py` its
|
||||
patterns and probes. The package sits beside `ats.py` rather than being
|
||||
`plugins/ats/`, since `desk.sources.registry` loads `plugins/*.py` by path and
|
||||
the tests patch names on the `ats` module.
|
||||
|
||||
A module is named after its host (`plugins/readers/workday.py`) and imports
|
||||
`desk` and `._shared` only, never `plugins.ats`. It holds:
|
||||
|
||||
NAME = "workday" # the host of a target `workday:<slug>`; equals the module's name
|
||||
LABEL = "Workday" # what the Sources page's select shows
|
||||
PATTERNS = (r"...",) # discovery: regexes over a URL, compiled with IGNORECASE
|
||||
RANK = 100 # optional int; lower comes first in the select and the probes
|
||||
KEEP_CASE = False # optional; True keeps a discovered slug's case
|
||||
ROBOTS_ANY_CASE = False # optional; True reads robots.txt paths without case
|
||||
|
||||
def slug_ok(slug: str) -> bool # strict, see below
|
||||
def allowed(slug: str) -> Callable[[str], bool] # the host names a fetch may reach
|
||||
def careers_url(slug: str) -> str # the page a person opens
|
||||
|
||||
and one of two ways to read a board:
|
||||
|
||||
def feed_url(slug: str) -> str # the whole board in one answer,
|
||||
def parse(body: bytes, slug: str) -> list[Row] # read with `parse`
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any) -> Page # a paged board, a page a call,
|
||||
def rows(decoded: Any, slug: str) -> list[Row] # its rows from `page`'s page
|
||||
|
||||
and, optionally:
|
||||
|
||||
def fallback_url(slug: str) -> str # feed hosts: a smaller feed, read
|
||||
# once when `feed_url` is over the cap
|
||||
def robots_page(slug: str) -> str # the URL robots.txt must allow
|
||||
def detail(get: Get, slug: str, row: Row) -> dict[str, Any] # from the role's page
|
||||
def probe_url(slug: str) -> str # discovery: 2xx only for a board
|
||||
def probe_ok(body: bytes) -> bool # ...and this, when 2xx is not enough
|
||||
def probe_slugs(guess: str) -> tuple[str, ...] # the slugs a guessed name is tried as
|
||||
|
||||
**Names.** `RANK` defaults to `DEFAULT_RANK` (100); the eight first hosts
|
||||
hold 10 to 80, which keeps the order they had before there were modules, so
|
||||
a new host comes after them unless it says otherwise. The order decides the
|
||||
select's default and which host wins when discovery's guessed slug exists on
|
||||
two.
|
||||
|
||||
**Slugs.** `slug_ok` matches the whole slug (`re.fullmatch`, never `^...$`,
|
||||
which lets a trailing newline through) and refuses anything that could add a
|
||||
path, a query, a fragment, a port or a user to the URLs built from it.
|
||||
`_shared.path_slug_ok` (one URL path segment) and `_shared.label_slug_ok`
|
||||
(one DNS label) cover the usual cases; a slug with several parts joins them
|
||||
with "/" (`lever:eu/tomtom`, `workday:lego.wd103/LEGO_External`).
|
||||
|
||||
**Patterns.** The slug is group 1, or, when a pattern has more than one
|
||||
group, the groups that matched joined by "/". Only group 1 is checked
|
||||
against discovery's words that name no board (`api`, `jobs`, `embed`, ...).
|
||||
The slug is lowercased unless `KEEP_CASE` is True: set it when the host's
|
||||
paths are case-sensitive (Workday's site), since robots.txt paths are too. A
|
||||
pattern without a group names the host with no slug. A host that
|
||||
`discover.py` names in its own `_UNREADABLE` table is taken from its module
|
||||
instead as soon as the module exists.
|
||||
|
||||
**Fetching.** `allowed(slug)` returns a predicate over a lowercased host
|
||||
name; `_shared.only(host)` is the usual one. Every request goes through
|
||||
`get`, which is `ats._get` bound to the board:
|
||||
|
||||
get(url, *, method="GET", payload=None, headers=None) -> bytes
|
||||
|
||||
A POST sends `payload` as JSON (`{}` when None); `headers` are added to the
|
||||
default User-Agent and `Accept: application/json, text/xml`, and replace
|
||||
either where they name it. `get` is https only on port 443, never to an IP
|
||||
address or a local name (`localhost`, `.local`, `.internal`, ...), follows at
|
||||
most three redirects each checked against `allowed(slug)`, cuts the body at
|
||||
`ats.MAX_BYTES` and ends the whole request within `ats.DEADLINE`. It returns
|
||||
the body and raises `httpx.HTTPStatusError` on a non-2xx answer (read
|
||||
`exc.response.status_code`), `httpx.HTTPError` on a timeout or a connection
|
||||
failure, and another exception (`ValueError` among them) for a refused host,
|
||||
an oversized body or a broken redirect. A module lets these through; the
|
||||
shared code logs, records and decides.
|
||||
|
||||
**Pages.** `Page` is `(decoded, results, total, cursor)`: the decoded page
|
||||
handed to `rows`, its raw results as dicts with an `id` (counted, by distinct
|
||||
id, to decide completeness), the total the host states or None, and the
|
||||
cursor of the next page or None after the last. The first call gets cursor
|
||||
None. A board with a total is complete when as many distinct ids were read;
|
||||
one without, when the read ended on an empty page. A 429 after the first page
|
||||
ends the read, incomplete; `ats.MAX_PAGES` pages at most, `ats.PAGE_DELAY`
|
||||
apart. A paged module may also have `parse(body, slug)` for one page's body.
|
||||
|
||||
**Rows.** A `Row` is a dict with `source_id` (the host's own id for the
|
||||
role, unique on the board), `title`, `location`, `url` (http or https,
|
||||
checked with `_shared.is_web_url`), `posted` (an aware datetime or None, see
|
||||
`_shared.parse_dt`), `department`, `description` (plain text, see
|
||||
`desk.sources.strip_html`), `remote` (True, False, a word or None) and
|
||||
`salary_text`; and, where the host states pay, `salary_min`, `salary_max`
|
||||
(yearly integers) and `currency` (ISO 4217), see `_shared.yearly_amount`.
|
||||
|
||||
**Detail.** `detail(get, slug, row)` reads one role's own page and returns a
|
||||
dict of the fields it found. The shared code calls it for roles not yet read
|
||||
(`ats:detailed:<target>`), at most `ats.DETAIL_CAP` (200) a board a pull,
|
||||
`ats.PAGE_DELAY` apart, the roles never tried before those that failed
|
||||
(`ats:detailfailed:<target>`). Only `ats.DETAIL_FIELDS` are taken:
|
||||
`description`, `location`, `remote`, `currency`, `salary_min`, `salary_max`
|
||||
and `posted`, the keys whose columns survive a later pull that sends the
|
||||
listing's empty value; any other key is left out with a debug log line.
|
||||
`title`, `url`, `department` and `salary_text` cannot be filled from
|
||||
`detail`: the listing rewrites them every pull. A role not read in detail
|
||||
on a pull is sent with an empty description, so the one read earlier stays.
|
||||
A 404 or a 410 counts as read; any other failure is logged and retried on a
|
||||
later pull; a 429 ends the detail reads of that board for the pull.
|
||||
A role read in detail on an earlier pull is sent without the
|
||||
listing's place and remote, so what its own page said about them stays.
|
||||
Completeness is decided by the listing alone.
|
||||
|
||||
**robots.txt.** With `robots_page(slug)`, the shared code reads
|
||||
`https://<its host>/robots.txt` through `get` before the listing, at most
|
||||
once a day per board (`ats:robots:<target>`), following a redirect to the
|
||||
host's `www.` twin or back and no other. The board fails with "robots.txt
|
||||
disallows it" when the rules for `*` or for the app's own token disallow the
|
||||
page (RFC 9309: the longest matching rule wins, Allow wins a tie, `*` and a
|
||||
final `$` in paths, case-sensitive paths, or without case for a host with
|
||||
`ROBOTS_ANY_CASE`, which is for one that serves a path in any case, as
|
||||
Workday does). A 4xx other than 429 means no
|
||||
rules, which allows; a 5xx, a 429, a timeout or a redirect elsewhere fails
|
||||
the board with "robots.txt could not be read" and is asked again next pull.
|
||||
`robots_page` must be on a host `allowed(slug)` admits.
|
||||
|
||||
**Discovery.** `probe_url(slug)` is a cheap URL that answers 2xx only when
|
||||
the slug is a board; `probe_ok(body)` narrows that when a host answers an
|
||||
empty listing for any slug. `probe_slugs(guess)` lists the slugs a guessed
|
||||
name is probed as, `(guess,)` without it. A host without `probe_url` is never
|
||||
probed, only found by a link.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import pkgutil
|
||||
import re
|
||||
from collections.abc import Iterable
|
||||
from types import ModuleType
|
||||
|
||||
#: The rank of a module that states none: after the eight first hosts, whose
|
||||
#: ranks (10 to 80) keep the order they had before there were modules.
|
||||
DEFAULT_RANK = 100
|
||||
|
||||
#: What every host module defines.
|
||||
REQUIRED = ("NAME", "LABEL", "PATTERNS", "slug_ok", "allowed", "careers_url")
|
||||
|
||||
#: The functions a module may define; each must be callable if present.
|
||||
FUNCTIONS = ("slug_ok", "allowed", "careers_url", "feed_url", "parse", "page", "rows",
|
||||
"fallback_url", "robots_page", "detail", "probe_url", "probe_ok", "probe_slugs")
|
||||
|
||||
|
||||
def problems(module: ModuleType, name: str | None = None) -> list[str]:
|
||||
"""What `module` lacks or gets wrong against the contract, one phrase
|
||||
each; empty for a module that keeps it. `name` is the module's own name
|
||||
in the package, which `NAME` must equal."""
|
||||
out = [f"no {attr}" for attr in REQUIRED if not hasattr(module, attr)]
|
||||
if name is not None and getattr(module, "NAME", name) != name:
|
||||
out.append(f"NAME is {module.NAME!r}, not the module's name {name!r}")
|
||||
if hasattr(module, "LABEL") and not (isinstance(module.LABEL, str) and module.LABEL):
|
||||
out.append("LABEL is not a non-empty string")
|
||||
if not isinstance(getattr(module, "KEEP_CASE", False), bool):
|
||||
out.append("KEEP_CASE is not a bool")
|
||||
if not isinstance(getattr(module, "ROBOTS_ANY_CASE", False), bool):
|
||||
out.append("ROBOTS_ANY_CASE is not a bool")
|
||||
rank = getattr(module, "RANK", DEFAULT_RANK)
|
||||
if not isinstance(rank, int) or isinstance(rank, bool):
|
||||
out.append("RANK is not an int")
|
||||
patterns = getattr(module, "PATTERNS", ())
|
||||
if isinstance(patterns, str) or not isinstance(patterns, Iterable):
|
||||
out.append("PATTERNS is not a tuple of strings")
|
||||
else:
|
||||
for pattern in patterns:
|
||||
try:
|
||||
re.compile(pattern)
|
||||
except (TypeError, re.error) as exc:
|
||||
out.append(f"PATTERNS has {pattern!r}, which does not compile ({exc})")
|
||||
out += [f"{attr} is not callable" for attr in FUNCTIONS
|
||||
if hasattr(module, attr) and not callable(getattr(module, attr))]
|
||||
if hasattr(module, "page"):
|
||||
if not hasattr(module, "rows"):
|
||||
out.append("page without rows")
|
||||
elif not (hasattr(module, "feed_url") and hasattr(module, "parse")):
|
||||
out.append("neither feed_url and parse nor page and rows")
|
||||
if hasattr(module, "probe_ok") and not hasattr(module, "probe_url"):
|
||||
out.append("probe_ok without probe_url")
|
||||
if hasattr(module, "probe_slugs") and not hasattr(module, "probe_url"):
|
||||
out.append("probe_slugs without probe_url")
|
||||
return out
|
||||
|
||||
|
||||
def load(path: Iterable[str], package: str) -> dict[str, ModuleType]:
|
||||
"""`NAME -> module` for every module of `package` (found on `path`) whose
|
||||
name does not start with `_`, in order of `(RANK, NAME)`. Raises
|
||||
ImportError naming the first module that breaks the contract."""
|
||||
modules: list[ModuleType] = []
|
||||
for info in sorted(pkgutil.iter_modules(path), key=lambda info: info.name):
|
||||
if info.name.startswith("_"):
|
||||
continue
|
||||
module = importlib.import_module(f"{package}.{info.name}")
|
||||
if found := problems(module, info.name):
|
||||
raise ImportError(f"{package}.{info.name} breaks the host contract: "
|
||||
+ "; ".join(found))
|
||||
modules.append(module)
|
||||
modules.sort(key=lambda module: (getattr(module, "RANK", DEFAULT_RANK), module.NAME))
|
||||
return {module.NAME: module for module in modules}
|
||||
|
||||
|
||||
#: host -> its module, in order of `(RANK, NAME)`.
|
||||
HOSTS: dict[str, ModuleType] = load(__path__, __name__)
|
||||
@@ -0,0 +1,192 @@
|
||||
"""What the host modules share: the row and page shapes, the fetch they are
|
||||
handed, and the date, URL and salary helpers more than one host needs.
|
||||
|
||||
Kept apart from `plugins/ats.py` so that a host module and `ats.py` can both
|
||||
import it without either importing the other. The leading underscore keeps
|
||||
`plugins.readers` from listing it as a host.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime
|
||||
from email.utils import parsedate_to_datetime
|
||||
from typing import Any, Protocol
|
||||
|
||||
import httpx
|
||||
|
||||
#: What every parser turns a feed into: one dict per row, with the keys
|
||||
#: `source_id`, `title`, `location`, `url`, `posted`, `department`,
|
||||
#: `description`, `remote` and `salary_text`, and, where a host states pay,
|
||||
#: `salary_min`, `salary_max` and `currency`.
|
||||
Row = dict[str, Any]
|
||||
|
||||
#: A paged host's one page: `(decoded, results, total, cursor)`, the decoded
|
||||
#: page, its raw results (dicts with an `id`, counted for completeness), the
|
||||
#: total it states (None for a host that states none), and the cursor of the
|
||||
#: next page (None after the last). The first call gets cursor None.
|
||||
Page = tuple[Any, list[dict[str, Any]], Any, Any]
|
||||
|
||||
|
||||
class Get(Protocol):
|
||||
"""The fetch a host is handed: `ats.py`'s own, bound to one board, so the
|
||||
allow-list, the redirect rule, the byte cap and the deadline apply to
|
||||
every request a host makes. A POST sends `payload` as JSON; `headers` are
|
||||
added to (and may replace) the default User-Agent and Accept. Returns the
|
||||
body, and raises `httpx.HTTPStatusError` on a non-2xx answer; see
|
||||
"Fetching" in the package's docstring for the rest."""
|
||||
|
||||
def __call__(self, url: str, *, method: str = "GET",
|
||||
payload: dict[str, Any] | None = None,
|
||||
headers: dict[str, str] | None = None) -> bytes: ...
|
||||
|
||||
|
||||
def is_web_url(url: str) -> bool:
|
||||
"""A trimmed http(s) URL, or refuse — a feed is external content and a
|
||||
`javascript:` or `data:` link must never reach an href. Port of
|
||||
openings.go's `webURL`."""
|
||||
url = (url or "").strip()
|
||||
if not url:
|
||||
return False
|
||||
parsed = httpx.URL(url)
|
||||
return parsed.scheme in ("http", "https") and bool(parsed.host)
|
||||
|
||||
|
||||
def only(host: str) -> Callable[[str], bool]:
|
||||
"""The allow-list of a board that is fetched from one host name only,
|
||||
which is every host so far. A host name has no case, and a slug that
|
||||
names one (`Acme` of acme.recruitee.com) may be written with capitals."""
|
||||
want = host.lower()
|
||||
return lambda h: h.lower() == want
|
||||
|
||||
|
||||
#: Matched whole (`fullmatch`, so not even a trailing newline slips past `$`).
|
||||
#: A slug that is one URL path segment: what the hosts that put the slug in
|
||||
#: the path accept, and nothing that could add a path, a query or a fragment.
|
||||
_SEGMENT = re.compile(r"[A-Za-z0-9._~%-]+")
|
||||
|
||||
#: A slug that is one DNS label, for the hosts that put it in the host name.
|
||||
_LABEL = re.compile(r"[A-Za-z0-9-]+")
|
||||
|
||||
|
||||
def path_slug_ok(slug: str) -> bool:
|
||||
return bool(_SEGMENT.fullmatch(slug)) and slug not in (".", "..")
|
||||
|
||||
|
||||
def label_slug_ok(slug: str) -> bool:
|
||||
return bool(_LABEL.fullmatch(slug))
|
||||
|
||||
|
||||
def parse_dt(value: Any) -> datetime | None:
|
||||
"""An aware datetime from whatever a board sends: ISO-8601 with an
|
||||
offset, `Z`, a bare date, epoch milliseconds, "YYYY-MM-DD HH:MM:SS UTC",
|
||||
or an RSS date ("Tue, 29 Sep 2026 13:36:59 +0200"). None for anything else, which `Item` reads as "now"."""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, int | float):
|
||||
if value <= 0:
|
||||
return None
|
||||
seconds = value / 1000 if value > 10**12 else value
|
||||
return datetime.fromtimestamp(seconds, tz=UTC)
|
||||
text = str(value).strip()
|
||||
if not text:
|
||||
return None
|
||||
try:
|
||||
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
match = re.match(r"^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})", text)
|
||||
if not match:
|
||||
try:
|
||||
parsed = parsedate_to_datetime(text)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
|
||||
parsed = datetime.fromisoformat(f"{match.group(1)}T{match.group(2)}+00:00")
|
||||
return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
|
||||
|
||||
|
||||
# ---------------------------------------------------------- structured pay
|
||||
#
|
||||
# A board's own salary range, mapped to yearly `salary_min`/`salary_max`
|
||||
# (integers) and `currency` (ISO 4217) — the columns `upsert_role` unpacks
|
||||
# `raw` into, per docs/decisions.md's "raw is transport, columns are
|
||||
# storage". Only Ashby, Greenhouse, Lever, Recruitee and Personio give
|
||||
# structured pay on their list endpoint; Workable, SmartRecruiters and
|
||||
# Teamtailor do not (checked live, 2026-09-29), so their rows leave the
|
||||
# columns out and `ATSSource.fetch` reads a plain '' or None for them.
|
||||
|
||||
|
||||
def num(value: Any) -> float | None:
|
||||
"""A positive number from whatever a board sends (int, float, or a
|
||||
numeral string), or None for anything else, including zero and a
|
||||
negative — a board's placeholder 0 (an unfilled field, not a real
|
||||
figure) must not become a salary floor of zero."""
|
||||
try:
|
||||
number = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return number if number > 0 else None
|
||||
|
||||
|
||||
def yearly_amount(min_value: Any, max_value: Any, currency: Any, interval: Any,
|
||||
factors: dict[str, int]) -> tuple[int | None, int | None, str]:
|
||||
"""`(salary_min, salary_max, currency)` as yearly integers, or `(None,
|
||||
None, "")` when there is no currency, no bound, or `interval` is not one
|
||||
of `factors`' own keys — which is how an hourly or a daily rate is left
|
||||
out rather than annualised into a number nobody stated. `factors` maps
|
||||
the board's own interval spelling to the multiplier that makes it
|
||||
yearly: 1 for a figure already yearly, 12 for a monthly one."""
|
||||
code = str(currency or "").strip().upper()
|
||||
factor = factors.get(str(interval or "").strip())
|
||||
if not code or not factor:
|
||||
return None, None, ""
|
||||
lo, hi = num(min_value), num(max_value)
|
||||
if lo is None and hi is None:
|
||||
return None, None, ""
|
||||
return (int(lo * factor) if lo is not None else None,
|
||||
int(hi * factor) if hi is not None else None, code)
|
||||
|
||||
|
||||
# ------------------------------------------------------------- salary_text
|
||||
#
|
||||
# `salary_min`/`salary_max`/`currency` above are yearly only, on purpose:
|
||||
# hourly and daily rates, a Greenhouse role with several pay zones, and a
|
||||
# range with no currency are all dropped from those columns rather than
|
||||
# guessed into a wrong yearly figure. `salary_text` is where that dropped
|
||||
# information survives — one human-readable line per parser, written
|
||||
# whether or not the pay conformed, so a person can read it and do their own
|
||||
# sums and a stricter analysis can find the roles the columns left out
|
||||
# (`Role.salary_state`, `app/models.py`).
|
||||
|
||||
#: Formats a number the way a salary line writes it: no decimals for a whole
|
||||
#: figure (the normal case for a yearly or monthly one), two for anything
|
||||
#: else (an hourly or daily rate quoted with cents).
|
||||
def fmt_amount(value: float) -> str:
|
||||
return f"{int(value):,}" if value == int(value) else f"{value:,.2f}"
|
||||
|
||||
|
||||
def bounds_words(lo: float | None, hi: float | None) -> str:
|
||||
""""70,000 to 90,000", or "from 70,000"/"up to 90,000" with only one
|
||||
bound stated, or "" with neither."""
|
||||
if lo is not None and hi is not None:
|
||||
return fmt_amount(lo) if lo == hi else f"{fmt_amount(lo)} to {fmt_amount(hi)}"
|
||||
if lo is not None:
|
||||
return f"from {fmt_amount(lo)}"
|
||||
if hi is not None:
|
||||
return f"up to {fmt_amount(hi)}"
|
||||
return ""
|
||||
|
||||
|
||||
def pay_line(lo: float | None, hi: float | None, currency: Any, period: str) -> str:
|
||||
"""One statement of pay as a human-readable line — "EUR 70,000 to 90,000
|
||||
per year" — or "" when there is no bound to report at all. A currency
|
||||
that was not stated does not drop the line, unlike the normalised
|
||||
columns: it is said plainly instead, "80,000 to 100,000 per year (no
|
||||
currency stated)", so the figure is not lost along with the currency."""
|
||||
bounds = bounds_words(lo, hi)
|
||||
if not bounds:
|
||||
return ""
|
||||
tail = f" per {period}" if period else ""
|
||||
code = str(currency or "").strip().upper()
|
||||
return f"{code} {bounds}{tail}" if code else f"{bounds}{tail} (no currency stated)"
|
||||
@@ -0,0 +1,122 @@
|
||||
"""Ashby: `ashby:<job board name>`, the whole board in one JSON answer. A
|
||||
board name may carry a dot (`jimdo.com`) or be a uuid.
|
||||
|
||||
The field map is ported from `openings.go` in
|
||||
[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._shared import Row, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, yearly_amount
|
||||
|
||||
NAME = "ashby"
|
||||
LABEL = "Ashby"
|
||||
RANK = 30
|
||||
PATTERNS = (
|
||||
r"jobs\.ashbyhq\.com/([A-Za-z0-9_.-]+)",
|
||||
r"api\.ashbyhq\.com/posting-api/job-board/([A-Za-z0-9_.-]+)",
|
||||
)
|
||||
|
||||
_API = "api.ashbyhq.com"
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
return path_slug_ok(slug)
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return only(_API)
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
return f"https://jobs.ashbyhq.com/{slug}"
|
||||
|
||||
|
||||
def feed_url(slug: str) -> str:
|
||||
return f"https://{_API}/posting-api/job-board/{slug}?includeCompensation=true"
|
||||
|
||||
|
||||
def probe_url(slug: str) -> str:
|
||||
return f"https://{_API}/posting-api/job-board/{slug}"
|
||||
|
||||
|
||||
def probe_ok(body: bytes) -> bool:
|
||||
"""Ashby answers an empty listing for an unknown name, so a listing
|
||||
without jobs counts as no board."""
|
||||
return b'"jobs":[' in body and b'"jobs":[]' not in body
|
||||
|
||||
|
||||
def _salary(components: list[dict[str, Any]]) -> tuple[int | None, int | None, str]:
|
||||
"""The yearly range from `compensation.summaryComponents` (needs
|
||||
`?includeCompensation=true` on the feed URL, see `feed_url`): the first
|
||||
`Salary` component, yearly or monthly (a monthly one multiplied by 12);
|
||||
an hourly one is left out rather than annualised."""
|
||||
for component in components:
|
||||
if component.get("compensationType") != "Salary":
|
||||
continue
|
||||
return yearly_amount(component.get("minValue"), component.get("maxValue"),
|
||||
component.get("currencyCode"), component.get("interval"),
|
||||
{"1 YEAR": 1, "1 MONTH": 12})
|
||||
return None, None, ""
|
||||
|
||||
|
||||
def _salary_text(compensation: dict[str, Any]) -> str:
|
||||
"""One line per `summaryComponents` entry that states a bound, labelled
|
||||
by the board's own `compensationType` when more than one does (a salary
|
||||
range beside a bonus range, say); with just the one, the label is left
|
||||
off, the same as Greenhouse's. `interval` is "<n> <PERIOD>" ("1 YEAR",
|
||||
"1 HOUR"); the last word, lowered, is the period. With nothing to build
|
||||
a line from, Ashby's own prose summary (`compensationTierSummary`, else
|
||||
`scrapeableCompensationSalarySummary`) is what is left to show."""
|
||||
components = compensation.get("summaryComponents") or []
|
||||
stated = [c for c in components if num(c.get("minValue")) is not None
|
||||
or num(c.get("maxValue")) is not None]
|
||||
lines = []
|
||||
for component in stated:
|
||||
period = str(component.get("interval") or "").split(" ")[-1].lower()
|
||||
line = pay_line(num(component.get("minValue")), num(component.get("maxValue")),
|
||||
component.get("currencyCode"), period)
|
||||
if not line:
|
||||
continue
|
||||
if len(stated) > 1:
|
||||
label = str(component.get("compensationType") or "").strip()
|
||||
line = f"{label}: {line}" if label else line
|
||||
lines.append(line)
|
||||
if lines:
|
||||
return "; ".join(lines)
|
||||
return str(compensation.get("compensationTierSummary")
|
||||
or compensation.get("scrapeableCompensationSalarySummary") or "")
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
data = json.loads(body)
|
||||
rows = []
|
||||
for j in data.get("jobs") or []:
|
||||
title = (j.get("title") or "").strip()
|
||||
url = (j.get("jobUrl") or j.get("applyUrl") or "").strip()
|
||||
if not title or not is_web_url(url):
|
||||
continue
|
||||
description = j.get("descriptionHtml") or j.get("descriptionPlain") or ""
|
||||
compensation = j.get("compensation") or {}
|
||||
salary_min, salary_max, currency = _salary(compensation.get("summaryComponents") or [])
|
||||
rows.append({
|
||||
"source_id": str(j.get("id") or ""),
|
||||
"title": title,
|
||||
"location": (j.get("location") or "").strip(),
|
||||
"url": url,
|
||||
"posted": parse_dt(j.get("publishedAt")),
|
||||
"department": (j.get("department") or j.get("team") or "").strip(),
|
||||
"description": strip_html(description),
|
||||
"remote": j.get("isRemote"),
|
||||
"salary_text": _salary_text(compensation),
|
||||
"salary_min": salary_min,
|
||||
"salary_max": salary_max,
|
||||
"currency": currency,
|
||||
})
|
||||
return rows
|
||||
@@ -0,0 +1,135 @@
|
||||
"""BambooHR: `bamboohr:<subdomain>` for acme.bamboohr.com, the whole board in
|
||||
one JSON answer and each role's text, date and country on its own page.
|
||||
|
||||
Written from Configit's board on 2026-09-29 (two roles); jobhunt-hq did not
|
||||
read BambooHR. Unverified: whether a large board pages `/careers/list` (this
|
||||
one states `meta.totalCount` and no paging keys, so a list shorter than its
|
||||
count is refused rather than read as the whole board), and the words of
|
||||
`locationType`, taken to be BambooHR's own settings, "0" in the office, "1"
|
||||
remote and "2" hybrid; Configit's hybrid roles say "2" and `isRemote` is null.
|
||||
|
||||
A posting that invites an unsolicited application without a role (Configit's
|
||||
first) is listed like any other role.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._shared import Get, Row, label_slug_ok, only, parse_dt
|
||||
|
||||
NAME = "bamboohr"
|
||||
LABEL = "BambooHR"
|
||||
PATTERNS = (
|
||||
# Not the company's own app or documentation subdomains, which BambooHR
|
||||
# itself serves and which name no board.
|
||||
r"https?://(?!(?:app|resources|documentation|help)\.)([a-z0-9-]+)\.bamboohr\.com",
|
||||
)
|
||||
|
||||
#: `locationType` -> `store._remote`'s words. Anything else says nothing.
|
||||
_REMOTE = {"0": "onsite", "1": "remote", "2": "hybrid"}
|
||||
|
||||
#: Sent on the detail read: without it the same URL answers the page's HTML.
|
||||
_JSON = {"Accept": "application/json"}
|
||||
|
||||
|
||||
def host(slug: str) -> str:
|
||||
return f"{slug.lower()}.bamboohr.com"
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
return label_slug_ok(slug)
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return only(host(slug))
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
return f"https://{host(slug)}/careers"
|
||||
|
||||
|
||||
def robots_page(slug: str) -> str:
|
||||
return careers_url(slug)
|
||||
|
||||
|
||||
def feed_url(slug: str) -> str:
|
||||
return f"https://{host(slug)}/careers/list"
|
||||
|
||||
|
||||
def _remote(j: dict[str, Any]) -> str | None:
|
||||
if j.get("isRemote") is True:
|
||||
return "remote"
|
||||
return _REMOTE.get(str(j.get("locationType") if j.get("locationType") is not None else ""))
|
||||
|
||||
|
||||
def _place(j: dict[str, Any]) -> str:
|
||||
"""City, state and country of a role, as far as either of its two
|
||||
location objects states them (`atsLocation` mirrors `location` and is
|
||||
filled by companies that use BambooHR's own applicant tracking)."""
|
||||
place, ats_place = j.get("location") or {}, j.get("atsLocation") or {}
|
||||
country = place.get("addressCountry") or ats_place.get("country")
|
||||
parts = (place.get("city") or ats_place.get("city"),
|
||||
place.get("state") or ats_place.get("state"), country)
|
||||
return ", ".join(str(p).strip() for p in parts if p and str(p).strip())
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
"""`/careers/list` -> `{"meta": {"totalCount": n}, "result": [...]}`. A
|
||||
role has `id`, `jobOpeningName`, `departmentLabel`, `location` (`city`,
|
||||
`state`), `atsLocation` (the same with a `country`, null so far),
|
||||
`isRemote` and `locationType`. The text, the date and the country are only
|
||||
on the role's own page (`detail`), which is why `description` and `posted`
|
||||
are empty here."""
|
||||
data = json.loads(body)
|
||||
listed = data.get("result") if isinstance(data, dict) else None
|
||||
if not isinstance(listed, list):
|
||||
raise ValueError("not a BambooHR listing: no result list")
|
||||
stated = (data.get("meta") or {}).get("totalCount")
|
||||
if isinstance(stated, int) and stated > len(listed):
|
||||
raise ValueError(f"BambooHR listing has {len(listed)} of {stated} roles")
|
||||
out = []
|
||||
for j in listed:
|
||||
title = (j.get("jobOpeningName") or "").strip()
|
||||
source_id = str(j.get("id") or "")
|
||||
if not title or not source_id:
|
||||
continue
|
||||
out.append({
|
||||
"source_id": source_id,
|
||||
"title": title,
|
||||
"location": _place(j),
|
||||
"url": f"https://{host(slug)}/careers/{source_id}",
|
||||
"posted": None,
|
||||
"department": str(j.get("departmentLabel") or "").strip(),
|
||||
"description": "",
|
||||
"remote": _remote(j),
|
||||
"salary_text": "", # the list states no pay; the detail's `compensation` was null
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
"""`/careers/<id>/detail` -> `{"result": {"jobOpening": {...}}}`, with the
|
||||
HTML `description`, `datePosted` (a date), `location` (`city`, `state`,
|
||||
`addressCountry`) and `locationType`. The location is the listing's with
|
||||
the country added, since the listing has none."""
|
||||
data = json.loads(get(f"https://{host(slug)}/careers/{row['source_id']}/detail",
|
||||
headers=_JSON))
|
||||
opening = ((data.get("result") or {}).get("jobOpening")
|
||||
if isinstance(data, dict) else None)
|
||||
if not isinstance(opening, dict):
|
||||
raise ValueError("not a BambooHR role: no result.jobOpening")
|
||||
out: dict[str, Any] = {
|
||||
"description": " ".join(html.unescape(strip_html(opening.get("description") or "")).split()),
|
||||
"posted": parse_dt(opening.get("datePosted")),
|
||||
}
|
||||
if place := _place(opening):
|
||||
out["location"] = place
|
||||
if remote := _remote(opening):
|
||||
out["remote"] = remote
|
||||
return out
|
||||
@@ -0,0 +1,148 @@
|
||||
"""Deel: `deel:<slug>` for jobs.deel.com/<slug>, the whole board in one HTML
|
||||
page.
|
||||
|
||||
Written from Klarna's board on 2026-09-29. Deel's job board is a Next.js page
|
||||
with no public JSON endpoint (its `/api/` is disallowed by robots.txt), but
|
||||
the server-rendered page carries the board in its React Server Components
|
||||
payload: a stream of `self.__next_f.push([1, "..."])` scripts whose strings,
|
||||
joined, hold `"jobPostings":[...]` with every open role, and, beside it, the
|
||||
rich-text descriptions as separate text rows that a posting names by
|
||||
reference (`"richtextDescription":"$1a"`). Klarna's 96 roles came in one
|
||||
page, the same 96 as the page's own JSON-LD list, so no description needs a
|
||||
read of the role's page.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._shared import Row, num, only, parse_dt, pay_line
|
||||
|
||||
NAME = "deel"
|
||||
LABEL = "Deel"
|
||||
KEEP_CASE = True # `jobs.deel.com/Klarna` is "Job Board Not Found", `/klarna` is the board
|
||||
PATTERNS = (
|
||||
# `/job-boards/<slug>` is the address Klarna's own site links to.
|
||||
r"jobs\.deel\.com/(?:job-boards/)?(?!job-boards\b)([A-Za-z0-9][A-Za-z0-9_-]*)",
|
||||
)
|
||||
|
||||
_HOST = "jobs.deel.com"
|
||||
|
||||
#: A slug is one path segment that starts with a letter or a digit, so `_next`
|
||||
#: and `.` never name a board.
|
||||
_SLUG = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]*")
|
||||
|
||||
#: One script of the payload, its JSON string literal captured. Kind 1 is
|
||||
#: the data; the others (`[0]`, `[2, ...]`) carry the bootstrap.
|
||||
_PUSH = re.compile(r'self\.__next_f\.push\(\[1,("(?:[^"\\]|\\.)*")\]\)')
|
||||
|
||||
#: One row of the joined stream: `<hex id>:` and then either `T<hex length>,`
|
||||
#: and that many bytes of text, or a line of JSON.
|
||||
_ROW = re.compile(rb"([0-9a-f]*):")
|
||||
_TEXT = re.compile(rb"T([0-9a-f]+),")
|
||||
|
||||
#: Deel's `workArrangementEnum` -> `store._remote`'s words.
|
||||
_REMOTE = {"ON_SITE": "onsite", "HYBRID": "hybrid", "REMOTE": "remote"}
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
return bool(_SLUG.fullmatch(slug))
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return only(_HOST)
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
return f"https://{_HOST}/{slug}"
|
||||
|
||||
|
||||
def robots_page(slug: str) -> str:
|
||||
return careers_url(slug)
|
||||
|
||||
|
||||
def feed_url(slug: str) -> str:
|
||||
return careers_url(slug)
|
||||
|
||||
|
||||
def _stream(body: bytes) -> dict[str, str]:
|
||||
"""The payload's rows by id. A text row's length is in bytes of UTF-8 and
|
||||
the next row follows it directly, so the stream is read as bytes, one row
|
||||
after another, rather than split on newlines."""
|
||||
text = "".join(json.loads(m.group(1)) for m in _PUSH.finditer(body.decode("utf-8", "replace")))
|
||||
data = text.encode()
|
||||
out: dict[str, str] = {}
|
||||
pos = 0
|
||||
while pos < len(data) and (row := _ROW.match(data, pos)):
|
||||
pos = row.end()
|
||||
if data[pos:pos + 1] == b"T" and (size := _TEXT.match(data, pos)):
|
||||
end = size.end() + int(size.group(1), 16)
|
||||
out[row.group(1).decode()] = data[size.end():end].decode("utf-8", "replace")
|
||||
pos = end
|
||||
else:
|
||||
end = data.find(b"\n", pos)
|
||||
end = len(data) if end < 0 else end
|
||||
out[row.group(1).decode()] = data[pos:end].decode("utf-8", "replace")
|
||||
pos = end + 1
|
||||
return out
|
||||
|
||||
|
||||
def _postings(body: bytes) -> tuple[list[dict[str, Any]], dict[str, str]]:
|
||||
"""The board's postings and the stream they refer into. A page without
|
||||
the postings array is not a board (a slug Deel does not know answers
|
||||
"Job Board Not Found"), and raises rather than read as an empty board."""
|
||||
stream = _stream(body)
|
||||
for value in stream.values():
|
||||
start = value.find('"jobPostings":[')
|
||||
if start >= 0:
|
||||
postings, _end = json.JSONDecoder().raw_decode(value[start + len('"jobPostings":'):])
|
||||
return postings, stream
|
||||
raise ValueError("not a Deel job board: no jobPostings in the page")
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
"""`jobs.deel.com/<slug>` -> one row per posting. `location` joins the
|
||||
role's places with "; ". `posted` is the day the posting was published
|
||||
(its publication's state), not the day the role was drafted.
|
||||
|
||||
Deel states pay as a currency and two amounts with no period, and the
|
||||
amounts run from 775,398 SEK to 55,206 SEK on one board, the latter
|
||||
plainly a month's pay. A yearly figure cannot be told from a monthly one,
|
||||
so the columns stay empty and `salary_text` gives the range as stated."""
|
||||
postings, stream = _postings(body)
|
||||
out = []
|
||||
for p in postings:
|
||||
title = (p.get("title") or "").strip()
|
||||
post_id = str(p.get("id") or "")
|
||||
if not title or not post_id:
|
||||
continue
|
||||
job = p.get("job") or {}
|
||||
places = [name for loc in job.get("jobLocations") or []
|
||||
if (name := ((loc.get("location") or {}).get("name") or "").strip())]
|
||||
departments = [name for d in job.get("jobDepartments") or []
|
||||
if (name := ((d.get("department") or {}).get("name") or "").strip())]
|
||||
pay = job.get("currentCompensation") or {}
|
||||
published = ((p.get("jobPostingPublications") or [{}])[0].get("currentState") or {})
|
||||
description = p.get("richtextDescription") or ""
|
||||
if description.startswith("$"):
|
||||
description = stream.get(description[1:], "")
|
||||
out.append({
|
||||
"source_id": post_id,
|
||||
"title": title,
|
||||
"location": "; ".join(dict.fromkeys(places)),
|
||||
"url": f"https://{_HOST}/{slug}/job-details/{post_id}/overview",
|
||||
"posted": parse_dt(published.get("createdAt") or p.get("createdAt")),
|
||||
"department": ", ".join(dict.fromkeys(departments)),
|
||||
"description": html.unescape(strip_html(description)),
|
||||
"remote": _REMOTE.get(str(job.get("workArrangementEnum") or "")),
|
||||
"salary_text": (pay_line(num(pay.get("minAmount")), num(pay.get("maxAmount")),
|
||||
pay.get("currencyIsoCode"), "")
|
||||
if p.get("isCompensationVisible") else ""),
|
||||
})
|
||||
return out
|
||||
@@ -0,0 +1,169 @@
|
||||
"""Eightfold: `eightfold:<tenant host>/<domain>`, for example
|
||||
`eightfold:infineon.eightfold.ai/infineon.com` or, for a career site on the
|
||||
company's own host, `eightfold:jobs.ericsson.com/ericsson.com`. The public
|
||||
PCSX search, paged by offset at ten roles a page with the total stated, and a
|
||||
role's text on its own request.
|
||||
|
||||
Written from Ericsson's site on 2026-09-29 (637 roles), with Infineon's and
|
||||
Microsoft's read the same day in the findings. Unverified on a tenant other
|
||||
than those three: that the search takes the same parameters and answers with
|
||||
the same fields.
|
||||
|
||||
The `domain` is the tenant's own name for the company, usually the company's
|
||||
domain, and it is not the host: the site's page carries it as `domain=` in its
|
||||
links. `/api/apply/v2/jobs` answers 403 "Not authorized for PCSX" on the sites
|
||||
tried, so it is not used.
|
||||
|
||||
The search ignores `num` and answers ten roles a page, so a board costs one
|
||||
request per ten roles, and `ats.MAX_PAGES` (100) stops a read at 1,000 roles: a
|
||||
larger board (Infineon's 1,292, Microsoft's 2,319) is read incomplete every pull,
|
||||
and an incomplete read never closes the roles it did not list. The rows are
|
||||
sorted newest first (`sort_by=timestamp`) so that what the cap keeps is the
|
||||
newest thousand and the pages do not shuffle between two requests, unlike the
|
||||
default order, which puts hot roles first. The search does take a `location`
|
||||
free text (Ericsson's "Sweden" found 157 of 637), which could bring a read
|
||||
under the cap, but a target names no filter and none is built here.
|
||||
|
||||
Discovery names Eightfold from an `eightfold.ai` link, and the host and the
|
||||
domain from one that carries `domain=`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
|
||||
|
||||
NAME = "eightfold"
|
||||
LABEL = "Eightfold"
|
||||
PATTERNS = (
|
||||
# Both parts, from a link to a tenant's site that says its domain.
|
||||
r"https?://([a-z0-9-]+\.eightfold\.ai)/[^\s\"'<>]*?[?&]domain=([a-z0-9.-]+\.[a-z]{2,})",
|
||||
# No slug: a link to a tenant's site that does not.
|
||||
r"[a-z0-9-]+\.eightfold\.ai",
|
||||
)
|
||||
|
||||
#: A whole host name: two labels or more, the last one letters, so neither an
|
||||
#: address, a bare word nor a port names a site. `ats._get` refuses every host
|
||||
#: but the tenant's, redirects included.
|
||||
_NAME = re.compile(r"[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*\.[a-z]{2,}")
|
||||
|
||||
#: Names a private network uses, never a company's career site.
|
||||
_LOCAL = (".localhost", ".local", ".internal", ".lan", ".home.arpa")
|
||||
|
||||
#: `workLocationOption` -> `store._remote`'s words. Only "onsite" and "hybrid"
|
||||
#: were seen; the other spellings are the obvious ones.
|
||||
_REMOTE = {"onsite": "onsite", "on_site": "onsite", "hybrid": "hybrid", "remote": "remote"}
|
||||
|
||||
|
||||
def parts(slug: str) -> tuple[str, str]:
|
||||
"""`(host, domain)` of a slug, lowercased."""
|
||||
host, _, domain = slug.lower().partition("/")
|
||||
return host, domain
|
||||
|
||||
|
||||
def _name_ok(name: str) -> bool:
|
||||
return bool(_NAME.fullmatch(name)) and not name.endswith(_LOCAL)
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
host, sep, domain = slug.lower().partition("/")
|
||||
return bool(sep) and _name_ok(host) and _name_ok(domain)
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return only(parts(slug)[0])
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
return f"https://{parts(slug)[0]}/careers"
|
||||
|
||||
|
||||
def robots_page(slug: str) -> str:
|
||||
return careers_url(slug)
|
||||
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any) -> Page:
|
||||
"""One page of `/api/pcsx/search` -> `{"status": 200, "data": {"positions":
|
||||
[...], "count": n}}`; the next starts at the offset after this page's
|
||||
positions, and an empty page ends the read."""
|
||||
host, domain = parts(slug)
|
||||
start = int(cursor or 0)
|
||||
answer = json.loads(get(f"https://{host}/api/pcsx/search?domain={domain}"
|
||||
f"&sort_by=timestamp&start={start}"))
|
||||
data = answer.get("data") if isinstance(answer, dict) else None
|
||||
positions = data.get("positions") if isinstance(data, dict) else None
|
||||
if not isinstance(positions, list):
|
||||
raise ValueError("not an Eightfold search answer: no data.positions"
|
||||
+ (f" (status {answer.get('status')})" if isinstance(answer, dict) else ""))
|
||||
count = data.get("count")
|
||||
results = [{"id": str(p.get("id") or "")} for p in positions]
|
||||
return (positions, results, count if isinstance(count, int) else None,
|
||||
start + len(positions) if positions else None)
|
||||
|
||||
|
||||
def _place(locations: Any) -> str:
|
||||
"""The role's places, "; " apart. Eightfold writes each as "city,region,
|
||||
country" with no space and repeats the name when a city is its own region
|
||||
("Mexico City,Mexico City,Mexico"), so the repeats go."""
|
||||
out: list[str] = []
|
||||
for loc in locations if isinstance(locations, list) else []:
|
||||
names: list[str] = []
|
||||
for name in str(loc).split(","):
|
||||
name = name.strip()
|
||||
if name and name not in names:
|
||||
names.append(name)
|
||||
if names and ", ".join(names) not in out:
|
||||
out.append(", ".join(names))
|
||||
return "; ".join(out)
|
||||
|
||||
|
||||
def rows(positions: list[dict[str, Any]], slug: str) -> list[Row]:
|
||||
"""A position has `id` (a number, kept as a string), `displayJobId`,
|
||||
`name`, `locations`, `postedTs` (epoch seconds), `department`,
|
||||
`workLocationOption` and `positionUrl`, the path of the role's page on the
|
||||
tenant's host. The list has no text, and no pay is stated anywhere."""
|
||||
host = parts(slug)[0]
|
||||
out = []
|
||||
for p in positions:
|
||||
title = (p.get("name") or "").strip()
|
||||
source_id = str(p.get("id") or "")
|
||||
path = str(p.get("positionUrl") or "").strip()
|
||||
url = f"https://{host}{path}" if path.startswith("/") and path[1:2] != "/" else ""
|
||||
if not title or not source_id or not is_web_url(url):
|
||||
continue
|
||||
out.append({
|
||||
"source_id": source_id,
|
||||
"title": title,
|
||||
"location": _place(p.get("locations") or p.get("standardizedLocations")),
|
||||
"url": url,
|
||||
"posted": parse_dt(p.get("postedTs")),
|
||||
"department": str(p.get("department") or "").strip(),
|
||||
"description": "",
|
||||
"remote": _REMOTE.get(str(p.get("workLocationOption") or "").strip().lower()),
|
||||
"salary_text": "",
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
"""`/api/pcsx/position_details` of the role -> `data.jobDescription`, the
|
||||
HTML text of the ad. The same answer repeats the listing's fields, and
|
||||
`publicUrl`, none of which is needed."""
|
||||
host, domain = parts(slug)
|
||||
role = str(row["source_id"])
|
||||
if not role.isdigit():
|
||||
raise ValueError(f"not an Eightfold position id: {role!r}")
|
||||
answer = json.loads(get(f"https://{host}/api/pcsx/position_details"
|
||||
f"?position_id={role}&domain={domain}&hl=en"))
|
||||
data = answer.get("data") if isinstance(answer, dict) else None
|
||||
if not isinstance(data, dict) or "jobDescription" not in data:
|
||||
raise ValueError("not an Eightfold position: no data.jobDescription")
|
||||
text = html.unescape(strip_html(data["jobDescription"] or ""))
|
||||
return {"description": " ".join(text.split())}
|
||||
@@ -0,0 +1,152 @@
|
||||
"""Gem: `gem:<board>` for jobs.gem.com/<board>, one GraphQL POST for the
|
||||
whole board and one more for a role's text and date. The board is the path
|
||||
after `jobs.gem.com/`, and it is case-sensitive (`Productboard` is an empty
|
||||
board where `productboard` is not).
|
||||
|
||||
The two queries below were read out of Gem's page scripts
|
||||
(`static.gem.com/scripts/jobBoards.*.js` and `ExternalJobBoardList.*.js`) on
|
||||
2026-09-29 and run against Productboard's and sennder's boards; the schema is
|
||||
Gem's own and not documented, so a new bundle can rename a field. Each query is
|
||||
one constant, and an answer that lacks the shape read here fails with an error
|
||||
that says which part is missing rather than reading as an empty board. The
|
||||
endpoint answered with no Origin and no Referer header, so none is sent. An
|
||||
unknown board answers 200 with an empty list, like a board with no role open.
|
||||
|
||||
The listing has no description and no date; both are on the role's page.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._shared import Get, Page, Row, only, parse_dt
|
||||
|
||||
NAME = "gem"
|
||||
LABEL = "Gem"
|
||||
KEEP_CASE = True
|
||||
PATTERNS = (r"jobs\.gem\.com/([A-Za-z0-9_-]+)",)
|
||||
|
||||
_HOST = "jobs.gem.com"
|
||||
_API = f"https://{_HOST}/api/public/graphql"
|
||||
|
||||
#: A board is one path segment of letters, digits, `-` and `_`. It travels as a
|
||||
#: GraphQL variable and never into a host name, so the host stays Gem's.
|
||||
_BOARD = re.compile(r"[A-Za-z0-9_-]+")
|
||||
|
||||
#: Every role of a board in one answer, with no paging and no total.
|
||||
LIST_QUERY = (
|
||||
"query JobBoardList($boardId: String!) { oatsExternalJobPostings(boardId: $boardId) "
|
||||
"{ jobPostings { id extId title locations { id name city isoCountry isRemote extId } "
|
||||
"job { id department { id name extId } locationType employmentType } } } }")
|
||||
|
||||
#: One role's description and the date it was first published.
|
||||
DETAIL_QUERY = (
|
||||
"query ExternalJobPosting($boardId: String!, $extId: String!) "
|
||||
"{ oatsExternalJobPosting(boardId: $boardId, extId: $extId) "
|
||||
"{ id title descriptionHtml extId firstPublishedTsSec "
|
||||
"locations { id extId name city isoCountry isRemote } "
|
||||
"job { id locationType employmentType requisitionId teamDisplayName "
|
||||
"department { id extId name } } compensationHtml } }")
|
||||
|
||||
#: `job.locationType` -> `store._remote`'s words.
|
||||
_REMOTE = {"IN_OFFICE": "onsite", "HYBRID": "hybrid", "REMOTE": "remote"}
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
return bool(_BOARD.fullmatch(slug))
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return only(_HOST)
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
return f"https://{_HOST}/{slug}"
|
||||
|
||||
|
||||
def robots_page(slug: str) -> str:
|
||||
return careers_url(slug)
|
||||
|
||||
|
||||
def _ask(get: Get, operation: str, query: str, variables: dict[str, str],
|
||||
path: str) -> Any:
|
||||
"""One GraphQL call, and the part of its answer at `data.<path>`. An
|
||||
`errors` list or a `data` without the key is Gem's schema having moved (or
|
||||
a refused query), and is raised as such; a `null` at the key is returned."""
|
||||
answer = json.loads(get(_API, method="POST", payload={
|
||||
"operationName": operation, "variables": variables, "query": query}))
|
||||
if not isinstance(answer, dict):
|
||||
raise ValueError(f"not a Gem answer: {type(answer).__name__}")
|
||||
if answer.get("errors"):
|
||||
first = answer["errors"][0]
|
||||
message = first.get("message") if isinstance(first, dict) else first
|
||||
raise ValueError(f"Gem's {operation} query was refused: {message}")
|
||||
data = answer.get("data")
|
||||
if not isinstance(data, dict) or path not in data:
|
||||
raise ValueError(f"Gem's {operation} answer has no {path}; the query may be out of date")
|
||||
return data[path]
|
||||
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any) -> Page:
|
||||
"""The one page there is: `data.oatsExternalJobPostings.jobPostings`."""
|
||||
board = _ask(get, "JobBoardList", LIST_QUERY, {"boardId": slug}, "oatsExternalJobPostings")
|
||||
postings = board.get("jobPostings") if isinstance(board, dict) else None
|
||||
if not isinstance(postings, list):
|
||||
raise ValueError("Gem's JobBoardList answer has no jobPostings list; "
|
||||
"the query may be out of date")
|
||||
return postings, [{"id": p.get("extId") or p.get("id") or ""} for p in postings], None, None
|
||||
|
||||
|
||||
def rows(postings: list[dict[str, Any]], slug: str) -> list[Row]:
|
||||
"""A role has `extId` (a number from a synced Greenhouse, else an opaque
|
||||
string; the role's page is `jobs.gem.com/<board>/<extId>`), `title`,
|
||||
`locations` (`name`, `city`, `isoCountry` as ISO alpha-3, `isRemote`) and
|
||||
`job` (`department`, `locationType`, `employmentType`). No pay is stated
|
||||
in the list."""
|
||||
out = []
|
||||
for p in postings:
|
||||
title = (p.get("title") or "").strip()
|
||||
ext_id = str(p.get("extId") or "")
|
||||
if not title or not ext_id:
|
||||
continue
|
||||
locations = p.get("locations") or []
|
||||
places: list[str] = []
|
||||
for loc in locations:
|
||||
place = str(loc.get("city") or loc.get("name") or "").strip()
|
||||
if place and place not in places:
|
||||
places.append(place)
|
||||
job = p.get("job") or {}
|
||||
remote = _REMOTE.get(str(job.get("locationType") or "").upper())
|
||||
if remote is None and locations and all(loc.get("isRemote") for loc in locations):
|
||||
remote = "remote"
|
||||
out.append({
|
||||
"source_id": ext_id,
|
||||
"title": title,
|
||||
"location": "; ".join(places),
|
||||
"url": f"{careers_url(slug)}/{ext_id}",
|
||||
"posted": None, # only the role's own query has a date
|
||||
"department": str((job.get("department") or {}).get("name") or "").strip(),
|
||||
"description": "",
|
||||
"remote": remote,
|
||||
"salary_text": "",
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
"""`oatsExternalJobPosting` of the role: `descriptionHtml` and
|
||||
`firstPublishedTsSec` (epoch seconds). `compensationHtml` was null on the
|
||||
roles read and is left alone."""
|
||||
role = _ask(get, "ExternalJobPosting", DETAIL_QUERY,
|
||||
{"boardId": slug, "extId": row["source_id"]}, "oatsExternalJobPosting")
|
||||
if not isinstance(role, dict):
|
||||
raise ValueError(f"Gem has no role {row['source_id']} on {slug}")
|
||||
text = html.unescape(strip_html(role.get("descriptionHtml") or ""))
|
||||
return {"description": " ".join(text.split()),
|
||||
"posted": parse_dt(role.get("firstPublishedTsSec"))}
|
||||
@@ -0,0 +1,139 @@
|
||||
"""Greenhouse: `greenhouse:<board token>`, the whole board in one JSON answer.
|
||||
|
||||
The field map is ported from `openings.go` in
|
||||
[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._shared import Row, is_web_url, num, only, parse_dt, path_slug_ok, pay_line
|
||||
|
||||
NAME = "greenhouse"
|
||||
LABEL = "Greenhouse"
|
||||
RANK = 10
|
||||
PATTERNS = (
|
||||
r"(?:boards|job-boards|api)\.(?:eu\.)?greenhouse\.io/(?:v1/boards/|embed/job_board\?[^\"'\s]*for=)?([A-Za-z0-9_-]+)",
|
||||
r"greenhouse\.io/embed/job_board/js\?for=([A-Za-z0-9_-]+)",
|
||||
)
|
||||
|
||||
_API = "boards-api.greenhouse.io"
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
return path_slug_ok(slug)
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return only(_API)
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
return f"https://job-boards.greenhouse.io/{slug}"
|
||||
|
||||
|
||||
def feed_url(slug: str) -> str:
|
||||
return f"https://{_API}/v1/boards/{slug}/jobs?content=true&pay_transparency=true"
|
||||
|
||||
|
||||
def fallback_url(slug: str) -> str:
|
||||
"""The feed without `content=true`, read once when the full one exceeds
|
||||
the byte cap: the descriptions are lost, but the titles land rather than
|
||||
the whole board being skipped."""
|
||||
return f"https://{_API}/v1/boards/{slug}/jobs"
|
||||
|
||||
|
||||
def probe_url(slug: str) -> str:
|
||||
return f"https://{_API}/v1/boards/{slug}"
|
||||
|
||||
|
||||
def _salary(ranges: list[dict[str, Any]]) -> tuple[int | None, int | None, str]:
|
||||
"""The yearly range from `pay_input_ranges` (needs `pay_transparency=true`
|
||||
on the feed URL, see `feed_url`), which the list endpoint already carries
|
||||
in the same request — no per-role fetch. Only when there is exactly one
|
||||
range: a board with several ("Zone 1 Pay Range", "US Tier 2 Locations",
|
||||
one per office) gives no way to tell which office the role in hand is
|
||||
at, and picking one would misrepresent the columns. Greenhouse's own
|
||||
ranges are cents, and yearly by convention except an hourly or a daily
|
||||
one, named as such in `title` with no separate field to check — those
|
||||
are skipped rather than annualised into a number nobody stated."""
|
||||
if len(ranges) != 1:
|
||||
return None, None, ""
|
||||
entry = ranges[0]
|
||||
title = str(entry.get("title") or "").lower()
|
||||
if any(word in title for word in ("hour", "/hr", "daily", "per day", "day rate")):
|
||||
return None, None, ""
|
||||
code = str(entry.get("currency_type") or "").strip().upper()
|
||||
if not code:
|
||||
return None, None, ""
|
||||
lo, hi = num(entry.get("min_cents")), num(entry.get("max_cents"))
|
||||
if lo is None and hi is None:
|
||||
return None, None, ""
|
||||
return (int(lo / 100) if lo is not None else None,
|
||||
int(hi / 100) if hi is not None else None, code)
|
||||
|
||||
|
||||
def _salary_text(ranges: list[dict[str, Any]]) -> str:
|
||||
"""One line per `pay_input_ranges` entry, joined with "; " — every range
|
||||
Greenhouse states, not only the single one `_salary` accepts for the
|
||||
columns. With more than one, each is labelled with the board's own
|
||||
`title` ("Zone 1 Pay Range: ..."), since that is the only way to tell the
|
||||
ranges apart; with just the one, the label adds nothing and is left off.
|
||||
The hour/day check is the same one `_salary` uses to keep those out of
|
||||
the yearly columns; here it only picks the word that follows "per"."""
|
||||
multi = len(ranges) > 1
|
||||
lines = []
|
||||
for entry in ranges:
|
||||
title = str(entry.get("title") or "").strip()
|
||||
low_title = title.lower()
|
||||
period = ("hour" if any(w in low_title for w in ("hour", "/hr"))
|
||||
else "day" if "day" in low_title else "year")
|
||||
lo, hi = num(entry.get("min_cents")), num(entry.get("max_cents"))
|
||||
line = pay_line(lo / 100 if lo is not None else None,
|
||||
hi / 100 if hi is not None else None,
|
||||
entry.get("currency_type"), period)
|
||||
if not line:
|
||||
continue
|
||||
if multi:
|
||||
label = title.rstrip(":").strip() or "Range"
|
||||
line = f"{label}: {line}"
|
||||
lines.append(line)
|
||||
return "; ".join(lines)
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
data = json.loads(body)
|
||||
rows = []
|
||||
for j in data.get("jobs") or []:
|
||||
title = (j.get("title") or "").strip()
|
||||
url = (j.get("absolute_url") or "").strip()
|
||||
if not title or not is_web_url(url):
|
||||
continue
|
||||
departments = j.get("departments") or []
|
||||
# Greenhouse ships the description HTML-entity-escaped inside the
|
||||
# JSON string (`<p>`), so it needs unescaping before the tags
|
||||
# themselves can be stripped.
|
||||
content = html.unescape(j.get("content") or "")
|
||||
ranges = j.get("pay_input_ranges") or []
|
||||
salary_min, salary_max, currency = _salary(ranges)
|
||||
rows.append({
|
||||
"source_id": str(j.get("id") or ""),
|
||||
"title": title,
|
||||
"location": ((j.get("location") or {}).get("name") or "").strip(),
|
||||
"url": url,
|
||||
"posted": parse_dt(j.get("first_published") or j.get("updated_at")),
|
||||
"department": (departments[0].get("name") if departments else "") or "",
|
||||
"description": strip_html(content),
|
||||
"remote": None, # Greenhouse's feed does not say
|
||||
"salary_text": _salary_text(ranges),
|
||||
"salary_min": salary_min,
|
||||
"salary_max": salary_max,
|
||||
"currency": currency,
|
||||
})
|
||||
return rows
|
||||
@@ -0,0 +1,113 @@
|
||||
"""HiBob: `hibob:<subdomain>` for <subdomain>.careers.hibob.com, the whole
|
||||
board in one JSON answer.
|
||||
|
||||
Written from Dixa's board (two general-application ads) and Liquidia's (14
|
||||
roles) on 2026-09-29. The careers site's own API,
|
||||
`<subdomain>.careers.hibob.com/api/job-ad`, answers 401 unless the request
|
||||
carries the header `companyIdentifier: <subdomain>`. That header is the
|
||||
company's public subdomain and no key or token, but it is why this is a paged
|
||||
module, whose `page` may send headers, although it reads one page. Every ad
|
||||
comes with its texts, so there is no `detail`. Pay is read from the
|
||||
`payTransparency*` fields as they are named; no board seen so far had them
|
||||
filled, so the period's spelling is a guess.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._shared import Get, Page, Row, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
|
||||
|
||||
NAME = "hibob"
|
||||
LABEL = "HiBob"
|
||||
PATTERNS = (r"https?://([a-z0-9-]+)\.careers\.hibob\.com",)
|
||||
|
||||
#: HiBob's `workspaceTypeId` -> `store._remote`'s words.
|
||||
_REMOTE = {"remote": "remote", "hybrid": "hybrid", "on_site": "onsite", "onsite": "onsite"}
|
||||
|
||||
#: The texts of an ad, in the order the ad shows them.
|
||||
_SECTIONS = ("description", "responsibilities", "requirements", "benefits")
|
||||
|
||||
#: A pay period's spellings, as the word `pay_line` and `yearly_amount` take.
|
||||
_PERIODS = {"year": "year", "yearly": "year", "annual": "year", "annually": "year",
|
||||
"month": "month", "monthly": "month", "hour": "hour", "hourly": "hour",
|
||||
"day": "day", "daily": "day", "week": "week", "weekly": "week"}
|
||||
|
||||
|
||||
def _host(slug: str) -> str:
|
||||
return f"{slug.lower()}.careers.hibob.com"
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
return label_slug_ok(slug)
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return only(_host(slug))
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
return f"https://{_host(slug)}/"
|
||||
|
||||
|
||||
def robots_page(slug: str) -> str:
|
||||
return careers_url(slug)
|
||||
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any) -> Page:
|
||||
"""The one page: every ad, and no total or cursor, so the board is
|
||||
complete when the answer parses. The host's robots.txt is its single-page
|
||||
app's index page (a 200 with no rules), which allows."""
|
||||
data = json.loads(get(f"https://{_host(slug)}/api/job-ad",
|
||||
headers={"companyIdentifier": slug.lower(), "Accept": "application/json"}))
|
||||
ads = data.get("jobAdDetails")
|
||||
if not isinstance(ads, list):
|
||||
raise ValueError("not a HiBob board: no jobAdDetails")
|
||||
return data, [{"id": ad.get("id")} for ad in ads], None, None
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
return rows(json.loads(body), slug)
|
||||
|
||||
|
||||
def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
"""One answer -> one row per ad. `location` is the site with its country
|
||||
(the country left off where the site already names it). `posted` is
|
||||
`publishedAt`, which has nanoseconds. The description joins the ad's
|
||||
texts, which are HTML in the ads with sections and plain text elsewhere."""
|
||||
out = []
|
||||
for ad in data.get("jobAdDetails") or []:
|
||||
title = (ad.get("title") or "").strip()
|
||||
ad_id = str(ad.get("id") or "").strip()
|
||||
if not title or not ad_id:
|
||||
continue
|
||||
site = str(ad.get("site") or "").strip()
|
||||
country = str(ad.get("country") or "").strip()
|
||||
location = f"{site}, {country}" if site and country and country.lower() not in site.lower() \
|
||||
else site or country
|
||||
text = " ".join(strip_html(str(ad.get(key) or "")) for key in _SECTIONS)
|
||||
lo, hi = num(ad.get("payTransparencyMinSalary")), num(ad.get("payTransparencyMaxSalary"))
|
||||
currency = ad.get("payTransparencySalaryCurrency")
|
||||
period = _PERIODS.get(str(ad.get("payTransparencySalaryPayPeriod") or "").strip().lower(), "")
|
||||
salary_min, salary_max, code = yearly_amount(lo, hi, currency, period,
|
||||
{"year": 1, "month": 12})
|
||||
out.append({
|
||||
"source_id": ad_id,
|
||||
"title": title,
|
||||
"location": location,
|
||||
"url": f"https://{_host(slug)}/jobs/{ad_id}",
|
||||
"posted": parse_dt(ad.get("publishedAt")),
|
||||
"department": str(ad.get("department") or "").strip(),
|
||||
"description": " ".join(html.unescape(text).split()),
|
||||
"remote": _REMOTE.get(str(ad.get("workspaceTypeId") or "").strip().lower()),
|
||||
"salary_text": pay_line(lo, hi, currency, period),
|
||||
"salary_min": salary_min,
|
||||
"salary_max": salary_max,
|
||||
"currency": code,
|
||||
})
|
||||
return out
|
||||
@@ -0,0 +1,114 @@
|
||||
"""HR Manager (Talentech): `hrmanager:<customer>`, the public RSS of a
|
||||
company's job portal, paged by `skip` and `take` with no stated total.
|
||||
|
||||
Written from four real boards on 2026-09-29: STS Group (6 roles), Flying
|
||||
Tiger and KPMG (25 or more), and NNIT (none). The feed is
|
||||
`api.hr-manager.net/JobPortal.svc/<customer>/PositionList/rss/?incads=true`;
|
||||
without `take` it answers the first 25 roles whatever the board holds, and
|
||||
`skip` and `take` were seen to page it (25 more with `skip=25`, 100 with
|
||||
`take=100`). The customer is not case-sensitive (`FLYINGTIGER` answers as
|
||||
`flyingtiger` does).
|
||||
|
||||
robots.txt: the portal's own pages on candidate.hr-manager.net disallow
|
||||
everything for `*` except the application form, and so `robots_page` is the
|
||||
feed's URL, on the API host, which has no robots.txt (a 404, which allows).
|
||||
The feed is the interface HR Manager publishes for readers like this one; the
|
||||
portal's HTML list, which the robots.txt does shut out, is never read. The
|
||||
role's link is the application form the robots.txt names as allowed, and is
|
||||
only shown, not fetched.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
|
||||
|
||||
NAME = "hrmanager"
|
||||
LABEL = "HR Manager"
|
||||
PATTERNS = (
|
||||
r"hr-manager\.net/vacancies/list\.aspx\?(?:[^\"'\s#<>]*?[&;])?customer=([A-Za-z0-9][A-Za-z0-9_-]*)",
|
||||
r"api\.hr-manager\.net/JobPortal\.svc/([A-Za-z0-9][A-Za-z0-9_-]*)/",
|
||||
)
|
||||
|
||||
_API = "api.hr-manager.net"
|
||||
|
||||
#: The roles asked for in one page. 100 were served at once at Flying Tiger; the
|
||||
#: cursor moves on by what a page returned, so a smaller cap on the host's
|
||||
#: side would not skip a role, and the read ends on an empty page.
|
||||
PAGE_SIZE = 100
|
||||
|
||||
_SLUG = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]*")
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
return bool(_SLUG.fullmatch(slug))
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return only(_API)
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
return f"https://candidate.hr-manager.net/vacancies/list.aspx?customer={slug}"
|
||||
|
||||
|
||||
def _feed(slug: str, skip: int) -> str:
|
||||
return (f"https://{_API}/JobPortal.svc/{slug}/PositionList/rss/"
|
||||
f"?incads=true&skip={skip}&take={PAGE_SIZE}")
|
||||
|
||||
|
||||
def robots_page(slug: str) -> str:
|
||||
return _feed(slug, 0)
|
||||
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any) -> Page:
|
||||
skip = int(cursor or 0)
|
||||
items = _items(get(_feed(slug, skip), headers={"Accept": "application/rss+xml"}))
|
||||
results = [{"id": item.findtext("guid") or item.findtext("link") or ""} for item in items]
|
||||
return items, results, None, skip + len(items) if items else None
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
return rows(_items(body), slug)
|
||||
|
||||
|
||||
def _items(body: bytes) -> list[ET.Element]:
|
||||
"""`<rss><channel><item>...`. A board with no open roles is a channel
|
||||
with no items, which is an empty board; an answer without a channel is
|
||||
not a feed, and raises."""
|
||||
channel = ET.fromstring(body).find("channel")
|
||||
if channel is None:
|
||||
raise ValueError("not an HR Manager feed: no channel")
|
||||
return channel.findall("item")
|
||||
|
||||
|
||||
def rows(items: list[ET.Element], slug: str) -> list[Row]:
|
||||
"""One page of items. An item has the `guid` (the position's id), `link`
|
||||
(its application form), `title`, the whole advertisement as escaped HTML
|
||||
in `description` and `pubDate`. It names no place, department or pay: the
|
||||
place, where there is one, is in the title or the text."""
|
||||
out = []
|
||||
for item in items:
|
||||
title = (item.findtext("title") or "").strip()
|
||||
url = (item.findtext("link") or "").strip()
|
||||
if not title or not is_web_url(url):
|
||||
continue
|
||||
out.append({
|
||||
"source_id": (item.findtext("guid") or url).strip(),
|
||||
"title": title,
|
||||
"location": "",
|
||||
"url": url,
|
||||
"posted": parse_dt(item.findtext("pubDate")),
|
||||
"department": "",
|
||||
"description": html.unescape(strip_html(item.findtext("description") or "")),
|
||||
"remote": None,
|
||||
"salary_text": "",
|
||||
})
|
||||
return out
|
||||
@@ -0,0 +1,132 @@
|
||||
"""Jibe (iCIMS Jibe): `jibe:<host>`, the career site's own host name, for
|
||||
example `jibe:jobs.booking.com`. A JSON API on that host, paged by page number
|
||||
at 100 roles a page, with the total stated.
|
||||
|
||||
Written from Booking.com's site on 2026-09-29 (145 roles, two pages), the only
|
||||
Jibe site read. Unverified: that another Jibe site serves `/api/jobs` with the
|
||||
same parameters and answer, and what one that does not gives. A host that is
|
||||
not a Jibe site fails as a board that is not a listing.
|
||||
|
||||
`GET /api/jobs?limit=100&page=N` answers `{"jobs": [{"data": {...}}], "totalCount"}`
|
||||
with each role's full description, so the roles need no `detail`; `limit=200` is
|
||||
a 422. The findings used `brand=Booking.com`, which is not needed: without it
|
||||
the site's whole board is read, 145 roles, of which the filter keeps the 97 of
|
||||
brand "Booking.com" and drops the 48 of brand "Booking Holdings", the parent's
|
||||
own roles (finance, legal and data teams in Bucharest, listed and applied for
|
||||
on the same site). The board is read whole, as the site shows it, and `brand`
|
||||
is not in the target.
|
||||
|
||||
A role's page is `/jobs/<slug>` on the host, which redirects to the site's own
|
||||
path (`/booking/jobs/<slug>` on Booking's), so no site's path prefix is built
|
||||
in. The site's robots.txt asks for `crawl-delay: 5`, which the shared reader
|
||||
does not honour (`ats.PAGE_DELAY` is 0.5 s); a board is two requests.
|
||||
|
||||
Discovery can name Jibe (a site loads its assets from `jibecdn.com` or applies
|
||||
on `jibeapply.com`) but not the host, since a Jibe site's own links look like
|
||||
any other site's, so the host is given by hand.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._shared import Get, Page, Row, only, parse_dt
|
||||
|
||||
NAME = "jibe"
|
||||
LABEL = "Jibe"
|
||||
PATTERNS = (
|
||||
# No slug: the host is the company's own, which no URL on the page names.
|
||||
r"jibeapply\.com|jibecdn\.com/prod",
|
||||
)
|
||||
|
||||
#: The page size asked for, which is also the most the host serves (200 is a 422).
|
||||
PAGE_SIZE = 100
|
||||
|
||||
#: A whole host name (two labels or more, the last one letters), so neither an
|
||||
#: address (`127.0.0.1`), a bare word (`localhost`) nor a port names a site.
|
||||
#: `ats._get` refuses every host but this one, redirects included, and a local
|
||||
#: name or an address in any case.
|
||||
_HOST = re.compile(r"[a-z0-9]([a-z0-9-]*[a-z0-9])?(\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)*\.[a-z]{2,}")
|
||||
|
||||
|
||||
#: Names a private network uses, never a company's career site; `ats._get`
|
||||
#: refuses them too, and a slug that names one is not a board at all.
|
||||
_LOCAL = (".localhost", ".local", ".internal", ".lan", ".home.arpa")
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
host = slug.lower()
|
||||
return bool(_HOST.fullmatch(host)) and not host.endswith(_LOCAL)
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return only(slug)
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
return f"https://{slug.lower()}/jobs"
|
||||
|
||||
|
||||
def robots_page(slug: str) -> str:
|
||||
return careers_url(slug)
|
||||
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any) -> Page:
|
||||
"""One page of `/api/jobs`; the next is the following number, and an empty
|
||||
page ends the read."""
|
||||
number = int(cursor or 1)
|
||||
data = json.loads(get(f"https://{slug.lower()}/api/jobs?limit={PAGE_SIZE}&page={number}"))
|
||||
jobs = data.get("jobs") if isinstance(data, dict) else None
|
||||
if not isinstance(jobs, list):
|
||||
raise ValueError("not a Jibe listing: no jobs list")
|
||||
total = data.get("totalCount")
|
||||
results = [{"id": (j.get("data") or {}).get("slug") or ""} for j in jobs]
|
||||
return data, results, total if isinstance(total, int) else None, \
|
||||
number + 1 if jobs else None
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
return rows(json.loads(body), slug)
|
||||
|
||||
|
||||
def _plain(markup: Any) -> str:
|
||||
return " ".join(html.unescape(strip_html(str(markup or ""))).split())
|
||||
|
||||
|
||||
def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
"""One page: each `jobs[].data` has `slug` (equal to `req_id`), `title`,
|
||||
the HTML `description` and, on some, `qualifications`, `full_location`
|
||||
("Amsterdam, Netherlands; Manchester, United Kingdom" for several),
|
||||
`city` and `country`, `categories`, `posted_date` and `apply_url`, on the
|
||||
iCIMS site. The `salary_*` fields were 0 on every role and there is no
|
||||
currency, so no pay is read; nothing says remote (`location_type` is a
|
||||
kind of coordinates)."""
|
||||
out = []
|
||||
for job in data.get("jobs") or []:
|
||||
j = job.get("data") or {}
|
||||
title = (j.get("title") or "").strip()
|
||||
role = str(j.get("slug") or j.get("req_id") or "")
|
||||
if not title or not role:
|
||||
continue
|
||||
categories = j.get("categories") or []
|
||||
text = "\n\n".join(p for p in (_plain(j.get("description")),
|
||||
_plain(j.get("qualifications"))) if p)
|
||||
out.append({
|
||||
"source_id": role,
|
||||
"title": title,
|
||||
"location": (j.get("full_location")
|
||||
or ", ".join(p for p in (j.get("city"), j.get("country")) if p)).strip(),
|
||||
"url": f"https://{slug.lower()}/jobs/{role}",
|
||||
"posted": parse_dt(j.get("posted_date")),
|
||||
"department": str((categories[0] if categories else {}).get("name") or "").strip(),
|
||||
"description": text,
|
||||
"remote": None,
|
||||
"salary_text": "",
|
||||
})
|
||||
return out
|
||||
@@ -0,0 +1,216 @@
|
||||
"""Jobylon: `jobylon:<company id>`, the whole board in one HTML page, the
|
||||
description on each role's own page.
|
||||
|
||||
Written from Einride's board (company 3143) and Vipps MobilePay's (1056, no
|
||||
open roles) on 2026-09-29. Jobylon's own JSON and XML feeds are addressed by a
|
||||
uuid the company sets and holds, and none is public for these two, so the
|
||||
board is read from the widget's embed page, `cdn.jobylon.com/jobs/companies/
|
||||
<id>/embed/v2/`, which assigns `JBL.embed_v2['jobs'] = [...]` as a JavaScript
|
||||
literal with every open role (33 of 33 at Einride; `page_size` in the URL
|
||||
the widget is embedded with only sets its client-side page). The literal has
|
||||
unquoted keys and single-quoted strings, which `_json` reads. It carries no
|
||||
description, so `detail` reads the role's page on emp.jobylon.com, whose
|
||||
JSON-LD `JobPosting` has the description, the exact date and every location.
|
||||
A slug Jobylon does not know answers 404.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._shared import Get, Row, parse_dt
|
||||
|
||||
NAME = "jobylon"
|
||||
LABEL = "Jobylon"
|
||||
PATTERNS = (
|
||||
r"(?:emp|cdn)\.jobylon\.com/(?:jobs/)?companies/(\d+)",
|
||||
# A company's own site says which board it embeds (careers.einride.tech
|
||||
# sets `jbl_company_id = 3143`).
|
||||
r"jbl_company_id[\"']?\s*[=:]\s*[\"']?(\d+)",
|
||||
)
|
||||
|
||||
_CDN = "cdn.jobylon.com"
|
||||
_EMP = "emp.jobylon.com"
|
||||
|
||||
#: A company id, matched whole. The host names are fixed, so a slug can only
|
||||
#: ever land in a path.
|
||||
_SLUG = re.compile(r"[0-9]{1,12}")
|
||||
|
||||
#: A role's page on the site, as the embed names it (`/jobs/385191-einride-
|
||||
#: financial-controller/`): the only kind of link the module builds a URL
|
||||
#: from.
|
||||
_ROLE_PATH = re.compile(r"/jobs/[0-9]+(?:-[A-Za-z0-9_-]*)?/?")
|
||||
|
||||
_ASSIGNMENT = "JBL.embed_v2['jobs'] = "
|
||||
_KEY = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
|
||||
_TRAILING_COMMA = re.compile(r",(\s*[\]}])")
|
||||
|
||||
#: Jobylon's `workspace` -> `store._remote`'s words.
|
||||
_REMOTE = {"remote": "remote", "hybrid": "hybrid", "on-site": "onsite", "onsite": "onsite"}
|
||||
|
||||
#: The English month names, by their first three letters, in the spellings
|
||||
#: `published_date` uses ("Sept. 25, 2026").
|
||||
_MONTHS = {m: i for i, m in enumerate(
|
||||
("jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"), 1)}
|
||||
_PUBLISHED = re.compile(r"([A-Za-z]{3})[A-Za-z]*\.? (\d{1,2}), (\d{4})")
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
return bool(_SLUG.fullmatch(slug))
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return lambda host: host.lower() in (_CDN, _EMP)
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
"""Jobylon answers this with a redirect to `/companies/<id>-<name>/`."""
|
||||
return f"https://{_EMP}/companies/{slug}/"
|
||||
|
||||
|
||||
def feed_url(slug: str) -> str:
|
||||
return f"https://{_CDN}/jobs/companies/{slug}/embed/v2/"
|
||||
|
||||
|
||||
def robots_page(slug: str) -> str:
|
||||
"""The page the board is read from. Both hosts serve the same robots.txt
|
||||
(compared on 2026-09-29), so the roles' pages on emp.jobylon.com are
|
||||
covered by the check on this one."""
|
||||
return feed_url(slug)
|
||||
|
||||
|
||||
def _json(text: str) -> Any:
|
||||
"""A JavaScript array or object literal as JSON: keys quoted, strings
|
||||
single-quoted with `\\uXXXX` escapes, trailing commas."""
|
||||
out: list[str] = []
|
||||
i, n = 0, len(text)
|
||||
while i < n:
|
||||
char = text[i]
|
||||
if char in "'\"":
|
||||
i += 1
|
||||
chars: list[str] = []
|
||||
while i < n and text[i] != char:
|
||||
if text[i] == "\\":
|
||||
i += 1
|
||||
if text[i] == "u":
|
||||
chars.append(chr(int(text[i + 1:i + 5], 16)))
|
||||
i += 4
|
||||
elif text[i] == "x":
|
||||
chars.append(chr(int(text[i + 1:i + 3], 16)))
|
||||
i += 2
|
||||
else:
|
||||
chars.append({"n": "\n", "t": "\t", "r": "\r"}.get(text[i], text[i]))
|
||||
else:
|
||||
chars.append(text[i])
|
||||
i += 1
|
||||
i += 1
|
||||
out.append(json.dumps("".join(chars)))
|
||||
elif word := _KEY.match(text, i):
|
||||
i = word.end()
|
||||
colon = re.compile(r"\s*:").match(text, i)
|
||||
out.append(json.dumps(word.group()) if colon else word.group())
|
||||
else:
|
||||
out.append(char)
|
||||
i += 1
|
||||
return json.loads(_TRAILING_COMMA.sub(r"\1", "".join(out)))
|
||||
|
||||
|
||||
def _literal(body: bytes) -> str:
|
||||
"""The text of the jobs array: from its `[` to the matching `]`, quotes
|
||||
respected."""
|
||||
text = body.decode("utf-8", "replace")
|
||||
start = text.find(_ASSIGNMENT)
|
||||
if start < 0:
|
||||
raise ValueError("not a Jobylon board: no JBL.embed_v2['jobs'] in the page")
|
||||
start += len(_ASSIGNMENT)
|
||||
depth, i, quote = 0, start, ""
|
||||
while i < len(text):
|
||||
char = text[i]
|
||||
if quote:
|
||||
if char == "\\":
|
||||
i += 1
|
||||
elif char == quote:
|
||||
quote = ""
|
||||
elif char in "'\"":
|
||||
quote = char
|
||||
elif char == "[":
|
||||
depth += 1
|
||||
elif char == "]":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return text[start:i + 1]
|
||||
i += 1
|
||||
raise ValueError("Jobylon's jobs array does not end")
|
||||
|
||||
|
||||
def _published(value: Any) -> Any:
|
||||
match = _PUBLISHED.fullmatch(str(value or "").strip())
|
||||
month = _MONTHS.get(match.group(1).lower()) if match else None
|
||||
if not match or not month:
|
||||
return None
|
||||
return parse_dt(f"{match.group(3)}-{month:02d}-{int(match.group(2)):02d}")
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
"""The embed page -> one row per role. `posted` is the day `published_date`
|
||||
names, which `detail` replaces with the exact time. `location` joins the
|
||||
names the embed lists, which can be fewer than the role's places
|
||||
("Houston +3 more" lists three); `detail` has them all. The embed states
|
||||
no pay. `department` is the role's `function`, unless it is the
|
||||
placeholder "All functions"."""
|
||||
out = []
|
||||
for j in _json(_literal(body)):
|
||||
title = (j.get("title") or "").strip()
|
||||
job_id = str(j.get("id") or "").strip()
|
||||
path = str(j.get("url") or "").strip()
|
||||
if not title or not job_id or not _ROLE_PATH.fullmatch(path):
|
||||
continue
|
||||
function = str(j.get("function") or "").strip()
|
||||
out.append({
|
||||
"source_id": job_id,
|
||||
"title": title,
|
||||
"location": "; ".join(dict.fromkeys(
|
||||
str(name).strip() for name in j.get("locations") or [] if str(name).strip())),
|
||||
"url": f"https://{_EMP}{path}",
|
||||
"posted": _published(j.get("published_date")),
|
||||
"department": "" if function == "All functions" else function,
|
||||
"description": "",
|
||||
"remote": _REMOTE.get(str(j.get("workspace") or "").strip().lower()),
|
||||
"salary_text": "", # the embed states no pay
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
"""The role's own page: its JSON-LD `JobPosting` has the description, the
|
||||
time it was posted and each place as "Houston, TX, USA"."""
|
||||
page = get(row["url"], headers={"Accept": "text/html"}).decode("utf-8", "replace")
|
||||
for block in re.findall(r'<script type="application/ld\+json"[^>]*>(.*?)</script>',
|
||||
page, re.S):
|
||||
try:
|
||||
data = json.loads(block, strict=False)
|
||||
except ValueError:
|
||||
continue
|
||||
if isinstance(data, dict) and data.get("@type") == "JobPosting":
|
||||
break
|
||||
else:
|
||||
raise ValueError("no JobPosting on the role's page")
|
||||
places = []
|
||||
for place in data.get("jobLocation") or []:
|
||||
address = (place or {}).get("address") or {}
|
||||
name = str(address.get("streetAddress") or address.get("addressLocality") or "").strip()
|
||||
if name and name not in places:
|
||||
places.append(name)
|
||||
found: dict[str, Any] = {
|
||||
"description": html.unescape(strip_html(data.get("description") or "")),
|
||||
"posted": parse_dt(data.get("datePosted")),
|
||||
}
|
||||
if places:
|
||||
found["location"] = "; ".join(places)
|
||||
return found
|
||||
@@ -0,0 +1,121 @@
|
||||
"""Lever: `lever:<site>`, or `lever:eu/<site>` for a board on Lever's EU
|
||||
instance, the whole board in one JSON answer.
|
||||
|
||||
Lever runs two instances that share nothing but the API: `api.lever.co` and
|
||||
`api.eu.lever.co`, with the pages on `jobs.lever.co` and `jobs.eu.lever.co`.
|
||||
A site exists on one of them only (TomTom and Prima answer 404 on the other,
|
||||
2026-09-29), and the same name on both would be two companies, so the region
|
||||
is part of the slug. It is a prefix a plain site name cannot carry, since a
|
||||
Lever site name has no slash, which leaves every `lever:<site>` target as it
|
||||
was.
|
||||
|
||||
The field map is ported from `openings.go` in
|
||||
[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._shared import Row, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, yearly_amount
|
||||
|
||||
NAME = "lever"
|
||||
LABEL = "Lever"
|
||||
RANK = 20
|
||||
#: Two groups on the EU hosts, `eu` and the site, which discovery joins into
|
||||
#: the slug `eu/<site>`.
|
||||
PATTERNS = (
|
||||
r"jobs\.lever\.co/([A-Za-z0-9_-]+)",
|
||||
r"api\.lever\.co/v0/postings/([A-Za-z0-9_-]+)",
|
||||
r"jobs\.(eu)\.lever\.co/([A-Za-z0-9_-]+)",
|
||||
r"api\.(eu)\.lever\.co/v0/postings/([A-Za-z0-9_-]+)",
|
||||
)
|
||||
|
||||
#: The region prefixes a slug may carry, to the infix of both host names.
|
||||
_REGIONS = {"eu": "eu."}
|
||||
|
||||
|
||||
def _split(slug: str) -> tuple[str, str]:
|
||||
"""`(host infix, site)`: `("eu.", "tomtom")` for `eu/tomtom`, `("", "acme")`
|
||||
for `acme`."""
|
||||
region, sep, site = slug.partition("/")
|
||||
return (_REGIONS.get(region, "?"), site) if sep else ("", slug)
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
infix, site = _split(slug)
|
||||
return infix != "?" and path_slug_ok(site)
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return only(f"api.{_split(slug)[0]}lever.co")
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
infix, site = _split(slug)
|
||||
return f"https://jobs.{infix}lever.co/{site}"
|
||||
|
||||
|
||||
def feed_url(slug: str) -> str:
|
||||
infix, site = _split(slug)
|
||||
return f"https://api.{infix}lever.co/v0/postings/{site}?mode=json"
|
||||
|
||||
|
||||
def probe_url(slug: str) -> str:
|
||||
infix, site = _split(slug)
|
||||
return f"https://api.{infix}lever.co/v0/postings/{site}?limit=1&mode=json"
|
||||
|
||||
|
||||
def probe_slugs(guess: str) -> tuple[str, ...]:
|
||||
"""A name is tried on both instances, the US one first, since a site that
|
||||
names neither leaves no other way to tell which one it is on."""
|
||||
return (guess, *(f"{region}/{guess}" for region in _REGIONS))
|
||||
|
||||
|
||||
def _salary_text(salary_range: dict[str, Any]) -> str:
|
||||
"""One line from `salaryRange`, whatever its `interval` — including an
|
||||
hourly or daily one, which `yearly_amount` drops from the columns.
|
||||
Lever's interval is "per-<period>-<kind>" ("per-year-salary",
|
||||
"per-hour-wage"); the middle word is the period."""
|
||||
interval = str(salary_range.get("interval") or "")
|
||||
parts = interval.split("-")
|
||||
period = parts[1] if len(parts) >= 3 else ""
|
||||
return pay_line(num(salary_range.get("min")), num(salary_range.get("max")),
|
||||
salary_range.get("currency"), period)
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
data = json.loads(body)
|
||||
if not isinstance(data, list):
|
||||
return []
|
||||
rows = []
|
||||
for j in data:
|
||||
title = (j.get("text") or "").strip()
|
||||
url = (j.get("hostedUrl") or "").strip()
|
||||
if not title or not is_web_url(url):
|
||||
continue
|
||||
categories = j.get("categories") or {}
|
||||
description = j.get("description") or j.get("descriptionPlain") or ""
|
||||
salary_range = j.get("salaryRange") or {}
|
||||
salary_min, salary_max, currency = yearly_amount(
|
||||
salary_range.get("min"), salary_range.get("max"), salary_range.get("currency"),
|
||||
salary_range.get("interval"), {"per-year-salary": 1, "per-month-salary": 12})
|
||||
rows.append({
|
||||
"source_id": str(j.get("id") or ""),
|
||||
"title": title,
|
||||
"location": (categories.get("location") or "").strip(),
|
||||
"url": url,
|
||||
"posted": parse_dt(j.get("createdAt")),
|
||||
"department": (categories.get("department") or categories.get("team") or "").strip(),
|
||||
"description": strip_html(description),
|
||||
"remote": (j.get("workplaceType") or "").strip() or None,
|
||||
"salary_text": _salary_text(salary_range),
|
||||
"salary_min": salary_min,
|
||||
"salary_max": salary_max,
|
||||
"currency": currency,
|
||||
})
|
||||
return rows
|
||||
@@ -0,0 +1,183 @@
|
||||
"""Oracle Recruiting Cloud: `oracle:<host>/<siteNumber>`, for example
|
||||
`oracle:fa-ewto-saasfaprod1.fa.ocs.oraclecloud.com/CX_1001`. A paged REST
|
||||
listing that states its total, and a per-role resource for the description.
|
||||
|
||||
The host is the tenant's Fusion pod (`fa-...ocs.oraclecloud.com`), not the
|
||||
company's own domain: Nokia's `jobs.nokia.com` only redirects to it. The site
|
||||
number is `CX_1001` or `CX_1` on the tenants read so far, and the site's own
|
||||
name (`SinchCareer`) is taken in its place. Written from live boards on
|
||||
2026-09-29 (Milestone Systems, Nokia, Danske Bank, Sinch); jobhunt-hq did not
|
||||
read Oracle.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._shared import Get, Page, Row, only, parse_dt
|
||||
|
||||
NAME = "oracle"
|
||||
LABEL = "Oracle Recruiting Cloud"
|
||||
KEEP_CASE = True
|
||||
PATTERNS = (
|
||||
# The candidate site or one role's page: `<pod>/hcmUI/CandidateExperience/en/sites/CX_1/jobs`.
|
||||
r"https?://([a-z0-9-]+(?:\.[a-z0-9-]+)*\.oraclecloud\.(?:com|eu))"
|
||||
r"/hcmUI/CandidateExperience/[A-Za-z_-]+/sites/([A-Za-z0-9_-]+)",
|
||||
# The REST call a company's own page makes, which names the site as `siteNumber=`.
|
||||
r"https?://([a-z0-9-]+(?:\.[a-z0-9-]+)*\.oraclecloud\.(?:com|eu))"
|
||||
r"/hcmRestApi/resources/[a-z0-9.]+/recruitingCEJobRequisitions[^\"'\s]*?siteNumber=([A-Za-z0-9_-]+)",
|
||||
# No slug: another page of a tenant's Fusion UI (its sign-in, say), which names
|
||||
# the host but no candidate site.
|
||||
r"oraclecloud\.(?:com|eu)/hcmUI(?!/CandidateExperience/[A-Za-z_-]+/sites/)",
|
||||
)
|
||||
|
||||
#: The page size asked for. 200 was answered in full by Nokia (600 roles in
|
||||
#: three calls of about 360 KB, 2026-09-29); the cursor advances by what a page
|
||||
#: returned, so a tenant that caps `limit` lower would not skip a role.
|
||||
PAGE_SIZE = 200
|
||||
|
||||
#: A pod: `fa-ewto-saasfaprod1.fa.ocs.oraclecloud.com`. Oracle's regions also
|
||||
#: use `.eu` (Danske Bank's `ejqi.fa.ocs.oraclecloud.eu`). A host name has no
|
||||
#: case, so a slug written with capitals is allowed as its lowercase.
|
||||
_HOST = r"[a-z0-9-]+(?:\.[a-z0-9-]+)*\.oraclecloud\.(?:com|eu)"
|
||||
_SLUG = re.compile(rf"({_HOST})/([A-Za-z0-9_-]+)", re.IGNORECASE)
|
||||
|
||||
#: `WorkplaceTypeCode` -> `store._remote`'s words. A role that states none
|
||||
#: has an empty code, which stays unknown.
|
||||
_REMOTE = {"ora_remote": "remote", "ora_hybrid": "hybrid", "ora_on_site": "onsite"}
|
||||
|
||||
|
||||
def _parts(slug: str) -> tuple[str, str]:
|
||||
"""`(host, site number)` of a slug that passed `slug_ok`."""
|
||||
host, _, site = slug.partition("/")
|
||||
return host.lower(), site
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
return bool(_SLUG.fullmatch(slug))
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return only(_parts(slug)[0])
|
||||
|
||||
|
||||
def _base(slug: str) -> str:
|
||||
host, site = _parts(slug)
|
||||
return f"https://{host}/hcmUI/CandidateExperience/en/sites/{site}"
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
return f"{_base(slug)}/jobs"
|
||||
|
||||
|
||||
def robots_page(slug: str) -> str:
|
||||
return careers_url(slug)
|
||||
|
||||
|
||||
def _api(slug: str) -> str:
|
||||
return f"https://{_parts(slug)[0]}/hcmRestApi/resources/latest"
|
||||
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any) -> Page:
|
||||
"""One page of `recruitingCEJobRequisitions`. The answer's outer `count`,
|
||||
`hasMore` and `limit` describe the one search wrapper in `items`, so the
|
||||
total (`TotalJobsCount`) and the roles (`requisitionList`) are read from
|
||||
`items[0]`. Without the `expand` only the facets come back; asking for
|
||||
`requisitionList.secondaryLocations` gives the roles and each one's other
|
||||
locations, which the plain `requisitionList` leaves out. Past the last role
|
||||
the list is empty."""
|
||||
offset = int(cursor or 0)
|
||||
site = _parts(slug)[1]
|
||||
data = json.loads(get(
|
||||
f"{_api(slug)}/recruitingCEJobRequisitions?onlyData=true"
|
||||
f"&expand=requisitionList.secondaryLocations"
|
||||
f"&finder=findReqs;siteNumber={site},limit={PAGE_SIZE},offset={offset}"))
|
||||
search = (data.get("items") or [{}])[0]
|
||||
roles = search.get("requisitionList") or []
|
||||
# `_paged` counts `id`, which Oracle spells `Id`.
|
||||
results = [{"id": str(role.get("Id") or "")} for role in roles]
|
||||
return search, results, search.get("TotalJobsCount"), offset + len(roles) if roles else None
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
return rows((json.loads(body).get("items") or [{}])[0], slug)
|
||||
|
||||
|
||||
def _text(value: Any) -> str:
|
||||
return value.strip() if isinstance(value, str) else ""
|
||||
|
||||
|
||||
def _plain(value: Any) -> str:
|
||||
"""Plain text from a field of HTML, entities (`&`, ` `) included."""
|
||||
return html.unescape(strip_html(_text(value))).replace("\xa0", " ").strip()
|
||||
|
||||
|
||||
def rows(search: dict[str, Any], slug: str) -> list[Row]:
|
||||
"""The roles of one page's search wrapper. The list carries the title, the
|
||||
id, `PostedDate` (a date), `PrimaryLocation` ("Copenhagen, Denmark"; a
|
||||
country alone where the role names no city), `secondaryLocations`,
|
||||
`WorkplaceTypeCode` and `ShortDescriptionStr`, one paragraph that Nokia
|
||||
and Milestone fill for most roles. The department fields (`JobFamily`,
|
||||
`Department`, `Organization`) were null on every role of all four boards
|
||||
read, and are taken when a tenant fills them. The full description is in
|
||||
the role's own resource, see `detail`. No pay is stated."""
|
||||
out = []
|
||||
for role in search.get("requisitionList") or []:
|
||||
role_id = str(role.get("Id") or "").strip()
|
||||
title = _text(role.get("Title"))
|
||||
# The id goes into a URL, and a listing is external content.
|
||||
if not title or not re.fullmatch(r"[A-Za-z0-9_-]+", role_id):
|
||||
continue
|
||||
places: list[str] = []
|
||||
for name in (_text(role.get("PrimaryLocation")),
|
||||
*(_text(s.get("Name")) for s in role.get("secondaryLocations") or []
|
||||
if isinstance(s, dict))):
|
||||
if name and name not in places:
|
||||
places.append(name)
|
||||
out.append({
|
||||
"source_id": role_id,
|
||||
"title": title,
|
||||
"location": "; ".join(places),
|
||||
"url": f"{_base(slug)}/job/{role_id}",
|
||||
"posted": parse_dt(role.get("PostedDate")),
|
||||
"department": (_text(role.get("JobFamily")) or _text(role.get("JobFunction"))
|
||||
or _text(role.get("Department")) or _text(role.get("Organization"))),
|
||||
"description": _plain(role.get("ShortDescriptionStr")),
|
||||
"remote": _REMOTE.get(_text(role.get("WorkplaceTypeCode")).lower()),
|
||||
"salary_text": "", # the resource states no pay
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
"""The role's own resource (`recruitingCEJobRequisitionDetails`, found by
|
||||
`Id` and site). Only here is the full text: `ExternalDescriptionStr`, with
|
||||
the responsibilities and qualifications strings after it where a tenant
|
||||
fills them (Milestone leaves them empty), all HTML. `ExternalPostedStartDate`
|
||||
is a timestamp where the listing has a date. A role that is no longer open
|
||||
answers 200 with no items, and has nothing to read."""
|
||||
site = _parts(slug)[1]
|
||||
role_id = quote(str(row["source_id"]), safe="")
|
||||
data = json.loads(get(
|
||||
f"{_api(slug)}/recruitingCEJobRequisitionDetails?expand=all&onlyData=true"
|
||||
f"&finder=ById;Id=%22{role_id}%22,siteNumber={site}"))
|
||||
items = data.get("items") or []
|
||||
if not items:
|
||||
return {}
|
||||
role = items[0]
|
||||
text = " ".join(part for key in ("ExternalDescriptionStr", "ExternalResponsibilitiesStr",
|
||||
"ExternalQualificationsStr")
|
||||
if (part := _plain(role.get(key))))
|
||||
out: dict[str, Any] = {}
|
||||
if text:
|
||||
out["description"] = text
|
||||
if posted := parse_dt(role.get("ExternalPostedStartDate")):
|
||||
out["posted"] = posted
|
||||
return out
|
||||
@@ -0,0 +1,101 @@
|
||||
"""Personio: `personio:<subdomain>`, the whole board in one XML answer from
|
||||
the company's own subdomain of jobs.personio.de.
|
||||
|
||||
Written from one real board on 2026-09-23; jobhunt-hq did not read
|
||||
Personio.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Callable
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._shared import Row, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
|
||||
|
||||
NAME = "personio"
|
||||
LABEL = "Personio"
|
||||
RANK = 70
|
||||
PATTERNS = (r"https?://([a-z0-9-]+)\.jobs\.personio\.(?:de|com)",)
|
||||
|
||||
#: Personio's `salaryInformation/type` spellings, to the plain period word
|
||||
#: `pay_line` wants ("yearly" -> "year"). Anything else is passed through.
|
||||
_PERIODS = {"yearly": "year", "monthly": "month", "hourly": "hour",
|
||||
"daily": "day", "weekly": "week"}
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
return label_slug_ok(slug)
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
"""The company's own subdomain, so one company's slug cannot pass for
|
||||
another's."""
|
||||
return only(f"{slug}.jobs.personio.de")
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
return f"https://{slug}.jobs.personio.de"
|
||||
|
||||
|
||||
def feed_url(slug: str) -> str:
|
||||
return f"https://{slug}.jobs.personio.de/xml"
|
||||
|
||||
|
||||
def probe_url(slug: str) -> str:
|
||||
return feed_url(slug)
|
||||
|
||||
|
||||
def probe_ok(body: bytes) -> bool:
|
||||
return b"<position>" in body
|
||||
|
||||
|
||||
def _salary_text(salary_info: ET.Element) -> str:
|
||||
"""One line from a `<salaryInformation>` element, whatever its `type`,
|
||||
including an hourly or daily one that `yearly_amount` drops from the
|
||||
columns."""
|
||||
period_raw = (salary_info.findtext("type") or "").strip().lower()
|
||||
period = _PERIODS.get(period_raw, period_raw)
|
||||
return pay_line(num(salary_info.findtext("min")), num(salary_info.findtext("max")),
|
||||
salary_info.findtext("currencyCode"), period)
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
"""`{slug}.jobs.personio.de/xml` -> `<workzag-jobs><position>...`. No job
|
||||
URL in the feed itself; Personio's own page is `/job/{id}`, confirmed
|
||||
live against a real posting (2026-09-23)."""
|
||||
root = ET.fromstring(body)
|
||||
rows = []
|
||||
for position in root.findall("position"):
|
||||
job_id = (position.findtext("id") or "").strip()
|
||||
title = (position.findtext("name") or "").strip()
|
||||
if not job_id or not title:
|
||||
continue
|
||||
description = " ".join(
|
||||
(value.text or "") for value in position.findall("jobDescriptions/jobDescription/value"))
|
||||
salary_info = position.find("salaryInformation")
|
||||
salary_min = salary_max = None
|
||||
currency = ""
|
||||
salary_text = ""
|
||||
if salary_info is not None:
|
||||
salary_min, salary_max, currency = yearly_amount(
|
||||
salary_info.findtext("min"), salary_info.findtext("max"),
|
||||
salary_info.findtext("currencyCode"), salary_info.findtext("type"),
|
||||
{"yearly": 1, "monthly": 12})
|
||||
salary_text = _salary_text(salary_info)
|
||||
rows.append({
|
||||
"source_id": job_id,
|
||||
"title": title,
|
||||
"location": (position.findtext("office") or "").strip(),
|
||||
"url": f"https://{slug}.jobs.personio.de/job/{job_id}",
|
||||
"posted": parse_dt(position.findtext("createdAt")),
|
||||
"department": (position.findtext("department") or "").strip(),
|
||||
"description": strip_html(description),
|
||||
"remote": "remote" in (position.findtext("office") or "").lower() or None,
|
||||
"salary_text": salary_text,
|
||||
"salary_min": salary_min,
|
||||
"salary_max": salary_max,
|
||||
"currency": currency,
|
||||
})
|
||||
return rows
|
||||
@@ -0,0 +1,229 @@
|
||||
"""Phenom: `phenom:<host>/<country>/<lang>`, the career site a company runs
|
||||
on its own domain, at the locale root it uses: `phenom:careers.king.com/us/en`,
|
||||
`phenom:www.careers.philips.com/global/en`. A JSON search that pages by offset
|
||||
and states its total.
|
||||
|
||||
The listing is the `POST /widgets` call the site's own search page makes. It
|
||||
answered without a cookie, an `Origin` header, a `pageId` or the site's
|
||||
`refNum`, so the body carries none of them: only the locale (`lang`,
|
||||
`country`, both read off the slug's path) and the paging. Nothing is taken
|
||||
from the page's script. Written from live sites on 2026-09-29 (King, Just Eat
|
||||
Takeaway, Philips); jobhunt-hq did not read Phenom.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._shared import Get, Page, Row, only, parse_dt
|
||||
|
||||
NAME = "phenom"
|
||||
LABEL = "Phenom"
|
||||
PATTERNS = (
|
||||
# A link to the site's search page, the one URL of a Phenom site that carries
|
||||
# both the host and the locale root and is named the same on every one.
|
||||
r"https?://([a-z0-9-]+(?:\.[a-z0-9-]+)*\.[a-z][a-z0-9-]*)/([a-z]{2,8})/([a-z]{2})/search-results",
|
||||
# No slug: a page that loads Phenom's own assets, on a site whose locale
|
||||
# root discovery has not seen. The host is known, the board is not.
|
||||
r"phenompeople\.com|phenom\.com/careers",
|
||||
)
|
||||
|
||||
#: The page size asked for. 100 is what the site's own search sends; `size`
|
||||
#: 200 was answered in full by Just Eat Takeaway and 500 by Philips (of 865),
|
||||
#: but a page is 340 KB at 100 roles already.
|
||||
PAGE_SIZE = 100
|
||||
|
||||
#: A DNS name with a dot and a letter first in its last label, so an address
|
||||
#: (`127.0.0.1`) or a single word (`localhost`) never names a site the fetch
|
||||
#: could be pointed at; `ats._get` still refuses local names and every host
|
||||
#: but this one, redirects included. Then the locale root: `country/lang`,
|
||||
#: which every Phenom site read so far writes as `global/en`, `na/en`, `us/en`.
|
||||
_SLUG = re.compile(r"[a-z0-9-]+(?:\.[a-z0-9-]+)*\.[a-z][a-z0-9-]*/[a-z]{2,8}/[a-z]{2}")
|
||||
|
||||
|
||||
def _parts(slug: str) -> tuple[str, str, str]:
|
||||
"""`(host, country, lang)` of a slug that passed `slug_ok`."""
|
||||
host, country, lang = slug.split("/")
|
||||
return host, country, lang
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
return bool(_SLUG.fullmatch(slug))
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return only(_parts(slug)[0])
|
||||
|
||||
|
||||
def _root(slug: str) -> str:
|
||||
host, country, lang = _parts(slug)
|
||||
return f"https://{host}/{country}/{lang}"
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
return f"{_root(slug)}/search-results"
|
||||
|
||||
|
||||
def robots_page(slug: str) -> str:
|
||||
return careers_url(slug)
|
||||
|
||||
|
||||
def _body(slug: str, offset: int) -> dict[str, Any]:
|
||||
"""What the site's search page sends for a listing of every role, newest
|
||||
first, less the parts that only tell the page what to draw. `lang` is
|
||||
`en_global` for `/global/en` and `en_us` for `/us/en`, the site's own
|
||||
spelling on the three sites read."""
|
||||
_host, country, lang = _parts(slug)
|
||||
return {
|
||||
"lang": f"{lang}_{country}", "deviceType": "desktop", "country": country,
|
||||
"pageName": "search-results", "ddoKey": "refineSearch", "sortBy": "",
|
||||
"subsearch": "", "from": offset, "jobs": True, "counts": True, "all_fetch": False,
|
||||
"pageType": "landingPage", "size": PAGE_SIZE, "clearAll": False,
|
||||
"jdsource": "facets", "isSliderEnable": False, "siteType": "external",
|
||||
"location": "", "keywords": "", "global": True, "selected_fields": {},
|
||||
"sort": {"order": "desc", "field": "postedDate"}, "locationData": {},
|
||||
}
|
||||
|
||||
|
||||
def _key(job: dict[str, Any]) -> str:
|
||||
"""The role's id on the board: `jobSeqNo`, which carries the site and the
|
||||
language and is the one of the three ids (`jobSeqNo`, `jobId`, `reqId`)
|
||||
that the site's job page takes on every site read. `reqId` is not unique
|
||||
across languages."""
|
||||
return str(job.get("jobSeqNo") or job.get("jobId") or job.get("reqId") or "").strip()
|
||||
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any) -> Page:
|
||||
offset = int(cursor or 0)
|
||||
host = _parts(slug)[0]
|
||||
data = json.loads(get(f"https://{host}/widgets", method="POST",
|
||||
payload=_body(slug, offset)))
|
||||
search = data.get("refineSearch")
|
||||
if not isinstance(search, dict) or search.get("status", 200) != 200:
|
||||
raise ValueError("not a Phenom search answer: no refineSearch")
|
||||
jobs = (search.get("data") or {}).get("jobs") or []
|
||||
results = [{"id": _key(job)} for job in jobs]
|
||||
return search, results, search.get("totalHits"), offset + len(jobs) if jobs else None
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
return rows(json.loads(body).get("refineSearch") or {}, slug)
|
||||
|
||||
|
||||
def _text(value: Any) -> str:
|
||||
return value.strip() if isinstance(value, str) else ""
|
||||
|
||||
|
||||
def _plain(value: Any) -> str:
|
||||
return html.unescape(strip_html(_text(value))).replace("\xa0", " ").strip()
|
||||
|
||||
|
||||
def _slug_of(title: str) -> str:
|
||||
"""The decorative last part of a role's URL, its title with runs of
|
||||
anything but letters and digits as one dash; it matches the site's own
|
||||
(checked on all 32 roles of King's sitemap). The page is found by the id."""
|
||||
return re.sub(r"[^A-Za-z0-9]+", "-", title).strip("-")
|
||||
|
||||
|
||||
def rows(search: dict[str, Any], slug: str) -> list[Row]:
|
||||
"""The roles of one page, `refineSearch.data.jobs`. A role has the title,
|
||||
`jobSeqNo`, `postedDate` (a date at midnight), `category`, the locations
|
||||
as `multi_location` (a list, one per place) beside `location` (the first),
|
||||
and `descriptionTeaser`, the first paragraph of the text; the full text is
|
||||
on the role's page, see `detail`. `applyUrl` points at the apply system
|
||||
(Workday for Just Eat Takeaway and Philips), so the role's URL is its page
|
||||
on the site. No remote flag and no pay are in the list."""
|
||||
out = []
|
||||
for job in search.get("data", {}).get("jobs") or []:
|
||||
key = _key(job)
|
||||
title = _text(job.get("title"))
|
||||
if not title or not key:
|
||||
continue
|
||||
places: list[str] = []
|
||||
for place in (*(job.get("multi_location") or []), job.get("location")):
|
||||
if isinstance(place, str) and place.strip() and place.strip() not in places:
|
||||
places.append(place.strip())
|
||||
tail = _slug_of(title)
|
||||
out.append({
|
||||
"source_id": key,
|
||||
"title": title,
|
||||
"location": "; ".join(places),
|
||||
"url": f"{_root(slug)}/job/{key}" + (f"/{tail}" if tail else ""),
|
||||
"posted": parse_dt(job.get("postedDate")),
|
||||
"department": _text(job.get("category")),
|
||||
"description": _plain(job.get("descriptionTeaser")),
|
||||
"remote": None, # the list does not say
|
||||
"salary_text": "", # nor does it state pay
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
#: The JSON-LD blocks of a page; a Phenom job page has one `JobPosting` and,
|
||||
#: on Philips's, a `BreadcrumbList` and more.
|
||||
_JSON_LD = re.compile(r"<script[^>]+application/ld\+json[^>]*>(.*?)</script>", re.S | re.I)
|
||||
|
||||
|
||||
#: Where a page assigns its state, and the words its `checkRemote` says that
|
||||
#: `store._remote` reads (King's pages say "On-site"; no other was seen).
|
||||
_STATE = "phApp.ddo = "
|
||||
_REMOTE = {"remote": "remote", "hybrid": "hybrid", "on-site": "onsite", "onsite": "onsite"}
|
||||
|
||||
|
||||
def _job_posting(body: str) -> dict[str, Any]:
|
||||
"""The `JobPosting` among a page's JSON-LD blocks, or {}."""
|
||||
for block in _JSON_LD.findall(body):
|
||||
try:
|
||||
found = json.loads(block)
|
||||
except ValueError:
|
||||
continue
|
||||
for candidate in found if isinstance(found, list) else [found]:
|
||||
if isinstance(candidate, dict) and candidate.get("@type") == "JobPosting":
|
||||
return candidate
|
||||
return {}
|
||||
|
||||
|
||||
def _page_job(body: str) -> dict[str, Any]:
|
||||
"""The role as the page's own state holds it, `phApp.ddo.jobDetail.data.job`,
|
||||
or {}. King's pages of roles posted the same day had no `JobPosting`, and
|
||||
this had the full description. Only that one object is read: the state
|
||||
around it holds the site's session values, which nothing here uses."""
|
||||
start = body.find(_STATE)
|
||||
if start < 0:
|
||||
return {}
|
||||
try:
|
||||
state, _end = json.JSONDecoder().raw_decode(body, start + len(_STATE))
|
||||
job = state["jobDetail"]["data"]["job"]
|
||||
except (ValueError, KeyError, TypeError):
|
||||
return {}
|
||||
return job if isinstance(job, dict) else {}
|
||||
|
||||
|
||||
def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
"""The full description from the role's page: the `JobPosting` in its
|
||||
JSON-LD, whose `description` is HTML with its entities escaped (`<p>`
|
||||
and `&` inside), so it is unescaped, stripped and unescaped again; or,
|
||||
where the page has none, the page state's own `description`, which is
|
||||
plain HTML. A page with neither raises, so the role is asked again on a
|
||||
later pull instead of counting as read with no text. `datePosted` is taken
|
||||
as `posted`: it can be earlier than the list's `postedDate` (Philips's list
|
||||
said the day of the read for a role whose page said 2026-09-08), and a role
|
||||
keeps the earliest date it ever had. King's page state says `checkRemote`."""
|
||||
body = get(row["url"], headers={"Accept": "text/html"}).decode("utf-8", "replace")
|
||||
posting, job = _job_posting(body), _page_job(body)
|
||||
out: dict[str, Any] = {}
|
||||
if description := (_plain(html.unescape(_text(posting.get("description"))))
|
||||
or _plain(job.get("description"))):
|
||||
out["description"] = description
|
||||
else:
|
||||
raise ValueError("no description on the role's page")
|
||||
if posted := parse_dt(posting.get("datePosted")):
|
||||
out["posted"] = posted
|
||||
if remote := _REMOTE.get(_text(job.get("checkRemote")).lower()):
|
||||
out["remote"] = remote
|
||||
return out
|
||||
@@ -0,0 +1,149 @@
|
||||
"""Pinpoint: `pinpoint:<subdomain>` for acme.pinpointhq.com, the whole board
|
||||
in one JSON answer. There is no probe: an unknown subdomain's answer was not
|
||||
checked, and a board is found by the link a career site carries.
|
||||
|
||||
Written from Xeneta's board on 2026-09-29 (two roles); jobhunt-hq did not read
|
||||
Pinpoint. Unverified: whether a large board pages `/postings.json` (the answer
|
||||
has a `data` list and no paging keys, so every board is read as complete
|
||||
here), and the spelling of `compensation_frequency`, since no board read so far
|
||||
shows its pay (`compensation_visible` was false on both roles). A board shared
|
||||
by several brands (inMusic's, with Native Instruments on it) is read whole;
|
||||
there is no filter on `job.division`.
|
||||
|
||||
There is no posted date in the listing, only in each posting's page as JSON-LD,
|
||||
so `posted` stays None rather than costing a request a role for it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._shared import Row, is_web_url, label_slug_ok, num, only, pay_line, yearly_amount
|
||||
|
||||
NAME = "pinpoint"
|
||||
LABEL = "Pinpoint"
|
||||
PATTERNS = (r"https?://([a-z0-9-]+)\.pinpointhq\.com",)
|
||||
|
||||
#: `workplace_type` -> `store._remote`'s words. Anything else says nothing.
|
||||
_REMOTE = {"onsite": "onsite", "on_site": "onsite", "hybrid": "hybrid", "remote": "remote"}
|
||||
|
||||
#: `compensation_frequency` -> the multiplier that makes a figure yearly. The
|
||||
#: spellings are a guess, since no board read so far states pay; a frequency
|
||||
#: not listed leaves the yearly columns empty, and `salary_text` still says it.
|
||||
_YEARLY = {"annually": 1, "annual": 1, "yearly": 1, "year": 1, "monthly": 12, "month": 12}
|
||||
|
||||
#: `compensation_frequency` -> the word `salary_text` says after "per".
|
||||
_PERIOD = {"annually": "year", "annual": "year", "yearly": "year", "monthly": "month",
|
||||
"weekly": "week", "daily": "day", "hourly": "hour"}
|
||||
|
||||
#: The sections of a posting's text, in the order the page shows them: the
|
||||
#: description, then these fields, each with its own heading in `<field>_header`.
|
||||
_SECTIONS = ("key_responsibilities", "skills_knowledge_expertise", "benefits")
|
||||
|
||||
|
||||
def host(slug: str) -> str:
|
||||
return f"{slug.lower()}.pinpointhq.com"
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
return label_slug_ok(slug)
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return only(host(slug))
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
return f"https://{host(slug)}/"
|
||||
|
||||
|
||||
def robots_page(slug: str) -> str:
|
||||
return careers_url(slug)
|
||||
|
||||
|
||||
def feed_url(slug: str) -> str:
|
||||
return f"https://{host(slug)}/postings.json"
|
||||
|
||||
|
||||
def _plain(markup: Any) -> str:
|
||||
"""HTML as one line of plain text: the tags out, the entities read (Pinpoint
|
||||
writes `&` and ` `), the whitespace, non-breaking included, collapsed."""
|
||||
return " ".join(html.unescape(strip_html(str(markup or ""))).split())
|
||||
|
||||
|
||||
def _text(j: dict[str, Any]) -> str:
|
||||
"""The description and the sections after it, each section under its own
|
||||
heading, as plain text with a blank line between them."""
|
||||
parts = [_plain(j.get("description"))]
|
||||
for field in _SECTIONS:
|
||||
body = _plain(j.get(field))
|
||||
if body:
|
||||
heading = str(j.get(f"{field}_header") or "").strip()
|
||||
parts.append(f"{heading}: {body}" if heading else body)
|
||||
return "\n\n".join(p for p in parts if p)
|
||||
|
||||
|
||||
def _place(location: dict[str, Any]) -> str:
|
||||
"""The place's name, then its `province` where that says more. Pinpoint
|
||||
fills `province` with a region or a country as the company typed it, so it
|
||||
is kept only when it differs from the name."""
|
||||
parts: list[str] = []
|
||||
for part in (location.get("name") or location.get("city"), location.get("province")):
|
||||
part = str(part or "").strip()
|
||||
if part and part.lower() not in (p.lower() for p in parts):
|
||||
parts.append(part)
|
||||
return ", ".join(parts)
|
||||
|
||||
|
||||
def _pay(j: dict[str, Any]) -> tuple[int | None, int | None, str, str]:
|
||||
"""`(salary_min, salary_max, currency, salary_text)` of a posting whose
|
||||
pay is shown (`compensation_visible`); the free-text `compensation` is the
|
||||
line when the bounds are not given."""
|
||||
if not j.get("compensation_visible"):
|
||||
return None, None, "", ""
|
||||
lo, hi = num(j.get("compensation_minimum")), num(j.get("compensation_maximum"))
|
||||
frequency = str(j.get("compensation_frequency") or "").strip().lower()
|
||||
period = _PERIOD.get(frequency, frequency)
|
||||
text = pay_line(lo, hi, j.get("compensation_currency"), period)
|
||||
text = text or str(j.get("compensation") or "").strip()
|
||||
low, high, code = yearly_amount(lo, hi, j.get("compensation_currency"), frequency, _YEARLY)
|
||||
return low, high, code, text
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
"""`{slug}.pinpointhq.com/postings.json` -> `{"data": [...]}`. A posting has
|
||||
`id`, `title`, `url`, `location` (`name`, `city`, `province`),
|
||||
`workplace_type`, `job` (`requisition_id`, `department`, `division`), the
|
||||
HTML `description` and its sections, and the `compensation_*` fields."""
|
||||
data = json.loads(body)
|
||||
if not isinstance(data, dict) or not isinstance(data.get("data"), list):
|
||||
raise ValueError("not a Pinpoint listing: no data list")
|
||||
out = []
|
||||
for j in data["data"]:
|
||||
title = (j.get("title") or "").strip()
|
||||
url = (j.get("url") or "").strip()
|
||||
source_id = str(j.get("id") or "")
|
||||
if not title or not source_id or not is_web_url(url):
|
||||
continue
|
||||
job = j.get("job") or {}
|
||||
low, high, currency, salary_text = _pay(j)
|
||||
out.append({
|
||||
"source_id": source_id,
|
||||
"title": title,
|
||||
"location": _place(j.get("location") or {}),
|
||||
"url": url,
|
||||
"posted": None, # only the posting's own page has a date
|
||||
"department": str((job.get("department") or {}).get("name") or "").strip(),
|
||||
"description": _text(j),
|
||||
"remote": _REMOTE.get(str(j.get("workplace_type") or "").strip().lower()),
|
||||
"salary_text": salary_text,
|
||||
"salary_min": low,
|
||||
"salary_max": high,
|
||||
"currency": currency,
|
||||
})
|
||||
return out
|
||||
@@ -0,0 +1,84 @@
|
||||
"""Recruitee: `recruitee:<subdomain>`, the whole board in one JSON answer
|
||||
from the company's own subdomain of recruitee.com.
|
||||
|
||||
Written from one real board on 2026-09-23; jobhunt-hq did not read
|
||||
Recruitee.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._shared import Row, is_web_url, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
|
||||
|
||||
NAME = "recruitee"
|
||||
LABEL = "Recruitee"
|
||||
RANK = 60
|
||||
PATTERNS = (r"https?://([a-z0-9-]+)\.recruitee\.com",)
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
return label_slug_ok(slug)
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
"""The company's own subdomain, so one company's slug cannot pass for
|
||||
another's."""
|
||||
return only(f"{slug}.recruitee.com")
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
return f"https://{slug}.recruitee.com"
|
||||
|
||||
|
||||
def feed_url(slug: str) -> str:
|
||||
return f"https://{slug}.recruitee.com/api/offers/"
|
||||
|
||||
|
||||
def probe_url(slug: str) -> str:
|
||||
return feed_url(slug)
|
||||
|
||||
|
||||
def probe_ok(body: bytes) -> bool:
|
||||
return b'"offers":[' in body and b'"offers":[]' not in body
|
||||
|
||||
|
||||
def _salary_text(salary: dict[str, Any]) -> str:
|
||||
"""One line from Recruitee's `salary` object; `period` is already a
|
||||
plain word ("year", "month")."""
|
||||
period = str(salary.get("period") or "").strip().lower()
|
||||
return pay_line(num(salary.get("min")), num(salary.get("max")),
|
||||
salary.get("currency"), period)
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
data = json.loads(body)
|
||||
rows = []
|
||||
for j in data.get("offers") or []:
|
||||
title = (j.get("title") or "").strip()
|
||||
url = (j.get("careers_url") or "").strip()
|
||||
if not title or not is_web_url(url):
|
||||
continue
|
||||
salary = j.get("salary") if isinstance(j.get("salary"), dict) else {}
|
||||
salary_min, salary_max, currency = yearly_amount(
|
||||
salary.get("min"), salary.get("max"), salary.get("currency"),
|
||||
salary.get("period"), {"year": 1, "month": 12})
|
||||
rows.append({
|
||||
"source_id": str(j.get("id") or ""),
|
||||
"title": title,
|
||||
"location": (j.get("location") or j.get("city") or "").strip(),
|
||||
"url": url,
|
||||
"posted": parse_dt(j.get("published_at") or j.get("created_at")),
|
||||
"department": (j.get("department") or "").strip(),
|
||||
"description": strip_html(j.get("description") or ""),
|
||||
"remote": j.get("remote"),
|
||||
"salary_text": _salary_text(salary),
|
||||
"salary_min": salary_min,
|
||||
"salary_max": salary_max,
|
||||
"currency": currency,
|
||||
})
|
||||
return rows
|
||||
@@ -0,0 +1,127 @@
|
||||
"""Rippling: `rippling:<board slug>` for ats.rippling.com/<slug>, the whole
|
||||
board in one JSON array, the description on each role's own record.
|
||||
|
||||
Written from Prisma's board (`prisma-careers`, two roles) and Rippling's own
|
||||
(`rippling`, 326 roles) on 2026-09-29. The careers site's script reads the
|
||||
public board API, `api.rippling.com/platform/api/ats/v1/board/<slug>/jobs`,
|
||||
which needs no key. The slug is case-sensitive (`Prisma-Careers` is "Job
|
||||
Board not found").
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._shared import Get, Row, parse_dt, yearly_amount
|
||||
|
||||
NAME = "rippling"
|
||||
LABEL = "Rippling"
|
||||
KEEP_CASE = True
|
||||
PATTERNS = (
|
||||
r"ats\.rippling\.com/([A-Za-z0-9][A-Za-z0-9_-]*)",
|
||||
r"api\.rippling\.com/platform/api/ats/v1/board/([A-Za-z0-9][A-Za-z0-9_-]*)",
|
||||
)
|
||||
|
||||
_API = "api.rippling.com"
|
||||
_SITE = "ats.rippling.com"
|
||||
|
||||
#: A slug is one path segment that starts with a letter or a digit, matched
|
||||
#: whole; it is also what discovery's patterns take.
|
||||
_SLUG = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]*")
|
||||
|
||||
#: A role's id in the board's own links.
|
||||
_UUID = re.compile(r"[0-9A-Za-z-]{8,64}")
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
return bool(_SLUG.fullmatch(slug))
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return lambda host: host.lower() in (_API, _SITE)
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
return f"https://{_SITE}/{slug}/jobs"
|
||||
|
||||
|
||||
def robots_page(slug: str) -> str:
|
||||
"""The page a person opens. The API host has no robots.txt (a 404, which
|
||||
allows), so the site's is the one that says something."""
|
||||
return careers_url(slug)
|
||||
|
||||
|
||||
def _board(slug: str) -> str:
|
||||
return f"https://{_API}/platform/api/ats/v1/board/{slug}"
|
||||
|
||||
|
||||
def feed_url(slug: str) -> str:
|
||||
return f"{_board(slug)}/jobs"
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
"""The board's array -> one row per role. The array has one entry per
|
||||
role and place, so a role in three cities is three entries with one
|
||||
`uuid` (Rippling's own board: 638 entries, 326 roles); the rows merge
|
||||
them, the places joined with "; ". The entry has no description, date
|
||||
or pay: `detail` reads them. `remote` is True only when every place is a
|
||||
"Remote (...)" one, since a label says nothing else about the work."""
|
||||
data = json.loads(body)
|
||||
if not isinstance(data, list):
|
||||
raise ValueError("not a Rippling board: the answer is not a list")
|
||||
roles: dict[str, Row] = {}
|
||||
places: dict[str, list[str]] = {}
|
||||
for entry in data:
|
||||
job_id = str(entry.get("uuid") or "").strip()
|
||||
title = (entry.get("name") or "").strip()
|
||||
if not title or not _UUID.fullmatch(job_id):
|
||||
continue
|
||||
place = str(((entry.get("workLocation") or {}).get("label")) or "").strip()
|
||||
places.setdefault(job_id, [])
|
||||
if place and place not in places[job_id]:
|
||||
places[job_id].append(place)
|
||||
roles.setdefault(job_id, {
|
||||
"source_id": job_id,
|
||||
"title": title,
|
||||
"url": f"https://{_SITE}/{slug}/jobs/{job_id}",
|
||||
"posted": None,
|
||||
"department": str(((entry.get("department") or {}).get("label")) or "").strip(),
|
||||
"description": "",
|
||||
"salary_text": "", # the board's list states no pay
|
||||
})
|
||||
for job_id, row in roles.items():
|
||||
row["location"] = "; ".join(places[job_id])
|
||||
row["remote"] = True if places[job_id] and all(
|
||||
p.lower().startswith("remote") for p in places[job_id]) else None
|
||||
return list(roles.values())
|
||||
|
||||
|
||||
def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
"""`<board>/jobs/<uuid>`: the description as HTML (`description.role`, or
|
||||
the company's text when a role has none), every place in `workLocations`,
|
||||
`createdOn` for `posted`, and `payRangeDetails`, a range per pay zone.
|
||||
The columns are only filled for exactly one yearly range, as Greenhouse's
|
||||
are: with several zones there is no telling which one the role has."""
|
||||
data = json.loads(get(f"{_board(slug)}/jobs/{row['source_id']}"))
|
||||
text = data.get("description") or {}
|
||||
found: dict[str, Any] = {
|
||||
"description": html.unescape(strip_html(text.get("role") or text.get("company") or "")),
|
||||
"posted": parse_dt(data.get("createdOn")),
|
||||
}
|
||||
if places := [p for p in (str(p).strip() for p in data.get("workLocations") or []) if p]:
|
||||
found["location"] = "; ".join(dict.fromkeys(places))
|
||||
ranges = data.get("payRangeDetails") or []
|
||||
if len(ranges) == 1:
|
||||
pay = ranges[0]
|
||||
lo, hi, currency = yearly_amount(pay.get("rangeStart"), pay.get("rangeEnd"),
|
||||
pay.get("currency"), pay.get("frequency"),
|
||||
{"YEAR": 1, "MONTH": 12})
|
||||
if currency:
|
||||
found |= {"salary_min": lo, "salary_max": hi, "currency": currency}
|
||||
return found
|
||||
@@ -0,0 +1,90 @@
|
||||
"""SmartRecruiters: `smartrecruiters:<company identifier>`, paged by offset
|
||||
with a stated total. The identifier is case-sensitive (`BoschGroup`).
|
||||
|
||||
Written from one real board on 2026-09-23; jobhunt-hq did not read
|
||||
SmartRecruiters.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok
|
||||
|
||||
NAME = "smartrecruiters"
|
||||
LABEL = "SmartRecruiters"
|
||||
RANK = 50
|
||||
PATTERNS = (
|
||||
r"(?:jobs|careers|api)\.smartrecruiters\.com/(?:v1/companies/)?([A-Za-z0-9_-]+)",
|
||||
)
|
||||
|
||||
_API = "api.smartrecruiters.com"
|
||||
|
||||
#: The page size asked for, which is also the most the host serves per page.
|
||||
PAGE_SIZE = 100
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
return path_slug_ok(slug)
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return only(_API)
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
return f"https://jobs.smartrecruiters.com/{slug}"
|
||||
|
||||
|
||||
def probe_url(slug: str) -> str:
|
||||
return f"https://{_API}/v1/companies/{slug}/postings?limit=1"
|
||||
|
||||
|
||||
def probe_ok(body: bytes) -> bool:
|
||||
"""An unknown identifier answers an empty listing, so a listing without
|
||||
postings counts as no board."""
|
||||
return b'"totalFound":' in body and b'"totalFound":0' not in body
|
||||
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any) -> Page:
|
||||
offset = int(cursor or 0)
|
||||
data = json.loads(get(f"https://{_API}/v1/companies/{slug}/postings"
|
||||
f"?offset={offset}&limit={PAGE_SIZE}"))
|
||||
content = data.get("content") or []
|
||||
return data, content, data.get("totalFound"), offset + len(content) if content else None
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
return rows(json.loads(body), slug)
|
||||
|
||||
|
||||
def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
"""One page of https://api.smartrecruiters.com/v1/companies/{slug}/postings
|
||||
-> `{"offset", "limit", "totalFound", "content": [...]}`. At most 100 a
|
||||
page, whatever `limit` asks for; the next is `?offset=` further on. The
|
||||
list omits the job description (only the per-posting detail endpoint has
|
||||
it, and fetching one per row would be hundreds of extra requests per
|
||||
board), so `description` stays empty here."""
|
||||
out = []
|
||||
for j in data.get("content") or []:
|
||||
title = (j.get("name") or "").strip()
|
||||
job_id = str(j.get("id") or "")
|
||||
if not title or not job_id:
|
||||
continue
|
||||
location = j.get("location") or {}
|
||||
loc_text = (location.get("fullLocation")
|
||||
or ", ".join(p for p in (location.get("city"), location.get("country")) if p))
|
||||
out.append({
|
||||
"source_id": job_id,
|
||||
"title": title,
|
||||
"location": (loc_text or "").strip(),
|
||||
"url": f"https://jobs.smartrecruiters.com/{slug}/{job_id}",
|
||||
"posted": parse_dt(j.get("releasedDate")),
|
||||
"department": ((j.get("department") or {}).get("label") or "").strip(),
|
||||
"description": "",
|
||||
"remote": location.get("remote"),
|
||||
"salary_text": "", # SmartRecruiters' list endpoint states no pay at all
|
||||
})
|
||||
return out
|
||||
@@ -0,0 +1,407 @@
|
||||
"""SAP SuccessFactors career sites (Recruiting Marketing): `successfactors:<host>`
|
||||
for the career site's own host name (`careers.vestas.com`), or
|
||||
`successfactors:<host>/<path prefix>` for the part of a shared site that is the
|
||||
company (`jobs.teradyne.com/MiR`, Universal Robots inside Teradyne). Paged HTML
|
||||
with a stated total.
|
||||
|
||||
Written from live sites on 2026-09-29 (Vestas, Demant, Scania, SAP, Novo
|
||||
Nordisk, Systematic and Teradyne); jobhunt-hq did not read SuccessFactors.
|
||||
|
||||
Only the career site's front end is read, and only the paths its robots.txt
|
||||
allows: `/search/` and `/job/`. Never `/services/` (the RSS feed, disallowed),
|
||||
and never the `career*.successfactors.*` or `*.sapsf.*` back end, whose
|
||||
robots.txt disallows everything; `allowed` admits the site's own host and no
|
||||
other, so a slug cannot point a fetch there. A company that has only the back
|
||||
end (Bang & Olufsen) is not readable. On these sites any unknown `*.xml` path
|
||||
answers with a feed of several megabytes, so no path but those two is asked.
|
||||
|
||||
The listing is the search page, `/search/?q=&locationsearch=&startrow=N`,
|
||||
which states a total and lists title, location and, on most sites, a date. It
|
||||
comes in two layouts, a table and tiles, both parsed by `rows`. The page size
|
||||
is the site's own (10, 15, 25 or 100) and cannot be set, so the next start row
|
||||
is the rows the last page returned. `sitemap.xml` is not used: it lists every
|
||||
locale, so a role written in two languages is there twice, and it carries no
|
||||
title or date.
|
||||
|
||||
**The search lists the default locale only.** A role that exists only in
|
||||
another locale (Carlsberg's French brand, 37 of 105) is not on the board this
|
||||
module reads, and neither is a translation of a listed role.
|
||||
|
||||
A path prefix names a brand of a shared site. `<host>/<prefix>/search/` lists
|
||||
only that brand on the server (Universal Robots 30 of Teradyne's 486, MiR 1),
|
||||
so the read is as small as the company; `rows` still keeps only the roles
|
||||
under `/<prefix>/job/`, in case a site does not scope. There is no server-side
|
||||
country filter in this version.
|
||||
|
||||
A role's own page carries microdata (`itemtype` schema.org JobPosting) with
|
||||
`datePosted` and the address; `detail` reads the description, the date and the
|
||||
location with its country from it. Novo Nordisk's pages have none: the
|
||||
description is still in `itemprop="description"`, and the listing's date and
|
||||
location stand.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin, urlsplit
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
|
||||
|
||||
NAME = "successfactors"
|
||||
LABEL = "SuccessFactors"
|
||||
# A career site is on the company's own domain and names no SuccessFactors
|
||||
# host, so discovery can read a board only off an absolute link to one of its
|
||||
# own paths that no other site has: a role, `/job/<title>/<numeric id>/`, a
|
||||
# landing page, `/go/<name>/<numeric id>/`, or a search with the query keys the
|
||||
# sites' own forms send (`createNewAlert`, `locationsearch`, `optionsFacetsDD_`).
|
||||
# The host and a brand segment before the path are the slug. A page that only
|
||||
# links the back end (`career5.successfactors.eu`, as Bang & Olufsen's does) or
|
||||
# loads the sites' image CDN names the host with no slug, which keeps the
|
||||
# company on the list of hosts the app names but has no board for.
|
||||
PATTERNS = (
|
||||
r"https?://([a-z0-9-]+(?:\.[a-z0-9-]+)+)(?:/([A-Za-z0-9_-]+))?"
|
||||
r"/(?:job/[^/\s\"'<>?#]+/\d{9,10}/|go/[^/\s\"'<>?#]+/\d{6,}/"
|
||||
r"|search/\?[^\s\"'<>]*(?:createNewAlert|locationsearch|optionsFacetsDD_))",
|
||||
r"(?:career|jobs|performancemanager)\d*\.successfactors\.(?:eu|com)|\.sapsf\.(?:eu|com)"
|
||||
r"|rmkcdn\.successfactors\.com",
|
||||
)
|
||||
# The prefix is case-sensitive (`MiR`, `Universal-Robots`), and so are the
|
||||
# robots.txt rules that name it.
|
||||
KEEP_CASE = True
|
||||
|
||||
#: A public DNS name in lower case with a letter-only last label, so an address
|
||||
#: (`127.0.0.1`) or a single word (`localhost`) never names a site, and an
|
||||
#: optional prefix of one or more path segments. `ats._get` refuses the local
|
||||
#: names again at every request.
|
||||
_SLUG = re.compile(r"((?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z]{2,})((?:/[A-Za-z0-9_-]+)*)")
|
||||
_LOCAL = (".local", ".localhost", ".internal", ".lan", ".home.arpa", ".arpa", ".test",
|
||||
".example", ".invalid")
|
||||
|
||||
#: The search pages' own layouts. A page that has none of these is not a
|
||||
#: search page (a bot wall or an error page answered with 200), which must not
|
||||
#: read as a board with no roles.
|
||||
_SEARCH_PAGE = re.compile(
|
||||
r'paginationLabel|tile-search-results-label|id="searchresults"|job-tile-list|class="data-row"')
|
||||
|
||||
#: The table layout: a row is `<tr class="data-row">`, with the job link
|
||||
#: (twice: once for the phone), `span.jobLocation`, `span.jobDate` and, on some
|
||||
#: sites, `span.jobDepartment`. The tile layout: `<li class="job-tile job-id-N">` and one labelled field per column.
|
||||
_ROW = re.compile(r'<tr\b[^>]*\bclass="[^"]*\bdata-row\b')
|
||||
_TILE = re.compile(r'<li\b[^>]*\bclass="job-tile[ "]')
|
||||
_LINK = re.compile(
|
||||
r'<a\b[^>]*\bhref="((?:https?://[^"/]+)?(?:/[^"/]+)*/job/[^"]*?/(\d+)/?)"[^>]*>(.*?)</a>', re.S)
|
||||
_SPAN = r'<span\b[^>]*\bclass="(?:[^"]* )?{}(?: [^"]*)?"[^>]*>(.*?)</span>'
|
||||
_TILE_VALUE = re.compile(r'id="job-\d+-desktop-section-([A-Za-z0-9]+)-value"[^>]*>(.*?)</div>', re.S)
|
||||
_TILE_LABEL = re.compile(r'id="job-\d+-desktop-section-([A-Za-z0-9]+)-label"[^>]*>(.*?)</span>', re.S)
|
||||
|
||||
#: The total: `Results <b>1 to 10</b> of <b>661</b>` (table) or `Showing 1 to 10 of
|
||||
#: 308 Jobs` (tiles); the last number of the label either way.
|
||||
_TABLE_TOTAL = re.compile(r'paginationLabel[^>]*>(.*?)</span>', re.S)
|
||||
_TILE_TOTAL = re.compile(r'id="tile-search-results-label"[^>]*>(.*?)</span>', re.S)
|
||||
|
||||
#: What a tile's labels call the place and its country, lower case. Demant
|
||||
#: gives the city as "Location" and the country name as "Country"; Scania and
|
||||
#: Carlsberg give a "Location" that ends in a country code.
|
||||
_PLACE_LABELS = ("location", "city", "town")
|
||||
_COUNTRY_LABELS = ("country", "land")
|
||||
|
||||
_MONTHS = {name: number for number, name in enumerate(
|
||||
("jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"), 1)}
|
||||
|
||||
#: ISO 3166-1 alpha-2 to the English name, which is what `country_of` reads:
|
||||
#: a bare "DE" or "SE" in a location line is taken for Delaware or nothing. A
|
||||
#: code that is not here stays as the site wrote it.
|
||||
_COUNTRIES = dict(entry.strip().split(" ", 1) for entry in """
|
||||
AD Andorra | AE United Arab Emirates | AF Afghanistan | AG Antigua and Barbuda | AI
|
||||
Anguilla | AL Albania | AM Armenia | AO Angola | AR Argentina | AS American Samoa | AT
|
||||
Austria | AU Australia | AW Aruba | AX Åland Islands | AZ Azerbaijan | BA Bosnia and
|
||||
Herzegovina | BB Barbados | BD Bangladesh | BE Belgium | BF Burkina Faso | BG Bulgaria
|
||||
| BH Bahrain | BI Burundi | BJ Benin | BM Bermuda | BN Brunei | BO Bolivia | BQ
|
||||
Caribbean Netherlands | BR Brazil | BS Bahamas | BT Bhutan | BW Botswana | BY Belarus
|
||||
| BZ Belize | CA Canada | CD Democratic Republic of the Congo | CF Central African
|
||||
Republic | CG Republic of the Congo | CH Switzerland | CI Côte d'Ivoire | CK Cook
|
||||
Islands | CL Chile | CM Cameroon | CN China | CO Colombia | CR Costa Rica | CU Cuba |
|
||||
CV Cabo Verde | CW Curaçao | CY Cyprus | CZ Czechia | DE Germany | DJ Djibouti | DK
|
||||
Denmark | DM Dominica | DO Dominican Republic | DZ Algeria | EC Ecuador | EE Estonia |
|
||||
EG Egypt | ER Eritrea | ES Spain | ET Ethiopia | FI Finland | FJ Fiji | FK Falkland
|
||||
Islands | FM Micronesia | FO Faroe Islands | FR France | GA Gabon | GB United Kingdom
|
||||
| GD Grenada | GE Georgia | GF French Guiana | GG Guernsey | GH Ghana | GI Gibraltar |
|
||||
GL Greenland | GM Gambia | GN Guinea | GP Guadeloupe | GQ Equatorial Guinea | GR
|
||||
Greece | GT Guatemala | GU Guam | GW Guinea-Bissau | GY Guyana | HK Hong Kong | HN
|
||||
Honduras | HR Croatia | HT Haiti | HU Hungary | ID Indonesia | IE Ireland | IL Israel
|
||||
| IM Isle of Man | IN India | IQ Iraq | IR Iran | IS Iceland | IT Italy | JE Jersey |
|
||||
JM Jamaica | JO Jordan | JP Japan | KE Kenya | KG Kyrgyzstan | KH Cambodia | KI
|
||||
Kiribati | KM Comoros | KN Saint Kitts and Nevis | KP North Korea | KR South Korea |
|
||||
KW Kuwait | KY Cayman Islands | KZ Kazakhstan | LA Laos | LB Lebanon | LC Saint Lucia
|
||||
| LI Liechtenstein | LK Sri Lanka | LR Liberia | LS Lesotho | LT Lithuania | LU
|
||||
Luxembourg | LV Latvia | LY Libya | MA Morocco | MC Monaco | MD Moldova | ME
|
||||
Montenegro | MG Madagascar | MH Marshall Islands | MK North Macedonia | ML Mali | MM
|
||||
Myanmar | MN Mongolia | MO Macao | MP Northern Mariana Islands | MQ Martinique | MR
|
||||
Mauritania | MS Montserrat | MT Malta | MU Mauritius | MV Maldives | MW Malawi | MX
|
||||
Mexico | MY Malaysia | MZ Mozambique | NA Namibia | NC New Caledonia | NE Niger | NG
|
||||
Nigeria | NI Nicaragua | NL Netherlands | NO Norway | NP Nepal | NR Nauru | NU Niue |
|
||||
NZ New Zealand | OM Oman | PA Panama | PE Peru | PF French Polynesia | PG Papua New
|
||||
Guinea | PH Philippines | PK Pakistan | PL Poland | PR Puerto Rico | PS Palestine | PT
|
||||
Portugal | PW Palau | PY Paraguay | QA Qatar | RE Réunion | RO Romania | RS Serbia |
|
||||
RU Russia | RW Rwanda | SA Saudi Arabia | SB Solomon Islands | SC Seychelles | SD
|
||||
Sudan | SE Sweden | SG Singapore | SI Slovenia | SK Slovakia | SL Sierra Leone | SM
|
||||
San Marino | SN Senegal | SO Somalia | SR Suriname | SS South Sudan | ST São Tomé and
|
||||
Príncipe | SV El Salvador | SY Syria | SZ Eswatini | TC Turks and Caicos Islands | TD
|
||||
Chad | TG Togo | TH Thailand | TJ Tajikistan | TL Timor-Leste | TM Turkmenistan | TN
|
||||
Tunisia | TO Tonga | TR Turkey | TT Trinidad and Tobago | TV Tuvalu | TW Taiwan | TZ
|
||||
Tanzania | UA Ukraine | UG Uganda | US United States | UY Uruguay | UZ Uzbekistan | VA
|
||||
Vatican City | VC Saint Vincent and the Grenadines | VE Venezuela | VG British Virgin
|
||||
Islands | VI U.S. Virgin Islands | VN Vietnam | VU Vanuatu | WS Samoa | XK Kosovo | YE
|
||||
Yemen | ZA South Africa | ZM Zambia | ZW Zimbabwe
|
||||
""".replace("\n", " ").split(" | "))
|
||||
|
||||
|
||||
def _parts(slug: str) -> tuple[str, str]:
|
||||
"""`(host, "/prefix" or "")` of a slug."""
|
||||
match = _SLUG.fullmatch(slug)
|
||||
return (match.group(1), match.group(2)) if match else ("", "")
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
host, _prefix = _parts(slug)
|
||||
return bool(host) and not host.endswith(_LOCAL)
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return only(_parts(slug)[0])
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
host, prefix = _parts(slug)
|
||||
return f"https://{host}{prefix}/search/"
|
||||
|
||||
|
||||
def robots_page(slug: str) -> str:
|
||||
return careers_url(slug)
|
||||
|
||||
|
||||
def _search_url(slug: str, start: int) -> str:
|
||||
host, prefix = _parts(slug)
|
||||
return f"https://{host}{prefix}/search/?q=&locationsearch=&startrow={start}"
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ listing
|
||||
|
||||
def _text(fragment: str) -> str:
|
||||
return html.unescape(strip_html(fragment))
|
||||
|
||||
|
||||
def _date(text: str) -> datetime | None:
|
||||
"""A listing's date as the site writes it in its language: "29 Sept 2026",
|
||||
"Sep 29, 2026", "2026/09/29" or "29.09.2026". A form that could be day or
|
||||
month first ("09/29/2026") is not guessed at and reads as no date."""
|
||||
text = _text(text)
|
||||
if match := re.search(r"(\d{1,2})\.?\s+([A-Za-z]{3})[A-Za-z]*\.?,?\s+(\d{4})", text):
|
||||
day, month, year = match.groups()
|
||||
elif match := re.search(r"([A-Za-z]{3})[A-Za-z]*\.?\s+(\d{1,2}),?\s+(\d{4})", text):
|
||||
month, day, year = match.groups()
|
||||
elif match := re.search(r"(\d{4})[-/.](\d{1,2})[-/.](\d{1,2})", text):
|
||||
year, month, day = match.groups()
|
||||
elif match := re.search(r"(\d{1,2})\.(\d{1,2})\.(\d{4})", text):
|
||||
day, month, year = match.groups()
|
||||
else:
|
||||
return None
|
||||
number = _MONTHS.get(month.lower()[:3]) or (int(month) if month.isdigit() else 0)
|
||||
try:
|
||||
return datetime(int(year), number, int(day), tzinfo=UTC)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _place(text: str) -> str:
|
||||
"""A site's location, "City, Region, CC, zip", as "City, Region, Country":
|
||||
the two-letter token near the end is the country and what follows it the
|
||||
postcode, which is dropped. The last such token wins, since a region can be
|
||||
one too ("Boston, MA, US"). A line without one is left as it is."""
|
||||
text = re.sub(r"\s+", " ", text).strip()
|
||||
parts = [part.strip() for part in text.split(",")]
|
||||
for i in range(len(parts) - 1, 0, -1):
|
||||
if len(parts) - i <= 2 and re.fullmatch(r"[A-Z]{2}", parts[i]) and parts[i] in _COUNTRIES:
|
||||
return ", ".join([*parts[:i], _COUNTRIES[parts[i]]])
|
||||
return text
|
||||
|
||||
|
||||
def _total(text: str) -> int | None:
|
||||
for pattern in (_TABLE_TOTAL, _TILE_TOTAL):
|
||||
if match := pattern.search(text):
|
||||
numbers = re.findall(r"\d[\d.,\u00a0 ]*", _text(match.group(1)))
|
||||
if numbers and (digits := re.sub(r"\D", "", numbers[-1])):
|
||||
return int(digits)
|
||||
return None
|
||||
|
||||
|
||||
def _table_fields(chunk: str) -> tuple[str, str, str]:
|
||||
"""`(place, date, department)` of a table row, from the spans the site marks
|
||||
them with."""
|
||||
def cell(name: str) -> str:
|
||||
found = re.search(_SPAN.format(name), chunk, re.S)
|
||||
return _text(found.group(1)) if found else ""
|
||||
return cell("jobLocation"), cell("jobDate"), cell("jobDepartment")
|
||||
|
||||
|
||||
def _tile_fields(chunk: str) -> tuple[str, str, str]:
|
||||
"""`(place, date, department)` of a tile. The fields differ per company, so
|
||||
the place and the country are found by their label and the date and the
|
||||
department by the field's own name, which is `date` and `department` on
|
||||
every site seen (Scania's "Work Area" is `department`)."""
|
||||
keyed = dict(_TILE_VALUE.findall(chunk))
|
||||
labelled = {_text(label).lower(): _text(keyed[key])
|
||||
for key, label in _TILE_LABEL.findall(chunk) if key in keyed}
|
||||
place = next((labelled[label] for label in _PLACE_LABELS if labelled.get(label)),
|
||||
_text(keyed.get("location", "")))
|
||||
country = next((labelled[label] for label in _COUNTRY_LABELS if labelled.get(label)), "")
|
||||
place = _place(place)
|
||||
if country and country.lower() not in place.lower():
|
||||
place = f"{place}, {country}" if place else country
|
||||
return place, _text(keyed.get("date", "")), _text(keyed.get("department", ""))
|
||||
|
||||
|
||||
def _entries(text: str) -> list[dict[str, Any]]:
|
||||
"""The roles of one search page, in the order shown, whichever layout it is
|
||||
in and before the prefix filter: `id`, `href` (as written, relative),
|
||||
`title`, `place`, `date` and `department`. A role is listed once, and a row
|
||||
without a link to a role is not one."""
|
||||
table = _ROW.split(text)[1:]
|
||||
chunks = table or _TILE.split(text)[1:]
|
||||
out: list[dict[str, Any]] = []
|
||||
seen: set[str] = set()
|
||||
for chunk in chunks:
|
||||
if table:
|
||||
chunk = chunk.split("</tr>", 1)[0]
|
||||
if not (link := _LINK.search(chunk)):
|
||||
continue
|
||||
href, job_id, title = html.unescape(link.group(1)), link.group(2), _text(link.group(3))
|
||||
if job_id in seen:
|
||||
continue
|
||||
seen.add(job_id)
|
||||
place, date, department = (_table_fields if table else _tile_fields)(chunk)
|
||||
out.append({"id": job_id, "href": href, "title": title, "place": _place(place),
|
||||
"date": date, "department": department})
|
||||
return out
|
||||
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any) -> Page:
|
||||
"""One search page. The cursor is `(start row, first role of the page
|
||||
before)`: the next start row is what this page returned, and a page that
|
||||
starts with the same role as the last one is the site ignoring `startrow`,
|
||||
which ends the read (incomplete) instead of looping."""
|
||||
start, previous = cursor or (0, None)
|
||||
text = get(_search_url(slug, start)).decode("utf-8", "replace")
|
||||
if not _SEARCH_PAGE.search(text):
|
||||
raise ValueError("not a SuccessFactors search page")
|
||||
found = _entries(text)
|
||||
total = _total(text)
|
||||
if found and found[0]["id"] == previous:
|
||||
return text, [], total, None
|
||||
more = bool(found) and (total is None or start + len(found) < total)
|
||||
return (text, [{"id": entry["id"]} for entry in found], total,
|
||||
(start + len(found), found[0]["id"]) if more else None)
|
||||
|
||||
|
||||
def rows(decoded: str, slug: str) -> list[Row]:
|
||||
"""One search page's roles. `href` is on the site's own host, under the
|
||||
slug's prefix when it has one; the id in it is the requisition's `source_id`.
|
||||
The table and the tile layout give a title, a location, a date and, where
|
||||
the site has the column, a department; nothing else about a role is on
|
||||
the list, so its description and pay are empty."""
|
||||
host, prefix = _parts(slug)
|
||||
base = f"https://{host}/"
|
||||
out = []
|
||||
for entry in _entries(decoded):
|
||||
url = urljoin(base, entry["href"])
|
||||
if urlsplit(url).hostname != host or not is_web_url(url):
|
||||
continue
|
||||
if prefix and not urlsplit(url).path.lower().startswith(f"{prefix.lower()}/job/"):
|
||||
continue
|
||||
if not entry["title"]:
|
||||
continue
|
||||
out.append({
|
||||
"source_id": entry["id"],
|
||||
"title": entry["title"],
|
||||
"location": entry["place"],
|
||||
"url": url,
|
||||
"posted": _date(entry["date"]),
|
||||
"department": entry["department"],
|
||||
"description": "",
|
||||
"remote": None, # the listing does not say
|
||||
"salary_text": "", # nor does it state pay
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- detail
|
||||
|
||||
_META = re.compile(r'<meta\b[^>]*\bitemprop="([A-Za-z]+)"[^>]*\bcontent="([^"]*)"', re.I)
|
||||
_JAVA_DATE = re.compile(r"[A-Za-z]{3} ([A-Za-z]{3}) (\d{1,2}) (\d\d:\d\d:\d\d) (?:UTC|GMT) (\d{4})")
|
||||
|
||||
|
||||
def _element(text: str, start: int) -> str:
|
||||
"""The inner HTML of the element whose start tag begins at `start`, up to
|
||||
its matching close tag: nested elements of the same name are counted."""
|
||||
open_end = text.index(">", start) + 1
|
||||
name = re.match(r"<([A-Za-z0-9]+)", text[start:]).group(1) # type: ignore[union-attr]
|
||||
depth = 1
|
||||
for tag in re.finditer(rf"<(/?){name}\b[^>]*>", text[open_end:], re.I):
|
||||
depth += -1 if tag.group(1) else 0 if tag.group(0).endswith("/>") else 1
|
||||
if not depth:
|
||||
return text[open_end:open_end + tag.start()]
|
||||
return text[open_end:]
|
||||
|
||||
|
||||
def _posted(value: str) -> datetime | None:
|
||||
"""`datePosted` is a Java `Date.toString()`, "Tue Sep 29 02:00:00 UTC 2026",
|
||||
on every site seen; an ISO date is read as well."""
|
||||
if match := _JAVA_DATE.fullmatch(value.strip()):
|
||||
month, day, clock, year = match.groups()
|
||||
return parse_dt(f"{year}-{_MONTHS[month.lower()]:02d}-{int(day):02d} {clock}")
|
||||
return parse_dt(value)
|
||||
|
||||
|
||||
def _location(meta: dict[str, str]) -> str:
|
||||
"""The address as "City, Country". Vestas and Demant state
|
||||
`addressLocality` and `addressCountry` (a code), the other sites one
|
||||
`streetAddress`, "City, CC, zip". The region is left out: the sites cut it
|
||||
to a few letters ("Seou" for Seoul-teukpyolsi)."""
|
||||
country = meta.get("addressCountry", "").strip()
|
||||
country = _COUNTRIES.get(country.upper(), country)
|
||||
if city := meta.get("addressLocality", "").strip():
|
||||
return f"{city}, {country}" if country else city
|
||||
if street := meta.get("streetAddress", "").strip():
|
||||
return _place(street)
|
||||
return country
|
||||
|
||||
|
||||
def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
"""A role's own page: `description` from the block marked
|
||||
`itemprop="description"`, which Novo Nordisk's pages have without the
|
||||
microdata around it, and, from a page that has the `JobPosting` microdata,
|
||||
`posted` (`datePosted`) and `location` with its country. A page that
|
||||
is not a role's (the site answers 200 for a closed one with its search) has
|
||||
no description block and gives an empty answer."""
|
||||
text = get(row["url"]).decode("utf-8", "replace")
|
||||
out: dict[str, Any] = {}
|
||||
match = re.search(r'<(?:span|div)\b[^>]*\bitemprop="description"', text)
|
||||
if match:
|
||||
# An employer's own layout can put a style block in the description.
|
||||
block = re.sub(r"<(style|script)\b.*?</\1>", " ", _element(text, match.start()),
|
||||
flags=re.S | re.I)
|
||||
if description := _text(block):
|
||||
out["description"] = description
|
||||
if re.search(r'itemtype="https?://schema\.org/JobPosting"', text):
|
||||
meta = {name: html.unescape(value) for name, value in _META.findall(text)}
|
||||
if posted := _posted(meta.get("datePosted", "")):
|
||||
out["posted"] = posted
|
||||
if place := _location(meta):
|
||||
out["location"] = place
|
||||
return out
|
||||
@@ -0,0 +1,137 @@
|
||||
"""Teamtailor: `teamtailor:<subdomain>` for acme.teamtailor.com, or, for a
|
||||
career site on the company's own domain, that whole host name:
|
||||
`teamtailor:careers.acme.com`. An RSS feed paged by offset that states no
|
||||
total.
|
||||
|
||||
Written from live feeds on 2026-09-29; jobhunt-hq did not read Teamtailor.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
|
||||
|
||||
NAME = "teamtailor"
|
||||
LABEL = "Teamtailor"
|
||||
RANK = 80
|
||||
PATTERNS = (
|
||||
r"https?://([a-z0-9-]+)\.teamtailor\.com",
|
||||
# No slug: a career site on the company's own domain, which only loads
|
||||
# Teamtailor's CDN. Discovery reads its board off the page's host.
|
||||
r"cdn\.teamtailor\.com|teamtailor-cdn|teamtailor\.com/assets",
|
||||
)
|
||||
|
||||
#: The page size, asked for as `per_page`. A feed with neither parameter
|
||||
#: answers with every role (73 of 73 at Shine, 2026-09-29), but says nowhere
|
||||
#: that it is the whole list, so this reads by page to an empty one;
|
||||
#: `per_page=200` was answered in full too, and the cursor advances by what a
|
||||
#: page returned, so a smaller cap on the host's side would not skip a role.
|
||||
PAGE_SIZE = 50
|
||||
|
||||
#: Only a DNS name with a letter in its last label is a site, so an address
|
||||
#: (`127.0.0.1`, `[::1]`) or a single word never names a host the fetch could
|
||||
#: be pointed at; `ats._get` still refuses every host but this one, redirects
|
||||
#: included.
|
||||
_HOST = re.compile(r"[a-z0-9-]+(\.[a-z0-9-]+)*")
|
||||
|
||||
#: Teamtailor's own namespace in the feed, the `tt:` of `tt:locations`.
|
||||
_TT = "{https://teamtailor.com/locations}"
|
||||
|
||||
#: Teamtailor's `remoteStatus` -> `store._remote`'s words. `none` says nothing
|
||||
#: about where the role is worked and `temporary` is remote for a spell only,
|
||||
#: so both stay unknown rather than become "onsite" or "remote".
|
||||
_REMOTE = {"fully": "remote", "hybrid": "hybrid", "onsite": "onsite"}
|
||||
|
||||
|
||||
def site(slug: str) -> str:
|
||||
"""The host a career site is served from: `acme` is acme.teamtailor.com,
|
||||
and a slug with a dot is a site on the company's own domain,
|
||||
`careers.acme.com`, taken whole (the feed path is the same on both)."""
|
||||
slug = slug.strip().lower()
|
||||
return slug if "." in slug else f"{slug}.teamtailor.com"
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
host = site(slug)
|
||||
return bool(slug) and slug == slug.strip() and bool(_HOST.fullmatch(host)) \
|
||||
and not host.rsplit(".", 1)[-1].isdigit()
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return only(site(slug))
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
return f"https://{site(slug)}/jobs"
|
||||
|
||||
|
||||
def probe_url(slug: str) -> str:
|
||||
return f"https://{site(slug)}/jobs.rss"
|
||||
|
||||
|
||||
def probe_ok(body: bytes) -> bool:
|
||||
return b"<item>" in body
|
||||
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any) -> Page:
|
||||
offset = int(cursor or 0)
|
||||
items = _items(get(f"https://{site(slug)}/jobs.rss?per_page={PAGE_SIZE}&offset={offset}"))
|
||||
results = [{"id": item.findtext("guid") or item.findtext("link") or ""} for item in items]
|
||||
return items, results, None, offset + len(items) if items else None
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
return rows(_items(body), slug)
|
||||
|
||||
|
||||
def _items(body: bytes) -> list[ET.Element]:
|
||||
"""`{site}/jobs.rss` -> `<rss><channel><item>...`, the feed every Teamtailor
|
||||
career site serves (checked live on a dozen sites, 2026-09-29). A missing
|
||||
`channel` is a page that is not a feed, and raises."""
|
||||
channel = ET.fromstring(body).find("channel")
|
||||
if channel is None:
|
||||
raise ValueError("not a Teamtailor feed: no channel")
|
||||
return channel.findall("item")
|
||||
|
||||
|
||||
def rows(items: list[ET.Element], slug: str) -> list[Row]:
|
||||
"""One page of items. The item has the title, the job URL (on the site's
|
||||
own domain), a `guid` uuid, `pubDate`, the HTML `description` in full,
|
||||
`remoteStatus` (fully, hybrid, onsite, none or temporary), and, in the
|
||||
`tt:` namespace, `locations` (a list: `city` and `country` each, `name` free
|
||||
text), `department` and `role`, either of which may be empty. `place` joins
|
||||
the distinct locations with "; ", which `country_of` reads a location at a
|
||||
time. No pay is stated anywhere in the feed."""
|
||||
out = []
|
||||
for item in items:
|
||||
title = (item.findtext("title") or "").strip()
|
||||
url = (item.findtext("link") or "").strip()
|
||||
if not title or not is_web_url(url):
|
||||
continue
|
||||
places: list[str] = []
|
||||
for loc in item.findall(f"{_TT}locations/{_TT}location"):
|
||||
city = (loc.findtext(f"{_TT}city") or loc.findtext(f"{_TT}name") or "").strip()
|
||||
country = (loc.findtext(f"{_TT}country") or "").strip()
|
||||
place = city if country == city else ", ".join(p for p in (city, country) if p)
|
||||
if place and place not in places:
|
||||
places.append(place)
|
||||
out.append({
|
||||
"source_id": (item.findtext("guid") or url).strip(),
|
||||
"title": title,
|
||||
"location": "; ".join(places),
|
||||
"url": url,
|
||||
"posted": parse_dt(item.findtext("pubDate")),
|
||||
"department": ((item.findtext(f"{_TT}department") or "").strip()
|
||||
or (item.findtext(f"{_TT}role") or "").strip()),
|
||||
"description": html.unescape(strip_html(item.findtext("description") or "")),
|
||||
"remote": _REMOTE.get((item.findtext("remoteStatus") or "").strip()),
|
||||
"salary_text": "", # Teamtailor's feed states no pay at all
|
||||
})
|
||||
return out
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Workable: `workable:<account>`, paged ten roles at a time with a stated
|
||||
total.
|
||||
|
||||
Written from one real board, Hugging Face's, on 2026-09-28, when the first
|
||||
board with roles was seen; jobhunt-hq did not read Workable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok
|
||||
|
||||
NAME = "workable"
|
||||
LABEL = "Workable"
|
||||
RANK = 40
|
||||
PATTERNS = (
|
||||
r"apply\.workable\.com/(?:api/v\d/accounts/)?([A-Za-z0-9_-]+)",
|
||||
r"https?://(?!apply\.|www\.)([a-z0-9-]+)\.workable\.com",
|
||||
)
|
||||
|
||||
_API = "apply.workable.com"
|
||||
|
||||
# No probe: Workable's API sits behind a rate limit that a run of guesses
|
||||
# trips, and a Workable site links its board plainly anyway.
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
return path_slug_ok(slug)
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return only(_API)
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
return f"https://{_API}/{slug}"
|
||||
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any) -> Page:
|
||||
"""One POST of the list; the next page is the same POST with
|
||||
`{"token": nextPage}`, and the last page has no `nextPage`."""
|
||||
url = f"https://{_API}/api/v3/accounts/{slug}/jobs"
|
||||
data = json.loads(get(url, method="POST", payload={"token": cursor} if cursor else {}))
|
||||
return data, data.get("results") or [], data.get("total"), data.get("nextPage") or None
|
||||
|
||||
|
||||
def parse(body: bytes, slug: str) -> list[Row]:
|
||||
return rows(json.loads(body), slug)
|
||||
|
||||
|
||||
def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
"""One page of POST https://apply.workable.com/api/v3/accounts/{slug}/jobs
|
||||
-> `{"total": int, "results": [...], "nextPage": token}`, as Hugging
|
||||
Face's board answered on 2026-09-28. A result carries no URL and no text:
|
||||
the job page is `https://apply.workable.com/{slug}/j/{shortcode}/`, and
|
||||
the description is only on the per-job endpoint, one request per role,
|
||||
so `description` stays empty here as it does for SmartRecruiters.
|
||||
`location` is an object and `department` a list; `remote` is a flag
|
||||
beside `workplace` (remote, hybrid, on_site)."""
|
||||
out = []
|
||||
for j in data.get("results") or []:
|
||||
title = (j.get("title") or "").strip()
|
||||
shortcode = (j.get("shortcode") or "").strip()
|
||||
if not title or not shortcode:
|
||||
continue
|
||||
loc = j.get("location") or {}
|
||||
location = ", ".join(p for p in (loc.get("city"), loc.get("region"), loc.get("country")) if p)
|
||||
dept = j.get("department") or ""
|
||||
if isinstance(dept, list):
|
||||
dept = ", ".join(d for d in dept if d)
|
||||
out.append({
|
||||
"source_id": str(j.get("id") or shortcode),
|
||||
"title": title,
|
||||
"location": location,
|
||||
"url": f"https://{_API}/{slug}/j/{shortcode}/",
|
||||
"posted": parse_dt(j.get("published")),
|
||||
"department": dept.strip(),
|
||||
"description": "",
|
||||
"remote": bool(j.get("remote")) or j.get("workplace") == "remote",
|
||||
"salary_text": "", # Workable's list endpoint states no pay at all
|
||||
})
|
||||
return out
|
||||
@@ -0,0 +1,233 @@
|
||||
"""Workday: `workday:<tenant>.<wdN>/<Site>`, for example
|
||||
`workday:lego.wd103/LEGO_External` (the career site
|
||||
https://lego.wd103.myworkdayjobs.com/LEGO_External), paged by offset with a
|
||||
total that only the first page states.
|
||||
|
||||
The tenant is what the company was named when Workday was set up and is not
|
||||
always the company (`unitytech`, `novonesis`); the data centre is not
|
||||
derivable either, and a tenant has several sites, so all three are stored
|
||||
as they are found. The site is case-insensitive for the API (Red Hat's
|
||||
`jobs` and `Jobs` both work) but its robots.txt paths are not, so the slug
|
||||
keeps the case the URL wrote (`KEEP_CASE`).
|
||||
|
||||
Written from live career sites on 2026-09-29 (`docs/plan-hosts/workday.md`).
|
||||
The listing carries no description, so `detail` reads each role's own JSON.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._shared import Get, Page, Row, only, parse_dt
|
||||
|
||||
NAME = "workday"
|
||||
LABEL = "Workday"
|
||||
KEEP_CASE = True
|
||||
#: A tenant answers `/jobs` and `/Jobs` alike, so a rule in either case
|
||||
#: forbids the site.
|
||||
ROBOTS_ANY_CASE = True
|
||||
|
||||
#: One group for `<tenant>.<wdN>` and one for the site, which discovery joins
|
||||
#: into the slug with "/". A locale segment (`/en-US/`, `/de-DE/`) may come
|
||||
#: between; it is matched case-sensitively, and a bare `/en-US` that is no
|
||||
#: site at all is refused, so a site is never mistaken for a locale or the
|
||||
#: other way round. `wday/` is the API path, read by the second pattern. The
|
||||
#: tenant is lower case only: a slug with capitals in it is no slug.
|
||||
_TENANT = r"(?-i:[a-z0-9-]+)"
|
||||
_LOCALE = r"(?-i:[a-z]{2}-[A-Z]{2})"
|
||||
PATTERNS = (
|
||||
rf"(?<![A-Za-z0-9.-])({_TENANT}\.wd\d+)\.myworkdayjobs\.com/(?:{_LOCALE}/)?"
|
||||
rf"(?!wday/)(?!{_LOCALE}(?![A-Za-z0-9_-]))([A-Za-z0-9_-]+)",
|
||||
rf"(?<![A-Za-z0-9.-])({_TENANT}\.wd\d+)\.myworkdayjobs\.com/wday/cxs/{_TENANT}/"
|
||||
r"([A-Za-z0-9_-]+)",
|
||||
)
|
||||
|
||||
#: The page size asked for, which is also the only one the host serves: 21 or
|
||||
#: more answers 400 on every tenant tried.
|
||||
PAGE_SIZE = 20
|
||||
|
||||
#: Matched whole, so a trailing newline is no slug.
|
||||
_SLUG = re.compile(r"([a-z0-9-]+)\.(wd\d+)/([A-Za-z0-9_-]+)")
|
||||
|
||||
#: The path of one role on its site, `externalPath` in the listing
|
||||
#: (`/job/...`): a path, and nothing that adds a query or a fragment.
|
||||
_ROLE_PATH = re.compile(r"/[^\s?#]+")
|
||||
|
||||
|
||||
def _parts(slug: str) -> tuple[str, str, str] | None:
|
||||
"""`(tenant, data centre, site)`, or None for a slug that is none."""
|
||||
match = _SLUG.fullmatch(slug)
|
||||
return match.groups() if match else None # type: ignore[return-value]
|
||||
|
||||
|
||||
def _host(slug: str) -> str:
|
||||
tenant, dc, _site = _parts(slug) or ("", "", "")
|
||||
return f"{tenant}.{dc}.myworkdayjobs.com"
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
return _parts(slug) is not None
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
"""Only the tenant's own host. A slug that is none admits nothing, rather
|
||||
than a host the company does not own."""
|
||||
return only(_host(slug)) if _parts(slug) else (lambda host: False)
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
return f"https://{_host(slug)}/{_parts(slug)[2]}" # type: ignore[index]
|
||||
|
||||
|
||||
def robots_page(slug: str) -> str:
|
||||
"""With the trailing slash, since the rules the tenants write are
|
||||
`Disallow: /<Site>/` and read as prefixes: `/<Site>` alone would slip
|
||||
past the very rule that forbids the site."""
|
||||
return careers_url(slug) + "/"
|
||||
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any) -> Page:
|
||||
"""`cursor` is `(offset, total)`. `total` is read on the first page only:
|
||||
on the later ones it is 0 on most tenants and the real figure on a few,
|
||||
so it is carried. An offset at or past the end does not answer an empty
|
||||
list but the first page again, so the read stops on `offset >= total`
|
||||
rather than on an empty page."""
|
||||
tenant, _dc, site = _parts(slug) # type: ignore[misc]
|
||||
offset, total = cursor or (0, None)
|
||||
data = json.loads(get(
|
||||
f"https://{_host(slug)}/wday/cxs/{tenant}/{site}/jobs", method="POST",
|
||||
payload={"appliedFacets": {}, "limit": PAGE_SIZE, "offset": offset, "searchText": ""}))
|
||||
if cursor is None:
|
||||
stated = data.get("total")
|
||||
total = stated if isinstance(stated, int) and not isinstance(stated, bool) else None
|
||||
postings = data.get("jobPostings") or []
|
||||
# Every posting is counted, a path-less one (LEGO lists one that has only
|
||||
# `bulletFields`) by its position, so the parser dropping it does not
|
||||
# leave the board short of its total.
|
||||
results = [{"id": _posting_id(p, offset + i)} for i, p in enumerate(postings)]
|
||||
following = offset + len(postings)
|
||||
more = bool(postings) and (total is None or following < total)
|
||||
return data, results, total, (following, total) if more else None
|
||||
|
||||
|
||||
def _posting_id(posting: dict[str, Any], position: int) -> str:
|
||||
return str(posting.get("externalPath") or "".join(posting.get("bulletFields") or [])
|
||||
or f"#{position}")
|
||||
|
||||
|
||||
def _posted(text: str) -> datetime | None:
|
||||
"""The day of "Posted Today", "Posted Yesterday" and "Posted 5 Days Ago"
|
||||
at midnight UTC; None for "Posted 30+ Days Ago", which says only that it
|
||||
is older, and for a text in another language. Workday counts the days in
|
||||
its own time zone, so the day can be one off; `detail` states the date
|
||||
itself and replaces this one."""
|
||||
match = re.fullmatch(r"Posted (?:(Today)|(Yesterday)|(\d+) Days? Ago)", (text or "").strip())
|
||||
if not match:
|
||||
return None
|
||||
days = 0 if match.group(1) else 1 if match.group(2) else int(match.group(3))
|
||||
return _today() - timedelta(days=days)
|
||||
|
||||
|
||||
def _today() -> datetime:
|
||||
return datetime.now(UTC).replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
|
||||
|
||||
#: `remoteType` -> `store._remote`'s words. Only some tenants send it, and
|
||||
#: each words it its own way (Zendesk's "Fully Flexible", Axis' "On-Site"); a
|
||||
#: word not listed, a bare "No" among them, says nothing and stays unknown.
|
||||
_REMOTE = {"remote": "remote", "fully flexible": "remote", "hybrid": "hybrid",
|
||||
"in office": "onsite", "on-site": "onsite", "onsite": "onsite"}
|
||||
|
||||
|
||||
def _requisition(posting: dict[str, Any], path: str) -> str:
|
||||
"""The requisition id, which `bulletFields` lead with, if the path ends in
|
||||
it (`..._R-059566`, and `..._R-059566-2` for a repost of it). Where a
|
||||
tenant puts something else first, a place, the path is the id."""
|
||||
first = next(iter(posting.get("bulletFields") or []), None)
|
||||
name = path.rsplit("/", 1)[-1]
|
||||
return first if isinstance(first, str) and re.fullmatch(
|
||||
rf".*_{re.escape(first)}(?:-\d+)?", name) else ""
|
||||
|
||||
|
||||
def _remote(value: Any) -> str | None:
|
||||
return _REMOTE.get(str(value or "").strip().lower())
|
||||
|
||||
|
||||
def rows(decoded: dict[str, Any], slug: str) -> list[Row]:
|
||||
"""One page of `.../wday/cxs/<tenant>/<Site>/jobs` -> `{"total",
|
||||
"jobPostings": [...], "facets": [...]}`. A posting has `title`,
|
||||
`externalPath` (`/job/<place>/<title>_<requisition id>`), `locationsText`
|
||||
(a place, or "2 Locations"), `postedOn` (relative text) and
|
||||
`bulletFields`, which lead with the requisition id, and on some tenants
|
||||
`remoteType`. No description, no date and no country: `detail` has
|
||||
those. The id is the requisition id where the path carries it, and the
|
||||
path where not; a posting with no title or no path is skipped."""
|
||||
out = []
|
||||
for p in decoded.get("jobPostings") or []:
|
||||
title = (p.get("title") or "").strip()
|
||||
path = (p.get("externalPath") or "").strip()
|
||||
if not title or not _ROLE_PATH.fullmatch(path):
|
||||
continue
|
||||
out.append({
|
||||
"source_id": _requisition(p, path) or path,
|
||||
"title": title,
|
||||
"location": (p.get("locationsText") or "").strip(),
|
||||
"url": f"https://{_host(slug)}/{_parts(slug)[2]}{path}", # type: ignore[index]
|
||||
"posted": _posted(p.get("postedOn") or ""),
|
||||
"department": "",
|
||||
"description": "",
|
||||
"remote": _remote(p.get("remoteType")),
|
||||
"salary_text": "", # no field for pay, only the description's text
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
"""The role's own JSON, `.../wday/cxs/<tenant>/<Site><externalPath>`. The
|
||||
path is taken back off the row's URL, which was built from it. Pay is in
|
||||
the description text when it is anywhere: there is no field for it on
|
||||
any tenant. `startDate` is the posting date (it matched the "Posted N
|
||||
Days Ago" text in every sample), not a start of employment."""
|
||||
tenant, _dc, site = _parts(slug) # type: ignore[misc]
|
||||
prefix = f"https://{_host(slug)}/{site}"
|
||||
path = str(row.get("url") or "").removeprefix(prefix)
|
||||
if path == row.get("url") or not _ROLE_PATH.fullmatch(path) or ".." in path.split("/"):
|
||||
raise ValueError(f"{row.get('url')!r} is not a role of {slug}")
|
||||
info = json.loads(get(f"https://{_host(slug)}/wday/cxs/{tenant}/{site}{path}"))
|
||||
posting = info.get("jobPostingInfo")
|
||||
if not isinstance(posting, dict):
|
||||
raise ValueError("not a Workday role: no jobPostingInfo")
|
||||
out: dict[str, Any] = {}
|
||||
if description := " ".join(html.unescape(strip_html(posting.get("jobDescription") or ""))
|
||||
.split()):
|
||||
out["description"] = description
|
||||
if location := _location(posting):
|
||||
out["location"] = location
|
||||
if posted := parse_dt(posting.get("startDate")):
|
||||
out["posted"] = posted
|
||||
if remote := _remote(posting.get("remoteType")):
|
||||
out["remote"] = remote
|
||||
return out
|
||||
|
||||
|
||||
def _location(posting: dict[str, Any]) -> str:
|
||||
"""The main location and the others, joined by "; " as Teamtailor does, so
|
||||
that `country_of` reads them one at a time and the listing's "2 Locations"
|
||||
is replaced by places. A single location that does not name its country
|
||||
(LEGO's "Billund") gets it from the role's `country`."""
|
||||
places: list[str] = []
|
||||
for place in [posting.get("location"), *(posting.get("additionalLocations") or [])]:
|
||||
place = str(place or "").strip()
|
||||
if place and place not in places:
|
||||
places.append(place)
|
||||
country = str((posting.get("country") or {}).get("descriptor") or "").strip()
|
||||
if len(places) == 1 and country and country.lower() not in places[0].lower():
|
||||
places[0] += f", {country}"
|
||||
return "; ".join(places)
|
||||
Reference in new issue
Block a user