Merge branch 'wp-d1-hosts'

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 09:32:46 +02:00
commit 1d7b286374
36 files changed
+1297 -366

No files matched your search

+58 -6
View File
@@ -8,12 +8,17 @@ The field map is ported from `openings.go` in
from __future__ import annotations
import json
import logging
import re
from collections.abc import Callable
from typing import Any
from desk.sources import strip_html
from ._shared import Row, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, yearly_amount
from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import is_web_url, num, only, parse_dt, path_slug_ok, pay_line, yearly_amount
log = logging.getLogger(__name__)
NAME = "ashby"
LABEL = "Ashby"
@@ -23,8 +28,23 @@ PATTERNS = (
r"api\.ashbyhq\.com/posting-api/job-board/([A-Za-z0-9_.-]+)",
)
READS = Reads(
scope="board", listing="whole", text="listing", complete="whole",
places="many", empty="suspect",
fields=Fields(pay="figures", posted="trusted", remote="stated", employment="stated"),
robots="api: no robots.txt check needed (public board API), 2026-09-30",
)
_API = "api.ashbyhq.com"
#: `workplaceType` -> `store._remote`'s words. `isRemote` is true for hybrid
#: roles too, so it is only the fallback when the role has no `workplaceType`.
_WORKPLACE = {"remote": "remote", "hybrid": "hybrid", "onsite": "onsite"}
#: `employmentType` -> the app's employment words.
_EMPLOYMENT = {"fulltime": "full_time", "parttime": "part_time", "intern": "intern",
"contract": "contract", "temporary": "temporary"}
def slug_ok(slug: str) -> bool:
return path_slug_ok(slug)
@@ -38,7 +58,7 @@ def careers_url(slug: str) -> str:
return f"https://jobs.ashbyhq.com/{slug}"
def feed_url(slug: str) -> str:
def feed_url(slug: str, ask: Ask) -> str:
return f"https://{_API}/posting-api/job-board/{slug}?includeCompensation=true"
@@ -94,6 +114,32 @@ def _salary_text(compensation: dict[str, Any]) -> str:
or compensation.get("scrapeableCompensationSalarySummary") or "")
def _places(j: dict[str, Any], workplace: str | None) -> list[Place]:
"""The primary `location`, then each of `secondaryLocations`, in the host's
words. The primary is remote or an office as `workplaceType` says; a
place that names itself remote is remote."""
primary = (j.get("location") or "").strip()
named = [(primary, {"remote": True, "hybrid": False, "onsite": False}.get(workplace or ""))]
named += [(str((s or {}).get("location") or "").strip(), None)
for s in j.get("secondaryLocations") or []]
out: list[Place] = []
for text, remote in named:
if text and text not in (p["text"] for p in out):
out.append({"text": text, "remote": True if re.search(r"\bremote\b", text, re.I)
else remote})
return out
def _employment(value: Any) -> str | None:
word = str(value or "").strip()
if not word:
return None
found = _EMPLOYMENT.get(word.lower())
if found is None:
log.debug("ashby: unknown employmentType %r", word)
return found
def parse(body: bytes, slug: str) -> list[Row]:
data = json.loads(body)
rows = []
@@ -105,18 +151,24 @@ def parse(body: bytes, slug: str) -> list[Row]:
description = j.get("descriptionHtml") or j.get("descriptionPlain") or ""
compensation = j.get("compensation") or {}
salary_min, salary_max, currency = _salary(compensation.get("summaryComponents") or [])
rows.append({
workplace = _WORKPLACE.get(str(j.get("workplaceType") or "").strip().lower())
places = _places(j, workplace)
row: Row = {
"source_id": str(j.get("id") or ""),
"title": title,
"location": (j.get("location") or "").strip(),
"places": places,
"location": "; ".join(p["text"] for p in places),
"url": url,
"posted": parse_dt(j.get("publishedAt")),
"department": (j.get("department") or j.get("team") or "").strip(),
"description": strip_html(description),
"remote": j.get("isRemote"),
"remote": workplace or j.get("isRemote"),
"salary_text": _salary_text(compensation),
"salary_min": salary_min,
"salary_max": salary_max,
"currency": currency,
})
}
if employment := _employment(j.get("employmentType")):
row["employment"] = employment
rows.append(row)
return rows
+46 -8
View File
@@ -16,12 +16,31 @@ from __future__ import annotations
import html
import json
import logging
import re
from collections.abc import Callable
from typing import Any
from desk.sources import strip_html
from ._shared import Get, Row, label_slug_ok, only, parse_dt
from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import Get, label_slug_ok, only, parse_dt
log = logging.getLogger(__name__)
READS = Reads(
scope="board", listing="whole", text="detail", complete="whole",
places="one",
fields=Fields(posted="trusted", remote="stated", employment="stated"),
robots="read: robots.txt allows /careers (checked 2026-09-29)",
)
#: `employmentStatusLabel` -> the app's employment words. BambooHR's labels are
#: the company's own ("Full-Time", "Part-Time", "Contract", "Intern").
_EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "part-time": "part_time",
"part time": "part_time", "contract": "contract", "contractor": "contract",
"temporary": "temporary", "intern": "intern", "internship": "intern",
"freelance": "freelance"}
NAME = "bamboohr"
LABEL = "BambooHR"
@@ -54,11 +73,7 @@ def careers_url(slug: str) -> str:
return f"https://{host(slug)}/careers"
def robots_page(slug: str) -> str:
return careers_url(slug)
def feed_url(slug: str) -> str:
def feed_url(slug: str, ask: Ask) -> str:
return f"https://{host(slug)}/careers/list"
@@ -79,6 +94,23 @@ def _place(j: dict[str, Any]) -> str:
return ", ".join(str(p).strip() for p in parts if p and str(p).strip())
def _places(j: dict[str, Any]) -> list[Place]:
"""The one place BambooHR states, if it states any."""
text = _place(j)
return [{"text": text, "remote": True if re.search(r"\bremote\b", text, re.I) else None}] \
if text else []
def _employment(j: dict[str, Any]) -> str | None:
label = str(j.get("employmentStatusLabel") or "").strip()
if not label:
return None
found = _EMPLOYMENT.get(label.lower())
if found is None:
log.debug("bamboohr: unknown employmentStatusLabel %r", label)
return found
def parse(body: bytes, slug: str) -> list[Row]:
"""`/careers/list` -> `{"meta": {"totalCount": n}, "result": [...]}`. A
role has `id`, `jobOpeningName`, `departmentLabel`, `location` (`city`,
@@ -99,9 +131,11 @@ def parse(body: bytes, slug: str) -> list[Row]:
source_id = str(j.get("id") or "")
if not title or not source_id:
continue
out.append({
places = _places(j)
row: Row = {
"source_id": source_id,
"title": title,
"places": places,
"location": _place(j),
"url": f"https://{host(slug)}/careers/{source_id}",
"posted": None,
@@ -109,7 +143,10 @@ def parse(body: bytes, slug: str) -> list[Row]:
"description": "",
"remote": _remote(j),
"salary_text": "", # the list states no pay; the detail's `compensation` was null
})
}
if employment := _employment(j):
row["employment"] = employment
out.append(row)
return out
@@ -130,6 +167,7 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
}
if place := _place(opening):
out["location"] = place
out["places"] = _places(opening)
if remote := _remote(opening):
out["remote"] = remote
return out
+41 -9
View File
@@ -16,13 +16,25 @@ from __future__ import annotations
import html
import json
import logging
import re
from collections.abc import Callable
from typing import Any
from desk.sources import strip_html
from ._shared import Row, num, only, parse_dt, pay_line
from ._contract import Ask, Fields, Reads, Row
from ._shared import num, only, parse_dt, pay_line
log = logging.getLogger(__name__)
READS = Reads(
scope="board", listing="whole", text="listing", complete="whole",
places="many",
fields=Fields(pay="line", posted="listed", remote="stated", employment="stated"),
robots="read: robots.txt allows the board page /<slug>; /api/ is disallowed and not read "
"(checked 2026-09-29)",
)
NAME = "deel"
LABEL = "Deel"
@@ -50,6 +62,12 @@ _TEXT = re.compile(rb"T([0-9a-f]+),")
#: Deel's `workArrangementEnum` -> `store._remote`'s words.
_REMOTE = {"ON_SITE": "onsite", "HYBRID": "hybrid", "REMOTE": "remote"}
#: The name of a `jobEmploymentTypes` entry -> the app's employment words.
_EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "part-time": "part_time",
"part time": "part_time", "contract": "contract", "contractor": "contract",
"consultant": "contract", "temporary": "temporary", "intern": "intern",
"internship": "intern", "freelance": "freelance"}
def slug_ok(slug: str) -> bool:
return bool(_SLUG.fullmatch(slug))
@@ -63,11 +81,7 @@ def careers_url(slug: str) -> str:
return f"https://{_HOST}/{slug}"
def robots_page(slug: str) -> str:
return careers_url(slug)
def feed_url(slug: str) -> str:
def feed_url(slug: str, ask: Ask) -> str:
return careers_url(slug)
@@ -106,9 +120,21 @@ def _postings(body: bytes) -> tuple[list[dict[str, Any]], dict[str, str]]:
raise ValueError("not a Deel job board: no jobPostings in the page")
def _employment(job: dict[str, Any]) -> str | None:
"""The first `jobEmploymentTypes` entry that names a known type."""
for entry in job.get("jobEmploymentTypes") or []:
name = str(((entry or {}).get("employmentType") or {}).get("name") or "").strip()
if not name:
continue
if found := _EMPLOYMENT.get(name.lower()):
return found
log.debug("deel: unknown employment type %r", name)
return None
def parse(body: bytes, slug: str) -> list[Row]:
"""`jobs.deel.com/<slug>` -> one row per posting. `location` joins the
role's places with "; ". `posted` is the day the posting was published
role's places with "; ", and `places` lists each. `posted` is the day the posting was published
(its publication's state), not the day the role was drafted.
Deel states pay as a currency and two amounts with no period, and the
@@ -132,9 +158,12 @@ def parse(body: bytes, slug: str) -> list[Row]:
description = p.get("richtextDescription") or ""
if description.startswith("$"):
description = stream.get(description[1:], "")
out.append({
arrangement = str(job.get("workArrangementEnum") or "")
remote = {"REMOTE": True, "HYBRID": False, "ON_SITE": False}.get(arrangement)
row: Row = {
"source_id": post_id,
"title": title,
"places": [{"text": name, "remote": remote} for name in dict.fromkeys(places)],
"location": "; ".join(dict.fromkeys(places)),
"url": f"https://{_HOST}/{slug}/job-details/{post_id}/overview",
"posted": parse_dt(published.get("createdAt") or p.get("createdAt")),
@@ -144,5 +173,8 @@ def parse(body: bytes, slug: str) -> list[Row]:
"salary_text": (pay_line(num(pay.get("minAmount")), num(pay.get("maxAmount")),
pay.get("currencyIsoCode"), "")
if p.get("isCompensationVisible") else ""),
})
}
if employment := _employment(job):
row["employment"] = employment
out.append(row)
return out
+27 -17
View File
@@ -38,7 +38,15 @@ from typing import Any
from desk.sources import strip_html
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import Get, Page, is_web_url, only, parse_dt
READS = Reads(
scope="board", listing="paged", page_size=10, text="detail", complete="stated",
places="many",
fields=Fields(posted="listed", remote="stated"),
robots="read: robots.txt allows /careers (checked 2026-09-29)",
)
NAME = "eightfold"
LABEL = "Eightfold"
@@ -85,11 +93,7 @@ def careers_url(slug: str) -> str:
return f"https://{parts(slug)[0]}/careers"
def robots_page(slug: str) -> str:
return careers_url(slug)
def page(get: Get, slug: str, cursor: Any) -> Page:
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
"""One page of `/api/pcsx/search` -> `{"status": 200, "data": {"positions":
[...], "count": n}}`; the next starts at the offset after this page's
positions, and an empty page ends the read."""
@@ -108,25 +112,27 @@ def page(get: Get, slug: str, cursor: Any) -> Page:
start + len(positions) if positions else None)
def _place(locations: Any) -> str:
"""The role's places, "; " apart. Eightfold writes each as "city,region,
country" with no space and repeats the name when a city is its own region
("Mexico City,Mexico City,Mexico"), so the repeats go."""
out: list[str] = []
def _places(locations: Any) -> list[Place]:
"""The role's places. Eightfold writes each as "city,region,country" with
no space and repeats the name when a city is its own region ("Mexico
City,Mexico City,Mexico"), so the repeats go."""
out: list[Place] = []
for loc in locations if isinstance(locations, list) else []:
names: list[str] = []
for name in str(loc).split(","):
name = name.strip()
if name and name not in names:
names.append(name)
if names and ", ".join(names) not in out:
out.append(", ".join(names))
return "; ".join(out)
text = ", ".join(names)
if text and text not in (p["text"] for p in out):
out.append({"text": text, "remote": True if "remote" in text.lower() else None})
return out
def rows(positions: list[dict[str, Any]], slug: str) -> list[Row]:
"""A position has `id` (a number, kept as a string), `displayJobId`,
`name`, `locations`, `postedTs` (epoch seconds), `department`,
`name`, `locations`, `creationTs` (epoch seconds; `postedTs` is
refreshed in bulk), `department`,
`workLocationOption` and `positionUrl`, the path of the role's page on the
tenant's host. The list has no text, and no pay is stated anywhere."""
host = parts(slug)[0]
@@ -138,12 +144,16 @@ def rows(positions: list[dict[str, Any]], slug: str) -> list[Row]:
url = f"https://{host}{path}" if path.startswith("/") and path[1:2] != "/" else ""
if not title or not source_id or not is_web_url(url):
continue
places = _places(p.get("locations") or p.get("standardizedLocations"))
out.append({
"source_id": source_id,
"title": title,
"location": _place(p.get("locations") or p.get("standardizedLocations")),
"places": places,
"location": "; ".join(pl["text"] for pl in places),
"url": url,
"posted": parse_dt(p.get("postedTs")),
# `postedTs` is refreshed in bulk (a whole page says today);
# `creationTs` is the posting date.
"posted": parse_dt(p.get("creationTs") or p.get("postedTs")),
"department": str(p.get("department") or "").strip(),
"description": "",
"remote": _REMOTE.get(str(p.get("workLocationOption") or "").strip().lower()),
+33 -13
View File
@@ -19,13 +19,24 @@ from __future__ import annotations
import html
import json
import logging
import re
from collections.abc import Callable
from typing import Any
from desk.sources import strip_html
from ._shared import Get, Page, Row, only, parse_dt
from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import Get, Page, only, parse_dt
log = logging.getLogger(__name__)
READS = Reads(
scope="board", listing="paged", text="detail", complete="end",
places="many", empty="suspect",
fields=Fields(posted="listed", remote="stated", employment="stated"),
robots="read: robots.txt allows /<board> (checked 2026-09-29)",
)
NAME = "gem"
LABEL = "Gem"
@@ -57,6 +68,11 @@ DETAIL_QUERY = (
#: `job.locationType` -> `store._remote`'s words.
_REMOTE = {"IN_OFFICE": "onsite", "HYBRID": "hybrid", "REMOTE": "remote"}
#: `job.employmentType` -> the app's employment words.
_EMPLOYMENT = {"FULL_TIME": "full_time", "PART_TIME": "part_time", "CONTRACT": "contract",
"CONTRACTOR": "contract", "INTERN": "intern", "INTERNSHIP": "intern",
"TEMPORARY": "temporary", "FREELANCE": "freelance"}
def slug_ok(slug: str) -> bool:
return bool(_BOARD.fullmatch(slug))
@@ -70,10 +86,6 @@ def careers_url(slug: str) -> str:
return f"https://{_HOST}/{slug}"
def robots_page(slug: str) -> str:
return careers_url(slug)
def _ask(get: Get, operation: str, query: str, variables: dict[str, str],
path: str) -> Any:
"""One GraphQL call, and the part of its answer at `data.<path>`. An
@@ -93,7 +105,7 @@ def _ask(get: Get, operation: str, query: str, variables: dict[str, str],
return data[path]
def page(get: Get, slug: str, cursor: Any) -> Page:
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
"""The one page there is: `data.oatsExternalJobPostings.jobPostings`."""
board = _ask(get, "JobBoardList", LIST_QUERY, {"boardId": slug}, "oatsExternalJobPostings")
postings = board.get("jobPostings") if isinstance(board, dict) else None
@@ -116,26 +128,34 @@ def rows(postings: list[dict[str, Any]], slug: str) -> list[Row]:
if not title or not ext_id:
continue
locations = p.get("locations") or []
places: list[str] = []
places: list[Place] = []
for loc in locations:
place = str(loc.get("city") or loc.get("name") or "").strip()
if place and place not in places:
places.append(place)
text = str(loc.get("city") or loc.get("name") or "").strip()
if text and text not in (pl["text"] for pl in places):
places.append({"text": text, "remote": loc["isRemote"]
if isinstance(loc.get("isRemote"), bool) else None})
job = p.get("job") or {}
remote = _REMOTE.get(str(job.get("locationType") or "").upper())
if remote is None and locations and all(loc.get("isRemote") for loc in locations):
remote = "remote"
out.append({
row: Row = {
"source_id": ext_id,
"title": title,
"location": "; ".join(places),
"places": places,
"location": "; ".join(pl["text"] for pl in places),
"url": f"{careers_url(slug)}/{ext_id}",
"posted": None, # only the role's own query has a date
"department": str((job.get("department") or {}).get("name") or "").strip(),
"description": "",
"remote": remote,
"salary_text": "",
})
}
if word := str(job.get("employmentType") or "").strip().upper():
if employment := _EMPLOYMENT.get(word):
row["employment"] = employment
else:
log.debug("gem: unknown employmentType %r", word)
out.append(row)
return out
+45 -3
View File
@@ -8,12 +8,14 @@ from __future__ import annotations
import html
import json
import re
from collections.abc import Callable
from typing import Any
from desk.sources import strip_html
from ._shared import Row, is_web_url, num, only, parse_dt, path_slug_ok, pay_line
from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import is_web_url, num, only, parse_dt, path_slug_ok, pay_line
NAME = "greenhouse"
LABEL = "Greenhouse"
@@ -23,8 +25,22 @@ PATTERNS = (
r"greenhouse\.io/embed/job_board/js\?for=([A-Za-z0-9_-]+)",
)
READS = Reads(
scope="board", listing="whole", text="listing", complete="whole",
places="many",
fields=Fields(pay="figures", posted="trusted"),
robots="api: no robots.txt check needed (public board API), 2026-09-30",
)
_API = "boards-api.greenhouse.io"
#: What a board writes as `location.name` when it means a way of working, not a
#: place (Cloudflare: "Hybrid", "In-Office", "Distributed; Hybrid"). The real
#: places are then in `offices`.
_PLACEHOLDER = re.compile(
r"(?:(?:hybrid|in[- ]?office|on[- ]?site|remote|distributed|flexible|office)"
r"(?:\s+or\s+|\s*[;/,&]\s*|\s+)?)+", re.I)
def slug_ok(slug: str) -> bool:
return path_slug_ok(slug)
@@ -38,7 +54,7 @@ def careers_url(slug: str) -> str:
return f"https://job-boards.greenhouse.io/{slug}"
def feed_url(slug: str) -> str:
def feed_url(slug: str, ask: Ask) -> str:
return f"https://{_API}/v1/boards/{slug}/jobs?content=true&pay_transparency=true"
@@ -107,6 +123,30 @@ def _salary_text(ranges: list[dict[str, Any]]) -> str:
return "; ".join(lines)
def _places(j: dict[str, Any]) -> list[Place]:
"""The role's places: each of `offices` (present with `content=true`), in
the host's words, or, when there is none, `location.name` as stated. A
`location.name` that only names a way of working ("Hybrid") is not a
place: the offices replace it."""
name = ((j.get("location") or {}).get("name") or "").strip()
out: list[Place] = []
for office in j.get("offices") or []:
text = str((office or {}).get("location") or (office or {}).get("name") or "").strip()
if text and text not in (p["text"] for p in out):
out.append({"text": text, "remote": True if re.search(r"\bremote\b", text, re.I)
else None})
if not out or not name or _PLACEHOLDER.fullmatch(name):
return out or ([{"text": name, "remote": True if re.search(r"\bremote\b", name, re.I)
else None}] if name else [])
# A name that says something the offices do not (a "Remote India" beside
# an office in Bangalore) is one more place; one they already contain,
# or that contains them, is the same place said twice.
low = name.lower()
if any(p["text"].lower() in low or low in p["text"].lower() for p in out):
return out
return [{"text": name, "remote": True if re.search(r"\bremote\b", name, re.I) else None}, *out]
def parse(body: bytes, slug: str) -> list[Row]:
data = json.loads(body)
rows = []
@@ -122,10 +162,12 @@ def parse(body: bytes, slug: str) -> list[Row]:
content = html.unescape(j.get("content") or "")
ranges = j.get("pay_input_ranges") or []
salary_min, salary_max, currency = _salary(ranges)
places = _places(j)
rows.append({
"source_id": str(j.get("id") or ""),
"title": title,
"location": ((j.get("location") or {}).get("name") or "").strip(),
"places": places,
"location": "; ".join(p["text"] for p in places),
"url": url,
"posted": parse_dt(j.get("first_published") or j.get("updated_at")),
"department": (departments[0].get("name") if departments else "") or "",
+37 -8
View File
@@ -16,12 +16,35 @@ from __future__ import annotations
import html
import json
import logging
from collections.abc import Callable
from typing import Any
from desk.sources import strip_html
from ._shared import Get, Page, Row, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
from ._contract import Ask, Fields, Reads, Row
from ._shared import Get, Page, label_slug_ok, num, only, parse_dt, pay_line, yearly_amount
log = logging.getLogger(__name__)
# One ad names one site: a company that hires at two sites posts two ads with
# their own ids (Dixa's "Expression of interest" at Copenhagen and London), so
# the host publishes one posting per place.
READS = Reads(
scope="board", listing="paged", text="listing", complete="end",
places="copies",
fields=Fields(pay="figures", posted="listed", remote="stated", employment="stated"),
robots="read: robots.txt allows the careers site; its robots.txt is the single-page "
"app's index page with no rules (checked 2026-09-29)",
)
#: `employmentType`, the company's own name for it -> the app's employment
#: words. "Permanent" says nothing about hours and stays unmapped.
_EMPLOYMENT = {"regular full time": "full_time", "full time": "full_time",
"full-time": "full_time", "regular part time": "part_time",
"part time": "part_time", "part-time": "part_time", "contract": "contract",
"contractor": "contract", "temporary": "temporary", "fixed term": "temporary",
"intern": "intern", "internship": "intern", "freelance": "freelance"}
NAME = "hibob"
LABEL = "HiBob"
@@ -55,11 +78,7 @@ def careers_url(slug: str) -> str:
return f"https://{_host(slug)}/"
def robots_page(slug: str) -> str:
return careers_url(slug)
def page(get: Get, slug: str, cursor: Any) -> Page:
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
"""The one page: every ad, and no total or cursor, so the board is
complete when the answer parses. The host's robots.txt is its single-page
app's index page (a 200 with no rules), which allows."""
@@ -96,9 +115,12 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
period = _PERIODS.get(str(ad.get("payTransparencySalaryPayPeriod") or "").strip().lower(), "")
salary_min, salary_max, code = yearly_amount(lo, hi, currency, period,
{"year": 1, "month": 12})
out.append({
remote = {"remote": True, "hybrid": False, "on_site": False, "onsite": False}.get(
str(ad.get("workspaceTypeId") or "").strip().lower())
row: Row = {
"source_id": ad_id,
"title": title,
"places": [{"text": location, "remote": remote}] if location else [],
"location": location,
"url": f"https://{_host(slug)}/jobs/{ad_id}",
"posted": parse_dt(ad.get("publishedAt")),
@@ -109,5 +131,12 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
"salary_min": salary_min,
"salary_max": salary_max,
"currency": code,
})
}
kind = str(ad.get("employmentType") or "").strip()
if kind:
if employment := _EMPLOYMENT.get(kind.lower()):
row["employment"] = employment
else:
log.debug("hibob: unknown employmentType %r", kind)
out.append(row)
return out
+18 -12
View File
@@ -10,12 +10,13 @@ without `take` it answers the first 25 roles whatever the board holds, and
`flyingtiger` does).
robots.txt: the portal's own pages on candidate.hr-manager.net disallow
everything for `*` except the application form, and so `robots_page` is the
feed's URL, on the API host, which has no robots.txt (a 404, which allows).
The feed is the interface HR Manager publishes for readers like this one; the
portal's HTML list, which the robots.txt does shut out, is never read. The
role's link is the application form the robots.txt names as allowed, and is
only shown, not fetched.
everything for `*` except the application form. The feed is on the API host,
which has no robots.txt (a 404, which allows); it is the interface HR Manager
publishes for readers like this one, and the portal's HTML list, which the
robots.txt does shut out, is never read. The role's link is the application
form the robots.txt names as allowed, and is only shown, not fetched. The
declaration says "disallowed" all the same, and the owner decides whether the
feed is read.
"""
from __future__ import annotations
@@ -28,7 +29,15 @@ from typing import Any
from desk.sources import strip_html
from ._shared import Get, Page, Row, is_web_url, only, parse_dt
from ._contract import Ask, Fields, Reads, Row
from ._shared import Get, Page, is_web_url, only, parse_dt
READS = Reads(
scope="board", listing="paged", page_size=100, text="listing", complete="end",
places="one",
fields=Fields(posted="listed"),
robots="disallowed by robots.txt; owner to decide",
)
NAME = "hrmanager"
LABEL = "HR Manager"
@@ -64,11 +73,7 @@ def _feed(slug: str, skip: int) -> str:
f"?incads=true&skip={skip}&take={PAGE_SIZE}")
def robots_page(slug: str) -> str:
return _feed(slug, 0)
def page(get: Get, slug: str, cursor: Any) -> Page:
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
skip = int(cursor or 0)
items = _items(get(_feed(slug, skip), headers={"Accept": "application/rss+xml"}))
results = [{"id": item.findtext("guid") or item.findtext("link") or ""} for item in items]
@@ -103,6 +108,7 @@ def rows(items: list[ET.Element], slug: str) -> list[Row]:
out.append({
"source_id": (item.findtext("guid") or url).strip(),
"title": title,
"places": [],
"location": "",
"url": url,
"posted": parse_dt(item.findtext("pubDate")),
+50 -10
View File
@@ -30,13 +30,29 @@ from __future__ import annotations
import html
import json
import logging
import re
from collections.abc import Callable
from typing import Any
from desk.sources import strip_html
from ._shared import Get, Page, Row, only, parse_dt
from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import Get, Page, only, parse_dt
log = logging.getLogger(__name__)
READS = Reads(
scope="board", listing="paged", page_size=100, text="listing", complete="stated",
places="many",
fields=Fields(posted="listed", country="stated", employment="stated"),
robots="read: robots.txt allows /jobs, with crawl-delay 5 not honoured (checked 2026-09-29)",
)
#: `employment_type` -> the app's employment words.
_EMPLOYMENT = {"FULL_TIME": "full_time", "PART_TIME": "part_time", "CONTRACTOR": "contract",
"CONTRACT": "contract", "TEMPORARY": "temporary", "INTERN": "intern",
"INTERNSHIP": "intern", "FREELANCE": "freelance"}
NAME = "jibe"
LABEL = "Jibe"
@@ -73,11 +89,7 @@ def careers_url(slug: str) -> str:
return f"https://{slug.lower()}/jobs"
def robots_page(slug: str) -> str:
return careers_url(slug)
def page(get: Get, slug: str, cursor: Any) -> Page:
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
"""One page of `/api/jobs`; the next is the following number, and an empty
page ends the read."""
number = int(cursor or 1)
@@ -99,6 +111,21 @@ def _plain(markup: Any) -> str:
return " ".join(html.unescape(strip_html(str(markup or ""))).split())
def _places(j: dict[str, Any]) -> list[Place]:
"""`full_location` names every place of the role, "; " apart, each as
"City, Country"; the site's `city` and `country` stand in when it is
missing."""
line = str(j.get("full_location") or "").strip() \
or ", ".join(p for p in (j.get("city"), j.get("country")) if p)
out: list[Place] = []
for part in line.split(";"):
text = part.strip()
if text and text not in (p["text"] for p in out):
out.append({"text": text, "remote": True if re.search(r"\bremote\b", text, re.I)
else None})
return out
def rows(data: dict[str, Any], slug: str) -> list[Row]:
"""One page: each `jobs[].data` has `slug` (equal to `req_id`), `title`,
the HTML `description` and, on some, `qualifications`, `full_location`
@@ -117,16 +144,29 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
categories = j.get("categories") or []
text = "\n\n".join(p for p in (_plain(j.get("description")),
_plain(j.get("qualifications"))) if p)
out.append({
places = _places(j)
row: Row = {
"source_id": role,
"title": title,
"location": (j.get("full_location")
or ", ".join(p for p in (j.get("city"), j.get("country")) if p)).strip(),
"places": places,
"location": "; ".join(p["text"] for p in places),
"url": f"https://{slug.lower()}/jobs/{role}",
"posted": parse_dt(j.get("posted_date")),
"department": str((categories[0] if categories else {}).get("name") or "").strip(),
"description": text,
"remote": None,
"salary_text": "",
})
}
# One code, for a role at one place: over several countries it would
# gate the others by the site's own.
code = str(j.get("country_code") or "").strip().upper()
if len(places) == 1 and re.fullmatch(r"[A-Z]{2}", code):
row["country"] = code
kind = str(j.get("employment_type") or "").strip()
if kind:
if employment := _EMPLOYMENT.get(kind.upper()):
row["employment"] = employment
else:
log.debug("jibe: unknown employment_type %r", kind)
out.append(row)
return out
+64 -13
View File
@@ -18,13 +18,24 @@ from __future__ import annotations
import html
import json
import logging
import re
from collections.abc import Callable
from typing import Any
from desk.sources import strip_html
from ._shared import Get, Row, parse_dt
from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import Get, parse_dt
log = logging.getLogger(__name__)
READS = Reads(
scope="board", listing="whole", text="detail", teaser=True, complete="whole",
places="many",
fields=Fields(posted="trusted", remote="stated", seniority="stated", employment="stated"),
robots="read: robots.txt allows /jobs/companies/ (checked 2026-09-29)",
)
NAME = "jobylon"
LABEL = "Jobylon"
@@ -47,6 +58,21 @@ _SLUG = re.compile(r"[0-9]{1,12}")
#: from.
_ROLE_PATH = re.compile(r"/jobs/[0-9]+(?:-[A-Za-z0-9_-]*)?/?")
#: The embed's `experience` -> `app.models.SENIORITIES`. Only the coarse levels
#: are mapped; "Experienced" sits between "Mid level" and the managerial
#: levels and says too little to place, so it stays unmapped and is logged.
_SENIORITY = {"student": "intern", "internship": "intern", "trainee": "intern",
"entry level": "junior", "entry-level": "junior", "junior": "junior",
"graduate": "junior", "mid level": "mid", "mid-level": "mid", "mid": "mid",
"senior": "senior", "lead": "lead", "manager": "manager", "director": "director",
"executive": "exec", "not applicable": ""}
#: The embed's `employment_type` -> the app's employment words.
_EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "part-time": "part_time",
"part time": "part_time", "contract": "contract", "consultant": "contract",
"temporary": "temporary", "internship": "intern", "intern": "intern",
"freelance": "freelance", "summer job": "temporary", "seasonal": "temporary"}
_ASSIGNMENT = "JBL.embed_v2['jobs'] = "
_KEY = re.compile(r"[A-Za-z_][A-Za-z0-9_]*")
_TRAILING_COMMA = re.compile(r",(\s*[\]}])")
@@ -74,17 +100,13 @@ def careers_url(slug: str) -> str:
return f"https://{_EMP}/companies/{slug}/"
def feed_url(slug: str) -> str:
def feed_url(slug: str, ask: Ask) -> str:
"""Both hosts serve the same robots.txt (compared on 2026-09-29), so the
roles' pages on emp.jobylon.com are covered by what was checked for this
one."""
return f"https://{_CDN}/jobs/companies/{slug}/embed/v2/"
def robots_page(slug: str) -> str:
"""The page the board is read from. Both hosts serve the same robots.txt
(compared on 2026-09-29), so the roles' pages on emp.jobylon.com are
covered by the check on this one."""
return feed_url(slug)
def _json(text: str) -> Any:
"""A JavaScript array or object literal as JSON: keys quoted, strings
single-quoted with `\\uXXXX` escapes, trailing commas."""
@@ -157,6 +179,22 @@ def _published(value: Any) -> Any:
return parse_dt(f"{match.group(3)}-{month:02d}-{int(match.group(2)):02d}")
def _mapped(table: dict[str, str], value: Any, what: str) -> str | None:
"""`table`'s word for the host's `value`; None when the host states none
or the table does not know it (logged, so the table can grow)."""
word = str(value or "").strip()
if not word:
return None
found = table.get(word.lower())
if found is None:
log.debug("jobylon: unknown %s %r", what, word)
return found or None
def _place(text: str) -> Place:
return {"text": text, "remote": True if re.search(r"\bremote\b", text, re.I) else None}
def parse(body: bytes, slug: str) -> list[Row]:
"""The embed page -> one row per role. `posted` is the day `published_date`
names, which `detail` replaces with the exact time. `location` joins the
@@ -172,18 +210,30 @@ def parse(body: bytes, slug: str) -> list[Row]:
if not title or not job_id or not _ROLE_PATH.fullmatch(path):
continue
function = str(j.get("function") or "").strip()
out.append({
places = [_place(name) for name in dict.fromkeys(
str(name).strip() for name in j.get("locations") or [] if str(name).strip())]
row: Row = {
"source_id": job_id,
"title": title,
"location": "; ".join(dict.fromkeys(
str(name).strip() for name in j.get("locations") or [] if str(name).strip())),
"places": places,
"location": "; ".join(pl["text"] for pl in places),
"url": f"https://{_EMP}{path}",
"posted": _published(j.get("published_date")),
"department": "" if function == "All functions" else function,
"description": "",
"remote": _REMOTE.get(str(j.get("workspace") or "").strip().lower()),
"salary_text": "", # the embed states no pay
})
}
# `summary` is the text the company gave the listing, the string
# "None" when it gave none.
summary = " ".join(html.unescape(strip_html(str(j.get("summary") or ""))).split())
if summary and summary != "None":
row["teaser"] = summary
if seniority := _mapped(_SENIORITY, j.get("experience"), "experience"):
row["seniority"] = seniority
if employment := _mapped(_EMPLOYMENT, j.get("employment_type"), "employment_type"):
row["employment"] = employment
out.append(row)
return out
@@ -213,4 +263,5 @@ def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
}
if places:
found["location"] = "; ".join(places)
found["places"] = [_place(name) for name in places]
return found
+52 -5
View File
@@ -16,12 +16,32 @@ The field map is ported from `openings.go` in
from __future__ import annotations
import json
import logging
import re
from collections.abc import Callable
from typing import Any
from desk.sources import strip_html
from ._shared import Row, is_web_url, num, only, parse_dt, path_slug_ok, pay_line, yearly_amount
from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import is_web_url, num, only, parse_dt, path_slug_ok, pay_line, yearly_amount
log = logging.getLogger(__name__)
READS = Reads(
scope="board", listing="whole", text="listing", complete="whole",
places="many",
fields=Fields(pay="figures", posted="trusted", remote="stated", country="stated",
employment="stated"),
robots="api: no robots.txt check needed (public board API), 2026-09-30",
)
#: `categories.commitment` -> the app's employment words. A commitment is free
#: text on Lever ("Permanent" is seen too and says nothing about hours).
_EMPLOYMENT = {"full-time": "full_time", "full time": "full_time", "fulltime": "full_time",
"part-time": "part_time", "part time": "part_time", "parttime": "part_time",
"intern": "intern", "internship": "intern", "contract": "contract",
"contractor": "contract", "temporary": "temporary", "freelance": "freelance"}
NAME = "lever"
LABEL = "Lever"
@@ -60,7 +80,7 @@ def careers_url(slug: str) -> str:
return f"https://jobs.{infix}lever.co/{site}"
def feed_url(slug: str) -> str:
def feed_url(slug: str, ask: Ask) -> str:
infix, site = _split(slug)
return f"https://api.{infix}lever.co/v0/postings/{site}?mode=json"
@@ -88,6 +108,19 @@ def _salary_text(salary_range: dict[str, Any]) -> str:
salary_range.get("currency"), period)
def _places(categories: dict[str, Any]) -> list[Place]:
"""`categories.allLocations` (every place of the role, the first being
`location`), or `location` alone when the list is missing."""
names = [categories.get("location"), *(categories.get("allLocations") or [])]
out: list[Place] = []
for name in names:
text = str(name or "").strip()
if text and text not in (p["text"] for p in out):
out.append({"text": text, "remote": True if re.search(r"\bremote\b", text, re.I)
else None})
return out
def parse(body: bytes, slug: str) -> list[Row]:
data = json.loads(body)
if not isinstance(data, list):
@@ -104,10 +137,12 @@ def parse(body: bytes, slug: str) -> list[Row]:
salary_min, salary_max, currency = yearly_amount(
salary_range.get("min"), salary_range.get("max"), salary_range.get("currency"),
salary_range.get("interval"), {"per-year-salary": 1, "per-month-salary": 12})
rows.append({
places = _places(categories)
row: Row = {
"source_id": str(j.get("id") or ""),
"title": title,
"location": (categories.get("location") or "").strip(),
"places": places,
"location": "; ".join(p["text"] for p in places),
"url": url,
"posted": parse_dt(j.get("createdAt")),
"department": (categories.get("department") or categories.get("team") or "").strip(),
@@ -117,5 +152,17 @@ def parse(body: bytes, slug: str) -> list[Row]:
"salary_min": salary_min,
"salary_max": salary_max,
"currency": currency,
})
}
# `country` is one code, for a role at one place; over several it would
# gate the others by the first's country.
country = str(j.get("country") or "").strip().upper()
if len(places) == 1 and re.fullmatch(r"[A-Z]{2}", country):
row["country"] = country
commitment = str(categories.get("commitment") or "").strip()
if commitment:
if employment := _EMPLOYMENT.get(commitment.lower()):
row["employment"] = employment
else:
log.debug("lever: unknown commitment %r", commitment)
rows.append(row)
return rows