The readers flattened a posting's HTML into one line with strip_html, so sections and bullets ran together. _shared.html_text gives a line per paragraph, '• ' before a list item and '## ' before a heading; every reader's description goes through it, sections read per role get their own headings, and the reader pane draws the lines as h3, ul and p. Excerpts leave the marks out. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
159 lines
6.8 KiB
Python
159 lines
6.8 KiB
Python
"""Gem: `gem:<board>` for jobs.gem.com/<board>, one GraphQL POST for the
|
|
whole board and one more for a role's text and date. The board is the path
|
|
after `jobs.gem.com/`, and it is case-sensitive (`Productboard` is an empty
|
|
board where `productboard` is not).
|
|
|
|
The two queries below were read out of Gem's page scripts
|
|
(`static.gem.com/scripts/jobBoards.*.js` and `ExternalJobBoardList.*.js`) on
|
|
2026-09-29 and run against Productboard's and sennder's boards; the schema is
|
|
Gem's own and not documented, so a new bundle can rename a field. Each query is
|
|
one constant, and an answer that lacks the shape read here fails with an error
|
|
that says which part is missing rather than reading as an empty board. The
|
|
endpoint answered with no Origin and no Referer header, so none is sent. An
|
|
unknown board answers 200 with an empty list, like a board with no role open.
|
|
|
|
The listing has no description and no date; both are on the role's page.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import re
|
|
from collections.abc import Callable
|
|
from typing import Any
|
|
|
|
|
|
from ._contract import Ask, Fields, Place, Reads, Row
|
|
from ._shared import Get, Page, add_place, html_text, only, parse_dt
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
READS = Reads(
|
|
scope="board", listing="paged", text="detail", complete="end",
|
|
places="many", empty="suspect",
|
|
fields=Fields(posted="listed", remote="stated", employment="stated"),
|
|
robots=("read: POST /api/public/graphql on jobs.gem.com, whose /robots.txt answers "
|
|
"404, so nothing is disallowed (checked 2026-09-30)"),
|
|
)
|
|
|
|
NAME = "gem"
|
|
LABEL = "Gem"
|
|
KEEP_CASE = True
|
|
PATTERNS = (r"jobs\.gem\.com/([A-Za-z0-9_-]+)",)
|
|
|
|
_HOST = "jobs.gem.com"
|
|
_API = f"https://{_HOST}/api/public/graphql"
|
|
|
|
#: A board is one path segment of letters, digits, `-` and `_`. It travels as a
|
|
#: GraphQL variable and never into a host name, so the host stays Gem's.
|
|
_BOARD = re.compile(r"[A-Za-z0-9_-]+")
|
|
|
|
#: Every role of a board in one answer, with no paging and no total.
|
|
LIST_QUERY = (
|
|
"query JobBoardList($boardId: String!) { oatsExternalJobPostings(boardId: $boardId) "
|
|
"{ jobPostings { id extId title locations { id name city isoCountry isRemote extId } "
|
|
"job { id department { id name extId } locationType employmentType } } } }")
|
|
|
|
#: One role's description and the date it was first published.
|
|
DETAIL_QUERY = (
|
|
"query ExternalJobPosting($boardId: String!, $extId: String!) "
|
|
"{ oatsExternalJobPosting(boardId: $boardId, extId: $extId) "
|
|
"{ id title descriptionHtml extId firstPublishedTsSec "
|
|
"locations { id extId name city isoCountry isRemote } "
|
|
"job { id locationType employmentType requisitionId teamDisplayName "
|
|
"department { id extId name } } compensationHtml } }")
|
|
|
|
|
|
def slug_ok(slug: str) -> bool:
|
|
return bool(_BOARD.fullmatch(slug))
|
|
|
|
|
|
def allowed(slug: str) -> Callable[[str], bool]:
|
|
return only(_HOST)
|
|
|
|
|
|
def careers_url(slug: str) -> str:
|
|
return f"https://{_HOST}/{slug}"
|
|
|
|
|
|
def _ask(get: Get, operation: str, query: str, variables: dict[str, str],
|
|
path: str) -> Any:
|
|
"""One GraphQL call, and the part of its answer at `data.<path>`. An
|
|
`errors` list or a `data` without the key is Gem's schema having moved (or
|
|
a refused query), and is raised as such; a `null` at the key is returned."""
|
|
answer = json.loads(get(_API, method="POST", payload={
|
|
"operationName": operation, "variables": variables, "query": query}))
|
|
if not isinstance(answer, dict):
|
|
raise ValueError(f"not a Gem answer: {type(answer).__name__}")
|
|
if answer.get("errors"):
|
|
first = answer["errors"][0]
|
|
message = first.get("message") if isinstance(first, dict) else first
|
|
raise ValueError(f"Gem's {operation} query was refused: {message}")
|
|
data = answer.get("data")
|
|
if not isinstance(data, dict) or path not in data:
|
|
raise ValueError(f"Gem's {operation} answer has no {path}; the query may be out of date")
|
|
return data[path]
|
|
|
|
|
|
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
|
|
"""The one page there is: `data.oatsExternalJobPostings.jobPostings`."""
|
|
board = _ask(get, "JobBoardList", LIST_QUERY, {"boardId": slug}, "oatsExternalJobPostings")
|
|
postings = board.get("jobPostings") if isinstance(board, dict) else None
|
|
if not isinstance(postings, list):
|
|
raise ValueError("Gem's JobBoardList answer has no jobPostings list; "
|
|
"the query may be out of date")
|
|
return postings, [{"id": p.get("extId") or p.get("id") or ""} for p in postings], None, None
|
|
|
|
|
|
def rows(postings: list[dict[str, Any]], slug: str) -> list[Row]:
|
|
"""A role has `extId` (a number from a synced Greenhouse, else an opaque
|
|
string; the role's page is `jobs.gem.com/<board>/<extId>`), `title`,
|
|
`locations` (`name`, `city`, `isoCountry` as ISO alpha-3, `isRemote`) and
|
|
`job` (`department`, `locationType`, `employmentType`). No pay is stated
|
|
in the list."""
|
|
out = []
|
|
for p in postings:
|
|
title = (p.get("title") or "").strip()
|
|
ext_id = str(p.get("extId") or "")
|
|
if not title or not ext_id:
|
|
continue
|
|
locations = p.get("locations") or []
|
|
places: list[Place] = []
|
|
for loc in locations:
|
|
text = str(loc.get("city") or loc.get("name") or "").strip()
|
|
add_place(places, {"text": text, "remote": loc["isRemote"]
|
|
if isinstance(loc.get("isRemote"), bool) else None})
|
|
job = p.get("job") or {}
|
|
remote: bool | str | None = str(job.get("locationType") or "").strip() or None
|
|
if remote is None and locations and all(loc.get("isRemote") for loc in locations):
|
|
remote = True
|
|
row: Row = {
|
|
"source_id": ext_id,
|
|
"title": title,
|
|
"places": places,
|
|
"location": "; ".join(pl["text"] for pl in places),
|
|
"url": f"{careers_url(slug)}/{ext_id}",
|
|
"posted": None, # only the role's own query has a date
|
|
"department": str((job.get("department") or {}).get("name") or "").strip(),
|
|
"description": "",
|
|
"remote": remote,
|
|
"salary_text": "",
|
|
}
|
|
if employment := str(job.get("employmentType") or "").strip():
|
|
row["employment"] = employment
|
|
out.append(row)
|
|
return out
|
|
|
|
|
|
def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
|
"""`oatsExternalJobPosting` of the role: `descriptionHtml` and
|
|
`firstPublishedTsSec` (epoch seconds). `compensationHtml` was null on the
|
|
roles read and is left alone."""
|
|
role = _ask(get, "ExternalJobPosting", DETAIL_QUERY,
|
|
{"boardId": slug, "extId": row["source_id"]}, "oatsExternalJobPosting")
|
|
if not isinstance(role, dict):
|
|
raise ValueError(f"Gem has no role {row['source_id']} on {slug}")
|
|
return {"description": html_text(role.get("descriptionHtml")),
|
|
"posted": parse_dt(role.get("firstPublishedTsSec"))}
|