Files
jobboard/plugins/readers/gem.py
T

157 lines
6.8 KiB
Python

"""Gem: `gem:<board>` for jobs.gem.com/<board>, one GraphQL POST for the
whole board and one more for a role's text and date. The board is the path
after `jobs.gem.com/`, and it is case-sensitive (`Productboard` is an empty
board where `productboard` is not).
The two queries below were read out of Gem's page scripts
(`static.gem.com/scripts/jobBoards.*.js` and `ExternalJobBoardList.*.js`) on
2026-09-29 and run against Productboard's and sennder's boards; the schema is
Gem's own and not documented, so a new bundle can rename a field. Each query is
one constant, and an answer that lacks the shape read here fails with an error
that says which part is missing rather than reading as an empty board. The
endpoint answered with no Origin and no Referer header, so none is sent. An
unknown board answers 200 with an empty list, like a board with no role open.
The listing has no description and no date; both are on the role's page.
"""
from __future__ import annotations
import json
import logging
import re
from collections.abc import Callable
from typing import Any
from ._contract import Ask, Fields, Place, Reads, Row
from ._shared import Get, Page, add_place, html_text, only, parse_dt
log = logging.getLogger(__name__)
READS = Reads(
scope="board", listing="paged", text="detail", complete="end",
places="many", empty="suspect",
fields=Fields(posted="listed", remote="stated", employment="stated"),
robots=("read: POST /api/public/graphql on jobs.gem.com, whose /robots.txt answers "
"404, so nothing is disallowed (checked 2026-09-30)"),
)
NAME = "gem"
LABEL = "Gem"
KEEP_CASE = True
PATTERNS = (r"jobs\.gem\.com/([A-Za-z0-9_-]+)",)
_HOST = "jobs.gem.com"
_API = f"https://{_HOST}/api/public/graphql"
#: A board is one path segment of letters, digits, `-` and `_`. It travels as a
#: GraphQL variable and never into a host name, so the host stays Gem's.
_BOARD = re.compile(r"[A-Za-z0-9_-]+")
#: Every role of a board in one answer, with no paging and no total.
LIST_QUERY = (
"query JobBoardList($boardId: String!) { oatsExternalJobPostings(boardId: $boardId) "
"{ jobPostings { id extId title locations { id name city isoCountry isRemote extId } "
"job { id department { id name extId } locationType employmentType } } } }")
#: One role's description and the date it was first published.
DETAIL_QUERY = (
"query ExternalJobPosting($boardId: String!, $extId: String!) "
"{ oatsExternalJobPosting(boardId: $boardId, extId: $extId) "
"{ id title descriptionHtml extId firstPublishedTsSec "
"locations { id extId name city isoCountry isRemote } "
"job { id locationType employmentType requisitionId teamDisplayName "
"department { id extId name } } compensationHtml } }")
def slug_ok(slug: str) -> bool:
return bool(_BOARD.fullmatch(slug))
def allowed(slug: str) -> Callable[[str], bool]:
return only(_HOST)
def careers_url(slug: str) -> str:
return f"https://{_HOST}/{slug}"
def _ask(get: Get, operation: str, query: str, variables: dict[str, str],
path: str) -> Any:
"""One GraphQL call, and the part of its answer at `data.<path>`. An
`errors` list or a `data` without the key is Gem's schema having moved (or
a refused query), and is raised as such; a `null` at the key is returned."""
answer = json.loads(get(_API, method="POST", payload={
"operationName": operation, "variables": variables, "query": query}))
if not isinstance(answer, dict):
raise ValueError(f"not a Gem answer: {type(answer).__name__}")
if answer.get("errors"):
first = answer["errors"][0]
message = first.get("message") if isinstance(first, dict) else first
raise ValueError(f"Gem's {operation} query was refused: {message}")
data = answer.get("data")
if not isinstance(data, dict) or path not in data:
raise ValueError(f"Gem's {operation} answer has no {path}; the query may be out of date")
return data[path]
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
"""The one page there is: `data.oatsExternalJobPostings.jobPostings`."""
board = _ask(get, "JobBoardList", LIST_QUERY, {"boardId": slug}, "oatsExternalJobPostings")
postings = board.get("jobPostings") if isinstance(board, dict) else None
if not isinstance(postings, list):
raise ValueError("Gem's JobBoardList answer has no jobPostings list; "
"the query may be out of date")
return postings, [{"id": p.get("extId") or p.get("id") or ""} for p in postings], None, None
def rows(postings: list[dict[str, Any]], slug: str) -> list[Row]:
"""A role has `extId` (a number from a synced Greenhouse, else an opaque
string; the role's page is `jobs.gem.com/<board>/<extId>`), `title`,
`locations` (`name`, `city`, `isoCountry` as ISO alpha-3, `isRemote`) and
`job` (`department`, `locationType`, `employmentType`). No pay is stated
in the list."""
out = []
for p in postings:
title = (p.get("title") or "").strip()
ext_id = str(p.get("extId") or "")
locations = p.get("locations") or []
places: list[Place] = []
for loc in locations:
text = str(loc.get("city") or loc.get("name") or "").strip()
add_place(places, {"text": text, "remote": loc["isRemote"]
if isinstance(loc.get("isRemote"), bool) else None})
job = p.get("job") or {}
remote: bool | str | None = str(job.get("locationType") or "").strip() or None
if remote is None and locations and all(loc.get("isRemote") for loc in locations):
remote = True
row: Row = {
"source_id": ext_id,
"title": title,
"places": places,
"location": "; ".join(pl["text"] for pl in places),
"url": f"{careers_url(slug)}/{ext_id}",
"posted": None, # only the role's own query has a date
"department": str((job.get("department") or {}).get("name") or "").strip(),
"description": "",
"remote": remote,
"salary_text": "",
}
if employment := str(job.get("employmentType") or "").strip():
row["employment"] = employment
out.append(row)
return out
def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
"""`oatsExternalJobPosting` of the role: `descriptionHtml` and
`firstPublishedTsSec` (epoch seconds). `compensationHtml` was null on the
roles read and is left alone."""
role = _ask(get, "ExternalJobPosting", DETAIL_QUERY,
{"boardId": slug, "extId": row["source_id"]}, "oatsExternalJobPosting")
if not isinstance(role, dict):
raise ValueError(f"Gem has no role {row['source_id']} on {slug}")
return {"description": html_text(role.get("descriptionHtml")),
"posted": parse_dt(role.get("firstPublishedTsSec"))}