"""Gem: `gem:` for jobs.gem.com/, one GraphQL POST for the whole board and one more for a role's text and date. The board is the path after `jobs.gem.com/`, and it is case-sensitive (`Productboard` is an empty board where `productboard` is not). The two queries below were read out of Gem's page scripts (`static.gem.com/scripts/jobBoards.*.js` and `ExternalJobBoardList.*.js`) on 2026-09-29 and run against Productboard's and sennder's boards; the schema is Gem's own and not documented, so a new bundle can rename a field. Each query is one constant, and an answer that lacks the shape read here fails with an error that says which part is missing rather than reading as an empty board. The endpoint answered with no Origin and no Referer header, so none is sent. An unknown board answers 200 with an empty list, like a board with no role open. The listing has no description and no date; both are on the role's page. """ from __future__ import annotations import html import json import logging import re from collections.abc import Callable from typing import Any from desk.sources import strip_html from ._contract import Ask, Fields, Place, Reads, Row from ._shared import Get, Page, add_place, only, parse_dt log = logging.getLogger(__name__) READS = Reads( scope="board", listing="paged", text="detail", complete="end", places="many", empty="suspect", fields=Fields(posted="listed", remote="stated", employment="stated"), robots=("read: POST /api/public/graphql on jobs.gem.com, whose /robots.txt answers " "404, so nothing is disallowed (checked 2026-09-30)"), ) NAME = "gem" LABEL = "Gem" KEEP_CASE = True PATTERNS = (r"jobs\.gem\.com/([A-Za-z0-9_-]+)",) _HOST = "jobs.gem.com" _API = f"https://{_HOST}/api/public/graphql" #: A board is one path segment of letters, digits, `-` and `_`. It travels as a #: GraphQL variable and never into a host name, so the host stays Gem's. _BOARD = re.compile(r"[A-Za-z0-9_-]+") #: Every role of a board in one answer, with no paging and no total. LIST_QUERY = ( "query JobBoardList($boardId: String!) { oatsExternalJobPostings(boardId: $boardId) " "{ jobPostings { id extId title locations { id name city isoCountry isRemote extId } " "job { id department { id name extId } locationType employmentType } } } }") #: One role's description and the date it was first published. DETAIL_QUERY = ( "query ExternalJobPosting($boardId: String!, $extId: String!) " "{ oatsExternalJobPosting(boardId: $boardId, extId: $extId) " "{ id title descriptionHtml extId firstPublishedTsSec " "locations { id extId name city isoCountry isRemote } " "job { id locationType employmentType requisitionId teamDisplayName " "department { id extId name } } compensationHtml } }") def slug_ok(slug: str) -> bool: return bool(_BOARD.fullmatch(slug)) def allowed(slug: str) -> Callable[[str], bool]: return only(_HOST) def careers_url(slug: str) -> str: return f"https://{_HOST}/{slug}" def _ask(get: Get, operation: str, query: str, variables: dict[str, str], path: str) -> Any: """One GraphQL call, and the part of its answer at `data.`. An `errors` list or a `data` without the key is Gem's schema having moved (or a refused query), and is raised as such; a `null` at the key is returned.""" answer = json.loads(get(_API, method="POST", payload={ "operationName": operation, "variables": variables, "query": query})) if not isinstance(answer, dict): raise ValueError(f"not a Gem answer: {type(answer).__name__}") if answer.get("errors"): first = answer["errors"][0] message = first.get("message") if isinstance(first, dict) else first raise ValueError(f"Gem's {operation} query was refused: {message}") data = answer.get("data") if not isinstance(data, dict) or path not in data: raise ValueError(f"Gem's {operation} answer has no {path}; the query may be out of date") return data[path] def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page: """The one page there is: `data.oatsExternalJobPostings.jobPostings`.""" board = _ask(get, "JobBoardList", LIST_QUERY, {"boardId": slug}, "oatsExternalJobPostings") postings = board.get("jobPostings") if isinstance(board, dict) else None if not isinstance(postings, list): raise ValueError("Gem's JobBoardList answer has no jobPostings list; " "the query may be out of date") return postings, [{"id": p.get("extId") or p.get("id") or ""} for p in postings], None, None def rows(postings: list[dict[str, Any]], slug: str) -> list[Row]: """A role has `extId` (a number from a synced Greenhouse, else an opaque string; the role's page is `jobs.gem.com//`), `title`, `locations` (`name`, `city`, `isoCountry` as ISO alpha-3, `isRemote`) and `job` (`department`, `locationType`, `employmentType`). No pay is stated in the list.""" out = [] for p in postings: title = (p.get("title") or "").strip() ext_id = str(p.get("extId") or "") if not title or not ext_id: continue locations = p.get("locations") or [] places: list[Place] = [] for loc in locations: text = str(loc.get("city") or loc.get("name") or "").strip() add_place(places, {"text": text, "remote": loc["isRemote"] if isinstance(loc.get("isRemote"), bool) else None}) job = p.get("job") or {} remote: bool | str | None = str(job.get("locationType") or "").strip() or None if remote is None and locations and all(loc.get("isRemote") for loc in locations): remote = True row: Row = { "source_id": ext_id, "title": title, "places": places, "location": "; ".join(pl["text"] for pl in places), "url": f"{careers_url(slug)}/{ext_id}", "posted": None, # only the role's own query has a date "department": str((job.get("department") or {}).get("name") or "").strip(), "description": "", "remote": remote, "salary_text": "", } if employment := str(job.get("employmentType") or "").strip(): row["employment"] = employment out.append(row) return out def detail(get: Get, slug: str, row: Row) -> dict[str, Any]: """`oatsExternalJobPosting` of the role: `descriptionHtml` and `firstPublishedTsSec` (epoch seconds). `compensationHtml` was null on the roles read and is left alone.""" role = _ask(get, "ExternalJobPosting", DETAIL_QUERY, {"boardId": slug, "extId": row["source_id"]}, "oatsExternalJobPosting") if not isinstance(role, dict): raise ValueError(f"Gem has no role {row['source_id']} on {slug}") text = html.unescape(strip_html(role.get("descriptionHtml") or "")) return {"description": " ".join(text.split()), "posted": parse_dt(role.get("firstPublishedTsSec"))}