157 lines
6.8 KiB
Python
157 lines
6.8 KiB
Python
"""Gem: `gem:<board>` for jobs.gem.com/<board>, one GraphQL POST for the
|
|
whole board and one more for a role's text and date. The board is the path
|
|
after `jobs.gem.com/`, and it is case-sensitive (`Productboard` is an empty
|
|
board where `productboard` is not).
|
|
|
|
The two queries below were read out of Gem's page scripts
|
|
(`static.gem.com/scripts/jobBoards.*.js` and `ExternalJobBoardList.*.js`) on
|
|
2026-09-29 and run against Productboard's and sennder's boards; the schema is
|
|
Gem's own and not documented, so a new bundle can rename a field. Each query is
|
|
one constant, and an answer that lacks the shape read here fails with an error
|
|
that says which part is missing rather than reading as an empty board. The
|
|
endpoint answered with no Origin and no Referer header, so none is sent. An
|
|
unknown board answers 200 with an empty list, like a board with no role open.
|
|
|
|
The listing has no description and no date; both are on the role's page.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import re
|
|
from collections.abc import Callable
|
|
from typing import Any
|
|
|
|
|
|
from ._contract import Ask, Fields, Place, Reads, Row
|
|
from ._shared import Get, Page, add_place, html_text, only, parse_dt
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
READS = Reads(
|
|
scope="board", listing="paged", text="detail", complete="end",
|
|
places="many", empty="suspect",
|
|
fields=Fields(posted="listed", remote="stated", employment="stated"),
|
|
robots=("read: POST /api/public/graphql on jobs.gem.com, whose /robots.txt answers "
|
|
"404, so nothing is disallowed (checked 2026-09-30)"),
|
|
)
|
|
|
|
NAME = "gem"
|
|
LABEL = "Gem"
|
|
KEEP_CASE = True
|
|
PATTERNS = (r"jobs\.gem\.com/([A-Za-z0-9_-]+)",)
|
|
|
|
_HOST = "jobs.gem.com"
|
|
_API = f"https://{_HOST}/api/public/graphql"
|
|
|
|
#: A board is one path segment of letters, digits, `-` and `_`. It travels as a
|
|
#: GraphQL variable and never into a host name, so the host stays Gem's.
|
|
_BOARD = re.compile(r"[A-Za-z0-9_-]+")
|
|
|
|
#: Every role of a board in one answer, with no paging and no total.
|
|
LIST_QUERY = (
|
|
"query JobBoardList($boardId: String!) { oatsExternalJobPostings(boardId: $boardId) "
|
|
"{ jobPostings { id extId title locations { id name city isoCountry isRemote extId } "
|
|
"job { id department { id name extId } locationType employmentType } } } }")
|
|
|
|
#: One role's description and the date it was first published.
|
|
DETAIL_QUERY = (
|
|
"query ExternalJobPosting($boardId: String!, $extId: String!) "
|
|
"{ oatsExternalJobPosting(boardId: $boardId, extId: $extId) "
|
|
"{ id title descriptionHtml extId firstPublishedTsSec "
|
|
"locations { id extId name city isoCountry isRemote } "
|
|
"job { id locationType employmentType requisitionId teamDisplayName "
|
|
"department { id extId name } } compensationHtml } }")
|
|
|
|
|
|
def slug_ok(slug: str) -> bool:
|
|
return bool(_BOARD.fullmatch(slug))
|
|
|
|
|
|
def allowed(slug: str) -> Callable[[str], bool]:
|
|
return only(_HOST)
|
|
|
|
|
|
def careers_url(slug: str) -> str:
|
|
return f"https://{_HOST}/{slug}"
|
|
|
|
|
|
def _ask(get: Get, operation: str, query: str, variables: dict[str, str],
|
|
path: str) -> Any:
|
|
"""One GraphQL call, and the part of its answer at `data.<path>`. An
|
|
`errors` list or a `data` without the key is Gem's schema having moved (or
|
|
a refused query), and is raised as such; a `null` at the key is returned."""
|
|
answer = json.loads(get(_API, method="POST", payload={
|
|
"operationName": operation, "variables": variables, "query": query}))
|
|
if not isinstance(answer, dict):
|
|
raise ValueError(f"not a Gem answer: {type(answer).__name__}")
|
|
if answer.get("errors"):
|
|
first = answer["errors"][0]
|
|
message = first.get("message") if isinstance(first, dict) else first
|
|
raise ValueError(f"Gem's {operation} query was refused: {message}")
|
|
data = answer.get("data")
|
|
if not isinstance(data, dict) or path not in data:
|
|
raise ValueError(f"Gem's {operation} answer has no {path}; the query may be out of date")
|
|
return data[path]
|
|
|
|
|
|
def page(get: Get, slug: str, cursor: Any, ask: Ask) -> Page:
|
|
"""The one page there is: `data.oatsExternalJobPostings.jobPostings`."""
|
|
board = _ask(get, "JobBoardList", LIST_QUERY, {"boardId": slug}, "oatsExternalJobPostings")
|
|
postings = board.get("jobPostings") if isinstance(board, dict) else None
|
|
if not isinstance(postings, list):
|
|
raise ValueError("Gem's JobBoardList answer has no jobPostings list; "
|
|
"the query may be out of date")
|
|
return postings, [{"id": p.get("extId") or p.get("id") or ""} for p in postings], None, None
|
|
|
|
|
|
def rows(postings: list[dict[str, Any]], slug: str) -> list[Row]:
|
|
"""A role has `extId` (a number from a synced Greenhouse, else an opaque
|
|
string; the role's page is `jobs.gem.com/<board>/<extId>`), `title`,
|
|
`locations` (`name`, `city`, `isoCountry` as ISO alpha-3, `isRemote`) and
|
|
`job` (`department`, `locationType`, `employmentType`). No pay is stated
|
|
in the list."""
|
|
out = []
|
|
for p in postings:
|
|
title = (p.get("title") or "").strip()
|
|
ext_id = str(p.get("extId") or "")
|
|
locations = p.get("locations") or []
|
|
places: list[Place] = []
|
|
for loc in locations:
|
|
text = str(loc.get("city") or loc.get("name") or "").strip()
|
|
add_place(places, {"text": text, "remote": loc["isRemote"]
|
|
if isinstance(loc.get("isRemote"), bool) else None})
|
|
job = p.get("job") or {}
|
|
remote: bool | str | None = str(job.get("locationType") or "").strip() or None
|
|
if remote is None and locations and all(loc.get("isRemote") for loc in locations):
|
|
remote = True
|
|
row: Row = {
|
|
"source_id": ext_id,
|
|
"title": title,
|
|
"places": places,
|
|
"location": "; ".join(pl["text"] for pl in places),
|
|
"url": f"{careers_url(slug)}/{ext_id}",
|
|
"posted": None, # only the role's own query has a date
|
|
"department": str((job.get("department") or {}).get("name") or "").strip(),
|
|
"description": "",
|
|
"remote": remote,
|
|
"salary_text": "",
|
|
}
|
|
if employment := str(job.get("employmentType") or "").strip():
|
|
row["employment"] = employment
|
|
out.append(row)
|
|
return out
|
|
|
|
|
|
def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
|
"""`oatsExternalJobPosting` of the role: `descriptionHtml` and
|
|
`firstPublishedTsSec` (epoch seconds). `compensationHtml` was null on the
|
|
roles read and is left alone."""
|
|
role = _ask(get, "ExternalJobPosting", DETAIL_QUERY,
|
|
{"boardId": slug, "extId": row["source_id"]}, "oatsExternalJobPosting")
|
|
if not isinstance(role, dict):
|
|
raise ValueError(f"Gem has no role {row['source_id']} on {slug}")
|
|
return {"description": html_text(role.get("descriptionHtml")),
|
|
"posted": parse_dt(role.get("firstPublishedTsSec"))}
|