Rename plugins/atshosts to plugins/readers
Every import, test and live doc follows the new path; behaviour is unchanged. tests/test_atshosts.py becomes tests/test_readers.py. plan-readers.md and review-ingest.md keep the old name, since they describe the state before. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
af337ef558
commit
73dec9028e
48 files changed
+104
-104
No files matched your search
@@ -0,0 +1,152 @@
|
||||
"""Gem: `gem:<board>` for jobs.gem.com/<board>, one GraphQL POST for the
|
||||
whole board and one more for a role's text and date. The board is the path
|
||||
after `jobs.gem.com/`, and it is case-sensitive (`Productboard` is an empty
|
||||
board where `productboard` is not).
|
||||
|
||||
The two queries below were read out of Gem's page scripts
|
||||
(`static.gem.com/scripts/jobBoards.*.js` and `ExternalJobBoardList.*.js`) on
|
||||
2026-09-29 and run against Productboard's and sennder's boards; the schema is
|
||||
Gem's own and not documented, so a new bundle can rename a field. Each query is
|
||||
one constant, and an answer that lacks the shape read here fails with an error
|
||||
that says which part is missing rather than reading as an empty board. The
|
||||
endpoint answered with no Origin and no Referer header, so none is sent. An
|
||||
unknown board answers 200 with an empty list, like a board with no role open.
|
||||
|
||||
The listing has no description and no date; both are on the role's page.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._shared import Get, Page, Row, only, parse_dt
|
||||
|
||||
NAME = "gem"
|
||||
LABEL = "Gem"
|
||||
KEEP_CASE = True
|
||||
PATTERNS = (r"jobs\.gem\.com/([A-Za-z0-9_-]+)",)
|
||||
|
||||
_HOST = "jobs.gem.com"
|
||||
_API = f"https://{_HOST}/api/public/graphql"
|
||||
|
||||
#: A board is one path segment of letters, digits, `-` and `_`. It travels as a
|
||||
#: GraphQL variable and never into a host name, so the host stays Gem's.
|
||||
_BOARD = re.compile(r"[A-Za-z0-9_-]+")
|
||||
|
||||
#: Every role of a board in one answer, with no paging and no total.
|
||||
LIST_QUERY = (
|
||||
"query JobBoardList($boardId: String!) { oatsExternalJobPostings(boardId: $boardId) "
|
||||
"{ jobPostings { id extId title locations { id name city isoCountry isRemote extId } "
|
||||
"job { id department { id name extId } locationType employmentType } } } }")
|
||||
|
||||
#: One role's description and the date it was first published.
|
||||
DETAIL_QUERY = (
|
||||
"query ExternalJobPosting($boardId: String!, $extId: String!) "
|
||||
"{ oatsExternalJobPosting(boardId: $boardId, extId: $extId) "
|
||||
"{ id title descriptionHtml extId firstPublishedTsSec "
|
||||
"locations { id extId name city isoCountry isRemote } "
|
||||
"job { id locationType employmentType requisitionId teamDisplayName "
|
||||
"department { id extId name } } compensationHtml } }")
|
||||
|
||||
#: `job.locationType` -> `store._remote`'s words.
|
||||
_REMOTE = {"IN_OFFICE": "onsite", "HYBRID": "hybrid", "REMOTE": "remote"}
|
||||
|
||||
|
||||
def slug_ok(slug: str) -> bool:
|
||||
return bool(_BOARD.fullmatch(slug))
|
||||
|
||||
|
||||
def allowed(slug: str) -> Callable[[str], bool]:
|
||||
return only(_HOST)
|
||||
|
||||
|
||||
def careers_url(slug: str) -> str:
|
||||
return f"https://{_HOST}/{slug}"
|
||||
|
||||
|
||||
def robots_page(slug: str) -> str:
|
||||
return careers_url(slug)
|
||||
|
||||
|
||||
def _ask(get: Get, operation: str, query: str, variables: dict[str, str],
|
||||
path: str) -> Any:
|
||||
"""One GraphQL call, and the part of its answer at `data.<path>`. An
|
||||
`errors` list or a `data` without the key is Gem's schema having moved (or
|
||||
a refused query), and is raised as such; a `null` at the key is returned."""
|
||||
answer = json.loads(get(_API, method="POST", payload={
|
||||
"operationName": operation, "variables": variables, "query": query}))
|
||||
if not isinstance(answer, dict):
|
||||
raise ValueError(f"not a Gem answer: {type(answer).__name__}")
|
||||
if answer.get("errors"):
|
||||
first = answer["errors"][0]
|
||||
message = first.get("message") if isinstance(first, dict) else first
|
||||
raise ValueError(f"Gem's {operation} query was refused: {message}")
|
||||
data = answer.get("data")
|
||||
if not isinstance(data, dict) or path not in data:
|
||||
raise ValueError(f"Gem's {operation} answer has no {path}; the query may be out of date")
|
||||
return data[path]
|
||||
|
||||
|
||||
def page(get: Get, slug: str, cursor: Any) -> Page:
|
||||
"""The one page there is: `data.oatsExternalJobPostings.jobPostings`."""
|
||||
board = _ask(get, "JobBoardList", LIST_QUERY, {"boardId": slug}, "oatsExternalJobPostings")
|
||||
postings = board.get("jobPostings") if isinstance(board, dict) else None
|
||||
if not isinstance(postings, list):
|
||||
raise ValueError("Gem's JobBoardList answer has no jobPostings list; "
|
||||
"the query may be out of date")
|
||||
return postings, [{"id": p.get("extId") or p.get("id") or ""} for p in postings], None, None
|
||||
|
||||
|
||||
def rows(postings: list[dict[str, Any]], slug: str) -> list[Row]:
|
||||
"""A role has `extId` (a number from a synced Greenhouse, else an opaque
|
||||
string; the role's page is `jobs.gem.com/<board>/<extId>`), `title`,
|
||||
`locations` (`name`, `city`, `isoCountry` as ISO alpha-3, `isRemote`) and
|
||||
`job` (`department`, `locationType`, `employmentType`). No pay is stated
|
||||
in the list."""
|
||||
out = []
|
||||
for p in postings:
|
||||
title = (p.get("title") or "").strip()
|
||||
ext_id = str(p.get("extId") or "")
|
||||
if not title or not ext_id:
|
||||
continue
|
||||
locations = p.get("locations") or []
|
||||
places: list[str] = []
|
||||
for loc in locations:
|
||||
place = str(loc.get("city") or loc.get("name") or "").strip()
|
||||
if place and place not in places:
|
||||
places.append(place)
|
||||
job = p.get("job") or {}
|
||||
remote = _REMOTE.get(str(job.get("locationType") or "").upper())
|
||||
if remote is None and locations and all(loc.get("isRemote") for loc in locations):
|
||||
remote = "remote"
|
||||
out.append({
|
||||
"source_id": ext_id,
|
||||
"title": title,
|
||||
"location": "; ".join(places),
|
||||
"url": f"{careers_url(slug)}/{ext_id}",
|
||||
"posted": None, # only the role's own query has a date
|
||||
"department": str((job.get("department") or {}).get("name") or "").strip(),
|
||||
"description": "",
|
||||
"remote": remote,
|
||||
"salary_text": "",
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
"""`oatsExternalJobPosting` of the role: `descriptionHtml` and
|
||||
`firstPublishedTsSec` (epoch seconds). `compensationHtml` was null on the
|
||||
roles read and is left alone."""
|
||||
role = _ask(get, "ExternalJobPosting", DETAIL_QUERY,
|
||||
{"boardId": slug, "extId": row["source_id"]}, "oatsExternalJobPosting")
|
||||
if not isinstance(role, dict):
|
||||
raise ValueError(f"Gem has no role {row['source_id']} on {slug}")
|
||||
text = html.unescape(strip_html(role.get("descriptionHtml") or ""))
|
||||
return {"description": " ".join(text.split()),
|
||||
"posted": parse_dt(role.get("firstPublishedTsSec"))}
|
||||
Reference in new issue
Block a user