Read the text of SmartRecruiters and Workable roles through detail, keeping Workable's shortcode in the row
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
a19c528e3f
commit
70aa4b3117
6 files changed
+176
-11
No files matched your search
@@ -13,16 +13,26 @@ A fetch term goes as `?q=engineer`, one a request (`Filters(query="one")`):
|
||||
it matches title and text, and with `country=` it is an AND (survey,
|
||||
2026-09-30: `q=engineer` answered 1,713 of Bosch's 4,830, with `country=dk`
|
||||
none). How several words in one `q` combine is untested.
|
||||
|
||||
The list has no text. The text is read per posting by `detail`, `GET
|
||||
/v1/companies/{slug}/postings/{id}` (checked 2026-09-30, Delivery Hero's
|
||||
744000151349134): `jobAd.sections` holds `companyDescription`,
|
||||
`jobDescription`, `qualifications` and `additionalInformation`, each with a
|
||||
`title` and its `text` as HTML. The description joins the last three; the
|
||||
company blurb is the same on every role of the board and is left out.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
from urllib.parse import quote
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Filters, Place, Reads
|
||||
from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok
|
||||
|
||||
@@ -35,7 +45,7 @@ PATTERNS = (
|
||||
|
||||
#: An unknown identifier answers an empty list, which is `empty="suspect"`.
|
||||
READS = Reads(
|
||||
scope="board", listing="paged", page_size=100, text="none", complete="stated",
|
||||
scope="board", listing="paged", page_size=100, text="detail", complete="stated",
|
||||
places="one", empty="suspect", filters=Filters(country="one", query="one"),
|
||||
fields=Fields(posted="trusted", remote="stated", country="stated", seniority="stated",
|
||||
employment="stated"),
|
||||
@@ -96,9 +106,9 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
"""One page of https://api.smartrecruiters.com/v1/companies/{slug}/postings
|
||||
-> `{"offset", "limit", "totalFound", "content": [...]}`. At most 100 a
|
||||
page, whatever `limit` asks for; the next is `?offset=` further on. The
|
||||
list omits the job description (only the per-posting detail endpoint has
|
||||
it, and fetching one per row would be hundreds of extra requests per
|
||||
board), so `description` stays empty here. A posting has one location,
|
||||
list omits the job description (only the per-posting endpoint has it,
|
||||
which `detail` reads for the roles worth it), so `description` stays
|
||||
empty here. A posting has one location,
|
||||
its `country` a lower-case ISO code, `experienceLevel` and
|
||||
`typeOfEmployment` (an `id` and a `label` each)."""
|
||||
out = []
|
||||
@@ -130,3 +140,28 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
"employment": _word(j, "typeOfEmployment"),
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
#: The sections of a posting's ad that make its description, in reading order.
|
||||
#: `companyDescription` is the employer's blurb, alike on every role.
|
||||
_SECTIONS = ("jobDescription", "qualifications", "additionalInformation")
|
||||
|
||||
|
||||
def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
"""`/v1/companies/{slug}/postings/{id}` -> `jobAd.sections`: the HTML of
|
||||
`jobDescription`, `qualifications` and `additionalInformation`, one
|
||||
text. An answer without `jobAd.sections` is
|
||||
not a posting and raises, so the role is asked again on a later pull."""
|
||||
posting = quote(str(row["source_id"]), safe="")
|
||||
data = json.loads(get(f"https://{_API}/v1/companies/{slug}/postings/{posting}"))
|
||||
sections = ((data.get("jobAd") or {}).get("sections")
|
||||
if isinstance(data, dict) else None)
|
||||
if not isinstance(sections, dict):
|
||||
raise ValueError("not a SmartRecruiters posting: no jobAd.sections")
|
||||
parts = []
|
||||
for key in _SECTIONS:
|
||||
text = " ".join(html.unescape(strip_html(str((sections.get(key) or {}).get("text") or "")))
|
||||
.split())
|
||||
if text:
|
||||
parts.append(text)
|
||||
return {"description": " ".join(parts)}
|
||||
@@ -3,14 +3,25 @@ total.
|
||||
|
||||
Written from one real board, Hugging Face's, on 2026-09-28, when the first
|
||||
board with roles was seen; jobhunt-hq did not read Workable.
|
||||
|
||||
The list carries no text. The text is read per role by `detail`, `GET
|
||||
/api/v2/accounts/{slug}/jobs/{shortcode}` (checked 2026-09-30, Hugging Face's
|
||||
DB4D7C0EC8): the list's fields again, and `description`, `requirements` and
|
||||
`benefits` as HTML. The list gives the numeric `id` and the `shortcode`; the
|
||||
role's `source_id` is the number, so the row keeps the `shortcode` for this
|
||||
request. The API answers 429 to a run of requests; the runner spaces the
|
||||
reads and stops on one.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
from desk.sources import strip_html
|
||||
|
||||
from ._contract import Ask, Fields, Place, Reads
|
||||
from ._shared import Get, Page, Row, add_place, only, parse_dt, path_slug_ok
|
||||
|
||||
@@ -23,7 +34,7 @@ PATTERNS = (
|
||||
)
|
||||
|
||||
READS = Reads(
|
||||
scope="board", listing="paged", page_size=10, text="none", complete="stated",
|
||||
scope="board", listing="paged", page_size=10, text="detail", complete="stated",
|
||||
places="many",
|
||||
fields=Fields(posted="trusted", remote="stated", employment="stated"),
|
||||
robots="api: no robots.txt check needed (public board API), 2026-09-30",
|
||||
@@ -82,8 +93,8 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
-> `{"total": int, "results": [...], "nextPage": token}`, as Hugging
|
||||
Face's board answered on 2026-09-28. A result carries no URL and no text:
|
||||
the job page is `https://apply.workable.com/{slug}/j/{shortcode}/`, and
|
||||
the description is only on the per-job endpoint, one request per role,
|
||||
so `description` stays empty here as it does for SmartRecruiters.
|
||||
the description is only on the per-job endpoint, which `detail` reads, so
|
||||
`description` stays empty here. The `shortcode` is kept in the row for it.
|
||||
`location` is the first place and `locations` all of them; `department`
|
||||
is a list; `remote` is a flag beside `workplace` (remote, hybrid,
|
||||
on_site); `type` is the hours or the contract (full, part, ...)."""
|
||||
@@ -101,6 +112,7 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
dept = ", ".join(d for d in dept if d)
|
||||
out.append({
|
||||
"source_id": str(j.get("id") or shortcode),
|
||||
"shortcode": shortcode,
|
||||
"title": title,
|
||||
"places": places,
|
||||
"location": "; ".join(p["text"] for p in places),
|
||||
@@ -113,3 +125,26 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
|
||||
"employment": str(j.get("type") or "").strip() or None,
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
#: The parts of a role's ad that make its description, in reading order.
|
||||
_PARTS = ("description", "requirements", "benefits")
|
||||
|
||||
|
||||
def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
|
||||
"""`/api/v2/accounts/{slug}/jobs/{shortcode}` -> the HTML of
|
||||
`description`, `requirements` and `benefits`, joined as one text. An
|
||||
answer with none of the three keys is not a role and raises, so the role
|
||||
is asked again on a later pull."""
|
||||
shortcode = str(row.get("shortcode") or "")
|
||||
if not path_slug_ok(shortcode):
|
||||
raise ValueError(f"no Workable shortcode in the row: {shortcode!r}")
|
||||
data = json.loads(get(f"https://{_API}/api/v2/accounts/{slug}/jobs/{shortcode}"))
|
||||
if not isinstance(data, dict) or not any(k in data for k in _PARTS):
|
||||
raise ValueError("not a Workable role: no description")
|
||||
parts = []
|
||||
for key in _PARTS:
|
||||
text = " ".join(html.unescape(strip_html(str(data.get(key) or ""))).split())
|
||||
if text:
|
||||
parts.append(text)
|
||||
return {"description": " ".join(parts)}
|
||||
Reference in new issue
Block a user