Read the text of SmartRecruiters and Workable roles through detail, keeping Workable's shortcode in the row

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 13:23:58 +02:00
1 parent a19c528e3f
commit 70aa4b3117
6 files changed
+176 -11

No files matched your search

+39 -4
View File
@@ -13,16 +13,26 @@ A fetch term goes as `?q=engineer`, one a request (`Filters(query="one")`):
it matches title and text, and with `country=` it is an AND (survey,
2026-09-30: `q=engineer` answered 1,713 of Bosch's 4,830, with `country=dk`
none). How several words in one `q` combine is untested.
The list has no text. The text is read per posting by `detail`, `GET
/v1/companies/{slug}/postings/{id}` (checked 2026-09-30, Delivery Hero's
744000151349134): `jobAd.sections` holds `companyDescription`,
`jobDescription`, `qualifications` and `additionalInformation`, each with a
`title` and its `text` as HTML. The description joins the last three; the
company blurb is the same on every role of the board and is left out.
"""
from __future__ import annotations
import html
import json
import re
from collections.abc import Callable
from typing import Any
from urllib.parse import quote
from desk.sources import strip_html
from ._contract import Ask, Fields, Filters, Place, Reads
from ._shared import Get, Page, Row, only, parse_dt, path_slug_ok
@@ -35,7 +45,7 @@ PATTERNS = (
#: An unknown identifier answers an empty list, which is `empty="suspect"`.
READS = Reads(
scope="board", listing="paged", page_size=100, text="none", complete="stated",
scope="board", listing="paged", page_size=100, text="detail", complete="stated",
places="one", empty="suspect", filters=Filters(country="one", query="one"),
fields=Fields(posted="trusted", remote="stated", country="stated", seniority="stated",
employment="stated"),
@@ -96,9 +106,9 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
"""One page of https://api.smartrecruiters.com/v1/companies/{slug}/postings
-> `{"offset", "limit", "totalFound", "content": [...]}`. At most 100 a
page, whatever `limit` asks for; the next is `?offset=` further on. The
list omits the job description (only the per-posting detail endpoint has
it, and fetching one per row would be hundreds of extra requests per
board), so `description` stays empty here. A posting has one location,
list omits the job description (only the per-posting endpoint has it,
which `detail` reads for the roles worth it), so `description` stays
empty here. A posting has one location,
its `country` a lower-case ISO code, `experienceLevel` and
`typeOfEmployment` (an `id` and a `label` each)."""
out = []
@@ -130,3 +140,28 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
"employment": _word(j, "typeOfEmployment"),
})
return out
#: The sections of a posting's ad that make its description, in reading order.
#: `companyDescription` is the employer's blurb, alike on every role.
_SECTIONS = ("jobDescription", "qualifications", "additionalInformation")
def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
"""`/v1/companies/{slug}/postings/{id}` -> `jobAd.sections`: the HTML of
`jobDescription`, `qualifications` and `additionalInformation`, one
text. An answer without `jobAd.sections` is
not a posting and raises, so the role is asked again on a later pull."""
posting = quote(str(row["source_id"]), safe="")
data = json.loads(get(f"https://{_API}/v1/companies/{slug}/postings/{posting}"))
sections = ((data.get("jobAd") or {}).get("sections")
if isinstance(data, dict) else None)
if not isinstance(sections, dict):
raise ValueError("not a SmartRecruiters posting: no jobAd.sections")
parts = []
for key in _SECTIONS:
text = " ".join(html.unescape(strip_html(str((sections.get(key) or {}).get("text") or "")))
.split())
if text:
parts.append(text)
return {"description": " ".join(parts)}
+38 -3
View File
@@ -3,14 +3,25 @@ total.
Written from one real board, Hugging Face's, on 2026-09-28, when the first
board with roles was seen; jobhunt-hq did not read Workable.
The list carries no text. The text is read per role by `detail`, `GET
/api/v2/accounts/{slug}/jobs/{shortcode}` (checked 2026-09-30, Hugging Face's
DB4D7C0EC8): the list's fields again, and `description`, `requirements` and
`benefits` as HTML. The list gives the numeric `id` and the `shortcode`; the
role's `source_id` is the number, so the row keeps the `shortcode` for this
request. The API answers 429 to a run of requests; the runner spaces the
reads and stops on one.
"""
from __future__ import annotations
import html
import json
from collections.abc import Callable
from typing import Any
from desk.sources import strip_html
from ._contract import Ask, Fields, Place, Reads
from ._shared import Get, Page, Row, add_place, only, parse_dt, path_slug_ok
@@ -23,7 +34,7 @@ PATTERNS = (
)
READS = Reads(
scope="board", listing="paged", page_size=10, text="none", complete="stated",
scope="board", listing="paged", page_size=10, text="detail", complete="stated",
places="many",
fields=Fields(posted="trusted", remote="stated", employment="stated"),
robots="api: no robots.txt check needed (public board API), 2026-09-30",
@@ -82,8 +93,8 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
-> `{"total": int, "results": [...], "nextPage": token}`, as Hugging
Face's board answered on 2026-09-28. A result carries no URL and no text:
the job page is `https://apply.workable.com/{slug}/j/{shortcode}/`, and
the description is only on the per-job endpoint, one request per role,
so `description` stays empty here as it does for SmartRecruiters.
the description is only on the per-job endpoint, which `detail` reads, so
`description` stays empty here. The `shortcode` is kept in the row for it.
`location` is the first place and `locations` all of them; `department`
is a list; `remote` is a flag beside `workplace` (remote, hybrid,
on_site); `type` is the hours or the contract (full, part, ...)."""
@@ -101,6 +112,7 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
dept = ", ".join(d for d in dept if d)
out.append({
"source_id": str(j.get("id") or shortcode),
"shortcode": shortcode,
"title": title,
"places": places,
"location": "; ".join(p["text"] for p in places),
@@ -113,3 +125,26 @@ def rows(data: dict[str, Any], slug: str) -> list[Row]:
"employment": str(j.get("type") or "").strip() or None,
})
return out
#: The parts of a role's ad that make its description, in reading order.
_PARTS = ("description", "requirements", "benefits")
def detail(get: Get, slug: str, row: Row) -> dict[str, Any]:
"""`/api/v2/accounts/{slug}/jobs/{shortcode}` -> the HTML of
`description`, `requirements` and `benefits`, joined as one text. An
answer with none of the three keys is not a role and raises, so the role
is asked again on a later pull."""
shortcode = str(row.get("shortcode") or "")
if not path_slug_ok(shortcode):
raise ValueError(f"no Workable shortcode in the row: {shortcode!r}")
data = json.loads(get(f"https://{_API}/api/v2/accounts/{slug}/jobs/{shortcode}"))
if not isinstance(data, dict) or not any(k in data for k in _PARTS):
raise ValueError("not a Workable role: no description")
parts = []
for key in _PARTS:
text = " ".join(html.unescape(strip_html(str(data.get(key) or ""))).split())
if text:
parts.append(text)
return {"description": " ".join(parts)}
+1
View File
@@ -0,0 +1 @@
{"id":"744000151349134","name":"Legal Counsel, EU Employment Law (Platform Workers)","uuid":"655a2a58-fd88-4265-9515-ed20950644d9","jobId":"4a203380-42d7-423b-a1cf-adfedae74ae5","jobAdId":"3546365e-c8d6-43b8-b753-73c4f9e13ef8","defaultJobAd":false,"refNumber":"JR0081588","company":{"name":"Delivery Hero","identifier":"DeliveryHero"},"location":{"city":"Stockholm","region":"Stockholm County","country":"se","remote":false,"hybrid":false,"latitude":"59.3327036","longitude":"18.0656255","fullLocation":"Stockholm, Stockholm County, Sweden"},"customField":[{"fieldId":"6564ad3fee5f2f5b84045448","fieldLabel":"Job Family Group","valueId":"74446f87526f01023ba05b2344241d1b","valueLabel":"Legal"},{"fieldId":"6564ae9ddd552072c3a97453","fieldLabel":"Time Type","valueId":"421938a4e1d40118e396324490147300","valueLabel":"Full time"},{"fieldId":"6564aa853d74d30a33089d2a","fieldLabel":"Supervisory Organization","valueId":"81e6b64e3ee2011754bb2c1f5b01c5d8","valueLabel":"Legal Commercial Regulatory - Logistics DHSE - HIE03090"},{"fieldId":"COUNTRY","fieldLabel":"Country/Region","valueId":"hu","valueLabel":"Hungary"},{"fieldId":"6564ad09560ccd2ba9e90d24","fieldLabel":"Management Level","valueId":"ef2e8d3982de0147be16f744d2163300","valueLabel":"IC4"},{"fieldId":"6564ae71dd552072c3a97452","fieldLabel":"Worker Sub-Type","valueId":"ef2e8d3982de0123e1696445d2164400","valueLabel":"Permanent"},{"fieldId":"655ce69d695b8c5663cf1f55","fieldLabel":"Brands","valueId":"Foodora","valueLabel":"foodora"},{"fieldId":"6564ad6b1cf48c421e22419d","fieldLabel":"Job Family","valueId":"7d466ad1cc7701d28bd20926b000a75e","valueLabel":"Commercial Law"},{"fieldId":"6564abc0d641ff6fc90ebd6d","fieldLabel":"Primary Location","valueId":"dd8bd6e8ef5e01f0b31a22394e012004","valueLabel":"Berlin - Oranienburger Straße"},{"fieldId":"6564ada8e5ee8c3d9bec3405","fieldLabel":"Job Profile","valueId":"a2e7316c822c10009473b15208810000","valueLabel":"Commercial Law - IC4"}],"releasedDate":"2026-09-23T11:04:44.986Z","creator":{"name":"","avatarUrl":""},"postingUrl":"https://jobs.smartrecruiters.com/DeliveryHero/744000151349134-legal-counsel-eu-employment-law-platform-workers-","applyUrl":"https://jobs.smartrecruiters.com/DeliveryHero/744000151349134-legal-counsel-eu-employment-law-platform-workers-?oga=true","referralUrl":"https://jobs.smartrecruiters.com/external-referrals/company/DeliveryHero/publication/655a2a58-fd88-4265-9515-ed20950644d9?dcr_ci=DeliveryHero","jobAd":{"sections":{"companyDescription":{"title":"Company Description","text":"<p>foodora is part of the Delivery Hero Group, the world’s pioneering local delivery platform, our mission is to deliver an amazing experience—fast, easy, and to your door. We operate in over 65+ countries worldwide. Headquartered in Berlin, Germany. Delivery Hero has been listed on the Frankfurt Stock Exchange since 2017 and is part of the MDAX stock market index.</p>"},"jobDescription":{"title":"Job Description","text":"<p>We are on the lookout for a <strong>Legal Counsel, EU Employment Law</strong> to join the Legal Team on our journey to always deliver amazing experiences.&#xa0;</p><p>Join the team that protects and enables Delivery Hero’s growth. In the Legal Team, you’ll support key initiatives by providing proactive legal guidance that aligns business goals with regulatory standards. Your work will help mitigate risks and ensure we’re positioned for sustainable, long-term success—making you a key contributor to our mission of delivering amazing experiences worldwide.</p><p>As a <strong>Legal Counsel</strong>, you will advise Delivery Hero on employment law challenges related to platform work across the world. Our ideal candidate is a seasoned employment lawyer with litigation experience, passionate about the gig economy, and excited to help shape the future of work.</p><p><strong>About the opportunity</strong></p><ul><li><p>Lead the management of platform worker and rider classification risks globally, working closely with international employment lawyers across the Group.</p></li><li><p>Monitor and manage high-profile litigation and regulatory investigations, and develop defense strategies in collaboration with local legal counsel.</p></li><li><p>Partner with Product, Tech, and Operations teams to deeply understand delivery operations and the tech stack, and support the development of next-generation delivery solutions.</p></li><li><p>Support cross-functional teams in exploring new business models and strategic opportunities related to platform work.</p></li><li><p>Track and analyze legislative and regulatory developments impacting platform work, and advise Public Policy teams at HQ and market level.</p></li><li><p>Act as a subject-matter expert, providing guidance to senior management and stakeholders across the Group, including availability for limited international travel (up to 10%).</p></li></ul>"},"qualifications":{"title":"Qualifications","text":"<ul><li><p><strong>Fully qualified employment lawyer in an EU Line truncated
+1
View File
@@ -0,0 +1 @@
{"id":6125026,"shortcode":"DB4D7C0EC8","title":"Senior Open-Source Python Engineer, ML Developer Tools - EMEA Remote","remote":true,"location":{"country":"France","countryCode":"FR","city":"","region":null},"locations":[{"country":"France","countryCode":"FR","city":"","region":null,"hidden":true}],"state":"published","isInternal":false,"code":"","published":"2026-09-21T00:00:00.000Z","type":"full","language":"en","department":["Product"],"accountUid":"940cf17f-c078-40ac-95e8-07704e754048","approvalStatus":"approved","workplace":"remote","description":"<p>At Hugging Face, we're on a journey to democratize good AI. We are building the fastest growing platform for AI builders with over 11 million users who collectively shared over 4 million models, 1 million datasets &amp; 1.5 million Gradio apps. Our open-source libraries have more than 700,000 stars on Github.</p><p></p><p><strong>About the Role</strong></p><p>Roles at Hugging Face are very fluid and dynamic — we are looking for someone who is comfortable taking on different challenges that will evolve over time.</p><p>You'll join the Applications team, whose goal is to build the tooling that helps developers create machine learning applications on top of the primitives we offer on the Hub: <a href=\"https://www.gradio.dev\" rel=\"nofollow noreferrer noopener\" class=\"external\">Gradio</a>, <a href=\"https://huggingface.co/spaces\" rel=\"nofollow noreferrer noopener\" class=\"external\">Spaces</a>, Inference Providers, Datasets, Xet, and more. Concretely, that means maintaining and growing libraries like <a href=\"https://github.com/gradio-app/trackio\" rel=\"nofollow noreferrer noopener\" class=\"external\">Trackio</a>, a local-first, agent-friendly experiment tracking platform, <a href=\"https://github.com/gradio-app/fastrtc\" rel=\"nofollow noreferrer noopener\" class=\"external\">FastRTC</a>, our library for real-time audio and video streaming in Python, and Gradio itself, used by more than 1 million developers each month. These tools have made machine learning accessible to a wide audience — for the first time in history, anyone with an internet connection can use cutting-edge models right from their browser.</p><p>Several of these libraries are still early in their life, and new ones will start from scratch. A large part of this role is taking a project from its first users to widespread adoption: shaping the API, shipping fast, and getting it into developers' hands.</p><p>Key responsibilities include:</p><ul><li>Designing, developing, and maintaining core features across the team's libraries, ensuring scalability, reliability, and ease of use for ML developers.</li><li>Architecting complex Python projects and frameworks that serve as foundational tools for the ML community.</li><li>Growing early-stage libraries: defining the API surface, writing the demos and documentation that make them click, and driving adoption in the developer community.</li><li>Working on real-time streaming and networking capabilities, a core priority for Gradio 7 and beyond.</li><li>Collaborating with open-source contributors and engaging with the broader community through code reviews, discussions, and support.</li><li>Integrating modern frontend technologies to create seamless and intuitive user experiences for Python-based web applications.</li><li>Adapting to evolving engineering challenges and contributing to the direction of our developer tools as the landscape changes.</li></ul>","requirements":"<p></p><p>If you love open-source, are passionate about making machine learning more accessible, have experience building Python frameworks, and want to contribute to some of the fastest-growing libraries in the ML ecosystem, then we can't wait to see your application!</p><p><strong>What you'll need</strong></p><ul><li>Bachelor's degree or equivalent in Computer Science or a related field</li><li>5+ years of professional Python development experience</li><li>Experience architecting complex Python projects or frameworks</li><li>A track record of taking early-stage libraries or products from first users to broad adoption</li><li>Professional experience with JavaScript or TypeScript</li><li>Professional experience with a modern frontend framework such as React or Svelte</li><li>Experience core-contributing to or maintaining open-source software projects</li></ul><p><strong>Nice to have</strong></p><ul><li>Experience with real-time audio/video streaming or networking (WebRTC, WebSockets)</li><li>Contributions to libraries such as <a href=\"https://github.com/pipecat-ai/pipecat\" rel=\"nofollow noreferrer noopener\" class=\"external\">pipecat</a>, <a href=\"https://github.com/livekit/livekit\" rel=\"nofollow noreferrer noopener\" class=\"external\">LiveKit</a>, <a href=\"https://github.com/fastapi\" rel=\"nofollow noreferrer noopener\" class=\"external\">FastAPI</a>, or <a href=\"https://github.com/Kludex/starlette\" rel=\"nofollow noreferrer noopener\" class=\"external\">StarlLine truncated
+47 -2
View File
@@ -7,12 +7,15 @@ import json
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from plugins.readers import Ask, smartrecruiters # noqa: E402
from desk.sources import strip_html # noqa: E402
from plugins.readers import Ask, _runner, smartrecruiters # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
@@ -30,7 +33,7 @@ def _rows(*postings: dict) -> list[dict]:
def test_the_declaration_says_what_the_list_gives() -> None:
reads = smartrecruiters.READS
assert (reads.listing, reads.page_size, reads.places, reads.text, reads.empty) == \
("paged", 100, "one", "none", "suspect")
("paged", 100, "one", "detail", "suspect")
assert reads.fields.posted == "trusted" and reads.fields.seniority == "stated"
assert reads.robots.startswith("api: no robots.txt check needed")
@@ -72,3 +75,45 @@ def test_the_saved_page_still_reads() -> None:
data = json.loads((FIXTURES / "smartrecruiters.json").read_bytes())
assert [r["location"] for r in smartrecruiters.rows(data, "Acme")] == ["Berlin, Germany",
"Remote"]
def test_detail_reads_the_ad_without_the_company_blurb() -> None:
asked: list[str] = []
def get(url: str, **_: object) -> bytes:
asked.append(url)
return (FIXTURES / "smartrecruiters-detail.json").read_bytes()
found = smartrecruiters.detail(get, "deliveryhero", {"source_id": "744000151349134"})
assert asked == ["https://api.smartrecruiters.com/v1/companies/deliveryhero/postings/"
"744000151349134"]
assert set(found) == {"description"} and set(found) <= set(_runner.DETAIL_FIELDS)
text = found["description"]
data = json.loads((FIXTURES / "smartrecruiters-detail.json").read_bytes())
sections = data["jobAd"]["sections"]
assert data["jobAd"]["sections"]["companyDescription"]["text"]
assert strip_html(sections["companyDescription"]["text"])[:60] not in text
assert strip_html(sections["jobDescription"]["text"])[:60] in text
assert strip_html(sections["qualifications"]["text"])[-40:] in text
assert strip_html(sections["additionalInformation"]["text"])[-40:] in text
assert "<" not in text and " " not in text and len(text) > 1500
def test_detail_leaves_out_a_section_the_ad_lacks() -> None:
body = json.dumps({"jobAd": {"sections": {
"jobDescription": {"text": "<p>Build &amp; run</p><ul><li>APIs</li></ul>"}}}}).encode()
assert smartrecruiters.detail(lambda url, **_: body, "Acme", {"source_id": "7"}) == \
{"description": "Build & run APIs"}
@pytest.mark.parametrize("body", [b"{}", b'{"jobAd": {}}', b'{"jobAd": null}', b"[]"])
def test_detail_refuses_an_answer_that_is_not_a_posting(body: bytes) -> None:
with pytest.raises(ValueError, match="not a SmartRecruiters posting"):
smartrecruiters.detail(lambda url, **_: body, "Acme", {"source_id": "7"})
def test_detail_cannot_add_to_the_path() -> None:
asked: list[str] = []
smartrecruiters.detail(lambda url, **_: asked.append(url) or b'{"jobAd":{"sections":{}}}',
"Acme", {"source_id": "7/../x"})
assert asked == ["https://api.smartrecruiters.com/v1/companies/Acme/postings/7%2F..%2Fx"]
+50 -2
View File
@@ -8,12 +8,15 @@ import json
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from plugins.readers import Ask, workable # noqa: E402
from desk.sources import strip_html # noqa: E402
from plugins.readers import Ask, _runner, workable # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
@@ -25,7 +28,7 @@ def _rows() -> list[dict]:
def test_the_declaration_says_what_the_list_gives() -> None:
reads = workable.READS
assert (reads.listing, reads.page_size, reads.places, reads.text) == \
("paged", 10, "many", "none")
("paged", 10, "many", "detail")
assert reads.fields.posted == "trusted" and reads.fields.employment == "stated"
assert reads.robots.startswith("api: no robots.txt check needed")
@@ -61,3 +64,48 @@ def test_the_stated_hours_are_the_hosts_word() -> None:
def test_the_shape_of_a_real_list_answer_is_read() -> None:
data = json.loads((FIXTURES / "workable-places.json").read_bytes())
assert workable.rows(data, "acme")[0]["url"] == "https://apply.workable.com/acme/j/AAA111/"
def test_a_row_keeps_the_shortcode_the_detail_url_needs() -> None:
first, remote, oslo = _rows()
assert [r["source_id"] for r in (first, remote, oslo)] == ["8001", "8002", "8003"]
assert [r["shortcode"] for r in (first, remote, oslo)] == ["AAA111", "BBB222", "CCC333"]
def test_detail_reads_description_requirements_and_benefits() -> None:
asked: list[str] = []
body = (FIXTURES / "workable-detail.json").read_bytes()
def get(url: str, **_: object) -> bytes:
asked.append(url)
return body
row = {"source_id": "6125026", "shortcode": "DB4D7C0EC8"}
found = workable.detail(get, "huggingface", row)
assert asked == ["https://apply.workable.com/api/v2/accounts/huggingface/jobs/DB4D7C0EC8"]
assert set(found) == {"description"} and set(found) <= set(_runner.DETAIL_FIELDS)
text, data = found["description"], json.loads(body)
for key in ("description", "requirements", "benefits"):
assert strip_html(data[key])[:40] in text
assert text.index(strip_html(data["description"])[:40]) < \
text.index(strip_html(data["benefits"])[:40])
assert "<" not in text and " " not in text and len(text) > 1500
def test_detail_joins_what_is_there_and_unescapes() -> None:
body = json.dumps({"description": "<p>Build &amp; run</p>", "requirements": None,
"benefits": "<ul><li>Pension</li></ul>"}).encode()
assert workable.detail(lambda url, **_: body, "acme", {"shortcode": "AAA111"}) == \
{"description": "Build & run Pension"}
@pytest.mark.parametrize("body", [b"{}", b"[]", b'{"title": "x"}'])
def test_detail_refuses_an_answer_that_is_not_a_role(body: bytes) -> None:
with pytest.raises(ValueError, match="not a Workable role"):
workable.detail(lambda url, **_: body, "acme", {"shortcode": "AAA111"})
@pytest.mark.parametrize("row", [{}, {"shortcode": ""}, {"shortcode": "a/b"}, {"shortcode": ".."}])
def test_detail_needs_a_plain_shortcode(row: dict) -> None:
with pytest.raises(ValueError, match="shortcode"):
workable.detail(lambda url, **_: b"{}", "acme", row)