diff --git a/plugins/atshosts/deel.py b/plugins/atshosts/deel.py new file mode 100644 index 0000000..f0b6291 --- /dev/null +++ b/plugins/atshosts/deel.py @@ -0,0 +1,148 @@ +"""Deel: `deel:` for jobs.deel.com/, the whole board in one HTML +page. + +Written from Klarna's board on 2026-09-29. Deel's job board is a Next.js page +with no public JSON endpoint (its `/api/` is disallowed by robots.txt), but +the server-rendered page carries the board in its React Server Components +payload: a stream of `self.__next_f.push([1, "..."])` scripts whose strings, +joined, hold `"jobPostings":[...]` with every open role, and, beside it, the +rich-text descriptions as separate text rows that a posting names by +reference (`"richtextDescription":"$1a"`). Klarna's 96 roles came in one +page, the same 96 as the page's own JSON-LD list, so no description needs a +read of the role's page. +""" + +from __future__ import annotations + +import html +import json +import re +from collections.abc import Callable +from typing import Any + +from desk.sources import strip_html + +from ._shared import Row, num, only, parse_dt, pay_line + +NAME = "deel" +LABEL = "Deel" +KEEP_CASE = True # `jobs.deel.com/Klarna` is "Job Board Not Found", `/klarna` is the board +PATTERNS = ( + # `/job-boards/` is the address Klarna's own site links to. + r"jobs\.deel\.com/(?:job-boards/)?(?!job-boards\b)([A-Za-z0-9][A-Za-z0-9_-]*)", +) + +_HOST = "jobs.deel.com" + +#: A slug is one path segment that starts with a letter or a digit, so `_next` +#: and `.` never name a board. +_SLUG = re.compile(r"[A-Za-z0-9][A-Za-z0-9_-]*") + +#: One script of the payload, its JSON string literal captured. Kind 1 is +#: the data; the others (`[0]`, `[2, ...]`) carry the bootstrap. +_PUSH = re.compile(r'self\.__next_f\.push\(\[1,("(?:[^"\\]|\\.)*")\]\)') + +#: One row of the joined stream: `:` and then either `T,` +#: and that many bytes of text, or a line of JSON. +_ROW = re.compile(rb"([0-9a-f]*):") +_TEXT = re.compile(rb"T([0-9a-f]+),") + +#: Deel's `workArrangementEnum` -> `store._remote`'s words. +_REMOTE = {"ON_SITE": "onsite", "HYBRID": "hybrid", "REMOTE": "remote"} + + +def slug_ok(slug: str) -> bool: + return bool(_SLUG.fullmatch(slug)) + + +def allowed(slug: str) -> Callable[[str], bool]: + return only(_HOST) + + +def careers_url(slug: str) -> str: + return f"https://{_HOST}/{slug}" + + +def robots_page(slug: str) -> str: + return careers_url(slug) + + +def feed_url(slug: str) -> str: + return careers_url(slug) + + +def _stream(body: bytes) -> dict[str, str]: + """The payload's rows by id. A text row's length is in bytes of UTF-8 and + the next row follows it directly, so the stream is read as bytes, one row + after another, rather than split on newlines.""" + text = "".join(json.loads(m.group(1)) for m in _PUSH.finditer(body.decode("utf-8", "replace"))) + data = text.encode() + out: dict[str, str] = {} + pos = 0 + while pos < len(data) and (row := _ROW.match(data, pos)): + pos = row.end() + if data[pos:pos + 1] == b"T" and (size := _TEXT.match(data, pos)): + end = size.end() + int(size.group(1), 16) + out[row.group(1).decode()] = data[size.end():end].decode("utf-8", "replace") + pos = end + else: + end = data.find(b"\n", pos) + end = len(data) if end < 0 else end + out[row.group(1).decode()] = data[pos:end].decode("utf-8", "replace") + pos = end + 1 + return out + + +def _postings(body: bytes) -> tuple[list[dict[str, Any]], dict[str, str]]: + """The board's postings and the stream they refer into. A page without + the postings array is not a board (a slug Deel does not know answers + "Job Board Not Found"), and raises rather than read as an empty board.""" + stream = _stream(body) + for value in stream.values(): + start = value.find('"jobPostings":[') + if start >= 0: + postings, _end = json.JSONDecoder().raw_decode(value[start + len('"jobPostings":'):]) + return postings, stream + raise ValueError("not a Deel job board: no jobPostings in the page") + + +def parse(body: bytes, slug: str) -> list[Row]: + """`jobs.deel.com/` -> one row per posting. `location` joins the + role's places with "; ". `posted` is the day the posting was published + (its publication's state), not the day the role was drafted. + + Deel states pay as a currency and two amounts with no period, and the + amounts run from 775,398 SEK to 55,206 SEK on one board, the latter + plainly a month's pay. A yearly figure cannot be told from a monthly one, + so the columns stay empty and `salary_text` gives the range as stated.""" + postings, stream = _postings(body) + out = [] + for p in postings: + title = (p.get("title") or "").strip() + post_id = str(p.get("id") or "") + if not title or not post_id: + continue + job = p.get("job") or {} + places = [name for loc in job.get("jobLocations") or [] + if (name := ((loc.get("location") or {}).get("name") or "").strip())] + departments = [name for d in job.get("jobDepartments") or [] + if (name := ((d.get("department") or {}).get("name") or "").strip())] + pay = job.get("currentCompensation") or {} + published = ((p.get("jobPostingPublications") or [{}])[0].get("currentState") or {}) + description = p.get("richtextDescription") or "" + if description.startswith("$"): + description = stream.get(description[1:], "") + out.append({ + "source_id": post_id, + "title": title, + "location": "; ".join(dict.fromkeys(places)), + "url": f"https://{_HOST}/{slug}/job-details/{post_id}/overview", + "posted": parse_dt(published.get("createdAt") or p.get("createdAt")), + "department": ", ".join(dict.fromkeys(departments)), + "description": html.unescape(strip_html(description)), + "remote": _REMOTE.get(str(job.get("workArrangementEnum") or "")), + "salary_text": (pay_line(num(pay.get("minAmount")), num(pay.get("maxAmount")), + pay.get("currencyIsoCode"), "") + if p.get("isCompensationVisible") else ""), + }) + return out diff --git a/tests/fixtures/ats/deel.html b/tests/fixtures/ats/deel.html new file mode 100644 index 0000000..0855ffa --- /dev/null +++ b/tests/fixtures/ats/deel.html @@ -0,0 +1,6 @@ +Klarna Careers
+ + + + + diff --git a/tests/test_ats_deel.py b/tests/test_ats_deel.py new file mode 100644 index 0000000..2d33524 --- /dev/null +++ b/tests/test_ats_deel.py @@ -0,0 +1,173 @@ +"""The Deel host module against a saved page (trimmed from Klarna's board, +2026-09-29), and the fetch around it. No network.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any + +import httpx +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from app.jobs.discover import boards_in # noqa: E402 +from plugins import ats, atshosts # noqa: E402 +from plugins.atshosts import deel # noqa: E402 + +PAGE = (ROOT / "tests" / "fixtures" / "ats" / "deel.html").read_bytes() +URL = "https://jobs.deel.com/klarna" + + +def test_rows_come_from_the_payload_with_descriptions_resolved() -> None: + rows = deel.parse(PAGE, "klarna") + assert [r["source_id"][:8] for r in rows] == [ + "21584046", "a66b42b6", "f96bb9d4", "a8be45ac", "a6649d86"] + first = rows[0] + assert first["title"] == "Incident Manager" + assert first["url"] == ("https://jobs.deel.com/klarna/job-details/" + "21584046-56d5-44c4-8b18-7e15dc659827/overview") + assert first["location"] == "Stockholm" and first["department"] == "Engineering" + assert first["remote"] == "onsite" + assert first["description"].startswith("Klarna, briefly At Klarna") + assert "<" not in first["description"] and "$" not in first["description"][:3] + assert all(r["description"] for r in rows) + # The day the posting was published, not the day it was drafted (12:27). + assert first["posted"] is not None and first["posted"].isoformat() == \ + "2026-09-07T12:33:38.484000+00:00" + + +def test_several_locations_are_joined_and_remote_uses_the_stores_words() -> None: + rows = {r["source_id"][:8]: r for r in deel.parse(PAGE, "klarna")} + assert rows["a8be45ac"]["location"] == "Stockholm; London; Milan; Warsaw" + assert rows["f96bb9d4"]["remote"] == "remote" + assert rows["f96bb9d4"]["location"] == "Ohio" + + +def test_pay_is_only_text_since_the_period_is_not_stated() -> None: + rows = {r["source_id"][:8]: r for r in deel.parse(PAGE, "klarna")} + assert rows["21584046"]["salary_text"] == "SEK 775,398 to 1,108,832" + assert rows["a66b42b6"]["salary_text"] == "GBP from 105,197" + assert rows["f96bb9d4"]["salary_text"] == "" # a role with no range + assert all("salary_min" not in r for r in rows.values()) + + +def _page_with(body: dict[str, Any]) -> bytes: + """A page whose one payload row holds `body`, and a description row of + multibyte text whose length is given in bytes, as the stream does.""" + text = "Zo\u00eb\u2019s caf\u00e9 \u2014 100 \u20ac" # 2, 3 and 1 bytes wide + row = f"5:T{len(text.encode()):x},{text}" + \ + '6:["$","div",null,' + json.dumps(body, ensure_ascii=False, separators=(",", ":")) + "]\n" + return f''.encode() + + +def test_a_text_row_is_read_by_its_length_in_bytes() -> None: + body = {"jobPostings": [{"id": "a", "title": "Baker", "richtextDescription": "$5", + "job": {}}, {"id": "b", "title": "Cook", + "richtextDescription": "

Inline & done

", + "job": {}}]} + rows = deel.parse(_page_with(body), "acme") + assert [r["description"] for r in rows] == ["Zo\u00eb\u2019s caf\u00e9 \u2014 100 \u20ac", + "Inline & done"] + assert rows[0]["remote"] is None and rows[0]["location"] == "" and rows[0]["posted"] is None + + +def test_a_board_without_roles_is_an_empty_board_but_another_page_is_not_a_board() -> None: + assert deel.parse(_page_with({"jobPostings": []}), "acme") == [] + with pytest.raises(ValueError, match="not a Deel job board"): + deel.parse(b"Job Board Not Found", "nobody") + + +def test_a_posting_without_a_title_or_id_is_skipped() -> None: + body = {"jobPostings": [{"id": "a", "title": " ", "job": {}}, {"title": "x", "job": {}}, + {"id": "c", "title": "Cook", "job": {}}]} + assert [r["source_id"] for r in deel.parse(_page_with(body), "acme")] == ["c"] + + +# ------------------------------------------------------------ the fetch + +@pytest.fixture +def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: + answers: dict[str, Any] = {} + asked: list[str] = [] + + def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None, + www_twin=False): + asked.append(url) + if not ats._allowed_host(host, slug)(httpx.URL(url).host): + raise ats._Refused(url) + answer = answers[url] + if isinstance(answer, BaseException): + raise answer + return answer + + monkeypatch.setattr(ats, "_get", fake_get) + return {"answers": answers, "asked": asked} + + +def test_the_board_is_one_request_and_complete(web: dict[str, Any]) -> None: + web["answers"][URL] = PAGE + with httpx.Client() as client: + rows, complete = ats._fetch_rows(client, "deel:klarna") + assert len(rows) == 5 and complete + assert web["asked"] == [URL] + + +def test_a_page_that_cannot_be_read_fails_the_board(web: dict[str, Any]) -> None: + web["answers"][URL] = b"Job Board Not Found" + with httpx.Client() as client, pytest.raises(ValueError): + ats._fetch_rows(client, "deel:klarna") + + +# ------------------------------------------------------- slugs and hosts + +def test_the_module_is_registered_as_a_feed_host() -> None: + assert atshosts.HOSTS["deel"] is deel and ats.HOSTS["deel"] == "Deel" + assert deel.NAME == "deel" and deel.KEEP_CASE + assert not hasattr(deel, "page") and not hasattr(deel, "detail") + + +def test_the_urls_and_the_robots_page_are_the_boards_own() -> None: + assert deel.careers_url("klarna") == URL + assert deel.feed_url("klarna") == URL + assert deel.robots_page("klarna") == URL + assert deel.allowed("klarna")("jobs.deel.com") and deel.allowed("klarna")("JOBS.DEEL.COM") + for host in ("deel.com", "api.deel.com", "jobs.deel.com.evil.example", "evil.example"): + assert not deel.allowed("klarna")(host) + + +@pytest.mark.parametrize("slug", ["klarna", "Klarna", "acme-2", "a_b", "9lives", "localhost"]) +def test_a_slug_is_one_path_segment(slug: str) -> None: + """`localhost` is only a path here: the host of every URL is fixed.""" + assert ats.is_known_target(f"deel:{slug}") + + +@pytest.mark.parametrize("slug", [ + "", "a@b", "a#b", "a?b", "..", ".", "a%2fb", "a/b", "a:443", "a b", "acme\n", " acme", + "127.0.0.1", "a.b", "_next", "-x", "a\\b", "évil", +]) +def test_a_hostile_slug_is_refused(slug: str) -> None: + assert not deel.slug_ok(slug) + assert not ats.is_known_target(f"deel:{slug}") + + +def test_no_slug_reaches_another_host() -> None: + """The host is fixed in every URL the module builds; a slug only ever + lands in the path.""" + for slug in ("klarna", "Klarna", "a-b_c"): + for url in (deel.careers_url(slug), deel.feed_url(slug), deel.robots_page(slug)): + assert httpx.URL(url).host == "jobs.deel.com" + + +def test_discovery_finds_the_slug_in_a_career_site_url() -> None: + page = ('All open positions' + 'Role' + 'Beta' + '' + 'List') + assert [(h, s) for h, s, _ in boards_in(page)] == [ + ("deel", "klarna"), ("deel", "acme"), ("deel", "Beta")]