diff --git a/plugins/atshosts/jobylon.py b/plugins/atshosts/jobylon.py new file mode 100644 index 0000000..64ba2b0 --- /dev/null +++ b/plugins/atshosts/jobylon.py @@ -0,0 +1,216 @@ +"""Jobylon: `jobylon:`, the whole board in one HTML page, the +description on each role's own page. + +Written from Einride's board (company 3143) and Vipps MobilePay's (1056, no +open roles) on 2026-09-29. Jobylon's own JSON and XML feeds are addressed by a +uuid the company sets and holds, and none is public for these two, so the +board is read from the widget's embed page, `cdn.jobylon.com/jobs/companies/ +/embed/v2/`, which assigns `JBL.embed_v2['jobs'] = [...]` as a JavaScript +literal with every open role (33 of 33 at Einride; `page_size` in the URL +the widget is embedded with only sets its client-side page). The literal has +unquoted keys and single-quoted strings, which `_json` reads. It carries no +description, so `detail` reads the role's page on emp.jobylon.com, whose +JSON-LD `JobPosting` has the description, the exact date and every location. +A slug Jobylon does not know answers 404. +""" + +from __future__ import annotations + +import html +import json +import re +from collections.abc import Callable +from typing import Any + +from desk.sources import strip_html + +from ._shared import Get, Row, parse_dt + +NAME = "jobylon" +LABEL = "Jobylon" +PATTERNS = ( + r"(?:emp|cdn)\.jobylon\.com/(?:jobs/)?companies/(\d+)", + # A company's own site says which board it embeds (careers.einride.tech + # sets `jbl_company_id = 3143`). + r"jbl_company_id[\"']?\s*[=:]\s*[\"']?(\d+)", +) + +_CDN = "cdn.jobylon.com" +_EMP = "emp.jobylon.com" + +#: A company id, matched whole. The host names are fixed, so a slug can only +#: ever land in a path. +_SLUG = re.compile(r"[0-9]{1,12}") + +#: A role's page on the site, as the embed names it (`/jobs/385191-einride- +#: financial-controller/`): the only kind of link the module builds a URL +#: from. +_ROLE_PATH = re.compile(r"/jobs/[0-9]+(?:-[A-Za-z0-9_-]*)?/?") + +_ASSIGNMENT = "JBL.embed_v2['jobs'] = " +_KEY = re.compile(r"[A-Za-z_][A-Za-z0-9_]*") +_TRAILING_COMMA = re.compile(r",(\s*[\]}])") + +#: Jobylon's `workspace` -> `store._remote`'s words. +_REMOTE = {"remote": "remote", "hybrid": "hybrid", "on-site": "onsite", "onsite": "onsite"} + +#: The English month names, by their first three letters, in the spellings +#: `published_date` uses ("Sept. 25, 2026"). +_MONTHS = {m: i for i, m in enumerate( + ("jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec"), 1)} +_PUBLISHED = re.compile(r"([A-Za-z]{3})[A-Za-z]*\.? (\d{1,2}), (\d{4})") + + +def slug_ok(slug: str) -> bool: + return bool(_SLUG.fullmatch(slug)) + + +def allowed(slug: str) -> Callable[[str], bool]: + return lambda host: host.lower() in (_CDN, _EMP) + + +def careers_url(slug: str) -> str: + """Jobylon answers this with a redirect to `/companies/-/`.""" + return f"https://{_EMP}/companies/{slug}/" + + +def feed_url(slug: str) -> str: + return f"https://{_CDN}/jobs/companies/{slug}/embed/v2/" + + +def robots_page(slug: str) -> str: + """The page the board is read from. Both hosts serve the same robots.txt + (compared on 2026-09-29), so the roles' pages on emp.jobylon.com are + covered by the check on this one.""" + return feed_url(slug) + + +def _json(text: str) -> Any: + """A JavaScript array or object literal as JSON: keys quoted, strings + single-quoted with `\\uXXXX` escapes, trailing commas.""" + out: list[str] = [] + i, n = 0, len(text) + while i < n: + char = text[i] + if char in "'\"": + i += 1 + chars: list[str] = [] + while i < n and text[i] != char: + if text[i] == "\\": + i += 1 + if text[i] == "u": + chars.append(chr(int(text[i + 1:i + 5], 16))) + i += 4 + elif text[i] == "x": + chars.append(chr(int(text[i + 1:i + 3], 16))) + i += 2 + else: + chars.append({"n": "\n", "t": "\t", "r": "\r"}.get(text[i], text[i])) + else: + chars.append(text[i]) + i += 1 + i += 1 + out.append(json.dumps("".join(chars))) + elif word := _KEY.match(text, i): + i = word.end() + colon = re.compile(r"\s*:").match(text, i) + out.append(json.dumps(word.group()) if colon else word.group()) + else: + out.append(char) + i += 1 + return json.loads(_TRAILING_COMMA.sub(r"\1", "".join(out))) + + +def _literal(body: bytes) -> str: + """The text of the jobs array: from its `[` to the matching `]`, quotes + respected.""" + text = body.decode("utf-8", "replace") + start = text.find(_ASSIGNMENT) + if start < 0: + raise ValueError("not a Jobylon board: no JBL.embed_v2['jobs'] in the page") + start += len(_ASSIGNMENT) + depth, i, quote = 0, start, "" + while i < len(text): + char = text[i] + if quote: + if char == "\\": + i += 1 + elif char == quote: + quote = "" + elif char in "'\"": + quote = char + elif char == "[": + depth += 1 + elif char == "]": + depth -= 1 + if depth == 0: + return text[start:i + 1] + i += 1 + raise ValueError("Jobylon's jobs array does not end") + + +def _published(value: Any) -> Any: + match = _PUBLISHED.fullmatch(str(value or "").strip()) + month = _MONTHS.get(match.group(1).lower()) if match else None + if not match or not month: + return None + return parse_dt(f"{match.group(3)}-{month:02d}-{int(match.group(2)):02d}") + + +def parse(body: bytes, slug: str) -> list[Row]: + """The embed page -> one row per role. `posted` is the day `published_date` + names, which `detail` replaces with the exact time. `location` joins the + names the embed lists, which can be fewer than the role's places + ("Houston +3 more" lists three); `detail` has them all. The embed states + no pay. `department` is the role's `function`, unless it is the + placeholder "All functions".""" + out = [] + for j in _json(_literal(body)): + title = (j.get("title") or "").strip() + job_id = str(j.get("id") or "").strip() + path = str(j.get("url") or "").strip() + if not title or not job_id or not _ROLE_PATH.fullmatch(path): + continue + function = str(j.get("function") or "").strip() + out.append({ + "source_id": job_id, + "title": title, + "location": "; ".join(dict.fromkeys( + str(name).strip() for name in j.get("locations") or [] if str(name).strip())), + "url": f"https://{_EMP}{path}", + "posted": _published(j.get("published_date")), + "department": "" if function == "All functions" else function, + "description": "", + "remote": _REMOTE.get(str(j.get("workspace") or "").strip().lower()), + "salary_text": "", # the embed states no pay + }) + return out + + +def detail(get: Get, slug: str, row: Row) -> dict[str, Any]: + """The role's own page: its JSON-LD `JobPosting` has the description, the + time it was posted and each place as "Houston, TX, USA".""" + page = get(row["url"], headers={"Accept": "text/html"}).decode("utf-8", "replace") + for block in re.findall(r'', + page, re.S): + try: + data = json.loads(block, strict=False) + except ValueError: + continue + if isinstance(data, dict) and data.get("@type") == "JobPosting": + break + else: + raise ValueError("no JobPosting on the role's page") + places = [] + for place in data.get("jobLocation") or []: + address = (place or {}).get("address") or {} + name = str(address.get("streetAddress") or address.get("addressLocality") or "").strip() + if name and name not in places: + places.append(name) + found: dict[str, Any] = { + "description": html.unescape(strip_html(data.get("description") or "")), + "posted": parse_dt(data.get("datePosted")), + } + if places: + found["location"] = "; ".join(places) + return found diff --git a/tests/fixtures/ats/jobylon-empty.html b/tests/fixtures/ats/jobylon-empty.html new file mode 100644 index 0000000..fdd01a5 --- /dev/null +++ b/tests/fixtures/ats/jobylon-empty.html @@ -0,0 +1,257 @@ + + + + + + + + Job Widget + + + + + + + + + + + + + + + + +
+ +
+ +
+ + + + + +
+
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+ +
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
    +
  • +« +
  • +
  • + +
  • +
  • +» +
  • +
+
+ + + + + + + diff --git a/tests/fixtures/ats/jobylon-role-2.html b/tests/fixtures/ats/jobylon-role-2.html new file mode 100644 index 0000000..d1da1cb --- /dev/null +++ b/tests/fixtures/ats/jobylon-role-2.html @@ -0,0 +1,55 @@ + +CDL Driver Recruiter - Einride | Jobylon + + diff --git a/tests/fixtures/ats/jobylon-role.html b/tests/fixtures/ats/jobylon-role.html new file mode 100644 index 0000000..a03d447 --- /dev/null +++ b/tests/fixtures/ats/jobylon-role.html @@ -0,0 +1,28 @@ + +Financial Controller - Einride | Jobylon + + diff --git a/tests/fixtures/ats/jobylon.html b/tests/fixtures/ats/jobylon.html new file mode 100644 index 0000000..3e4c395 --- /dev/null +++ b/tests/fixtures/ats/jobylon.html @@ -0,0 +1,939 @@ + + + + + + + + Job Widget + + + + + + + + + + + + + + + + +
+ +
+ +
+ + + + + +
+
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+ +
+ +
+
+ +
+
+ +
+
+ +
+
+ +
+
    +
  • +« +
  • +
  • + +
  • +
  • +» +
  • +
+
+ + + + + + + diff --git a/tests/test_ats_jobylon.py b/tests/test_ats_jobylon.py new file mode 100644 index 0000000..0133fb2 --- /dev/null +++ b/tests/test_ats_jobylon.py @@ -0,0 +1,199 @@ +"""The Jobylon host module against a saved embed page (Einride's board, +trimmed to five roles, and Vipps MobilePay's empty one) and two saved role +pages, 2026-09-29. No network.""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import Any + +import httpx +import pytest + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + +from app.jobs.discover import boards_in # noqa: E402 +from plugins import ats, atshosts # noqa: E402 +from plugins.atshosts import jobylon # noqa: E402 + +FIXTURES = ROOT / "tests" / "fixtures" / "ats" +BOARD = (FIXTURES / "jobylon.html").read_bytes() +EMPTY = (FIXTURES / "jobylon-empty.html").read_bytes() +FEED = "https://cdn.jobylon.com/jobs/companies/3143/embed/v2/" +ROLE = "https://emp.jobylon.com/jobs/385191-einride-financial-controller/" +ROLE_2 = "https://emp.jobylon.com/jobs/384238-einride-cdl-driver-recruiter/" + + +def test_rows_come_from_the_embeds_literal() -> None: + rows = jobylon.parse(BOARD, "3143") + assert [r["source_id"] for r in rows] == ["385191", "384238", "382124", "380347", "382860"] + first = rows[0] + assert first["title"] == "Financial Controller" + assert first["url"] == ROLE + assert first["location"] == "Stockholm" + assert first["department"] == "Accounting & Finance" # `&` read as `&` + assert first["remote"] == "hybrid" + assert first["description"] == "" and first["salary_text"] == "" + # A day, with no time of day: the embed says "Sept. 25, 2026". + assert first["posted"] is not None and first["posted"].isoformat() == \ + "2026-09-25T00:00:00+00:00" + + +def test_places_department_and_remote_follow_the_embeds_words() -> None: + rows = {r["source_id"]: r for r in jobylon.parse(BOARD, "3143")} + assert rows["384238"]["location"] == "Houston; Dallas; Atlanta" + assert rows["384238"]["department"] == "" # "All functions" names none + assert rows["382124"]["remote"] == "remote" + assert rows["382860"]["remote"] == "onsite" + assert rows["380347"]["remote"] is None # a role with no workspace + + +def test_an_empty_board_is_read_as_no_roles_and_another_page_is_not_a_board() -> None: + assert jobylon.parse(EMPTY, "1056") == [] + with pytest.raises(ValueError, match="not a Jobylon board"): + jobylon.parse(b"Not found", "1") + + +def test_the_literal_reader_takes_quotes_escapes_and_trailing_commas() -> None: + assert jobylon._json("[{id: 'a\\u0026b', 'x-y': true, list: ['it\\'s',], n: null,},]") == \ + [{"id": "a&b", "x-y": True, "list": ["it's"], "n": None}] + body = b"JBL.embed_v2['jobs'] = [{title: 'A ] B', url: '/jobs/1-a/', id: '1'}];\n" + assert [r["title"] for r in jobylon.parse(body, "9")] == ["A ] B"] + + +def test_a_role_whose_link_is_not_a_role_page_is_skipped() -> None: + body = (b"JBL.embed_v2['jobs'] = [{id: '1', title: 'A', url: 'https://evil.example/jobs/1/'}," + b"{id: '2', title: 'B', url: '//evil.example/jobs/2/'}," + b"{id: '3', title: 'C', url: '/companies/3/'}," + b"{id: '4', title: 'D', url: '/jobs/4-d/'}];\n") + assert [r["source_id"] for r in jobylon.parse(body, "9")] == ["4"] + + +# ---------------------------------------------------------------- detail + +def _get(pages: dict[str, bytes], asked: list[str] | None = None) -> Any: + def get(url: str, *, method: str = "GET", payload: Any = None, + headers: dict[str, str] | None = None) -> bytes: + if asked is not None: + asked.append(url) + return pages[url] + return get + + +def test_detail_reads_the_role_pages_json_ld() -> None: + rows = {r["source_id"]: r for r in jobylon.parse(BOARD, "3143")} + asked: list[str] = [] + pages = {ROLE: (FIXTURES / "jobylon-role.html").read_bytes(), + ROLE_2: (FIXTURES / "jobylon-role-2.html").read_bytes()} + first = jobylon.detail(_get(pages, asked), "3143", rows["385191"]) + assert asked == [ROLE] + assert set(first) <= set(ats.DETAIL_FIELDS) + assert first["description"].startswith("Want to be part of transforming road freight?") + assert "<" not in first["description"] and "world's leading" in first["description"] + assert first["posted"] is not None and first["posted"].isoformat() == \ + "2026-09-25T13:59:14+00:00" + assert first["location"] == "Stockholm, Sweden" + # All four places, where the embed named three. + second = jobylon.detail(_get(pages), "3143", rows["384238"]) + assert second["location"] == "Houston, TX, USA; Dallas, TX, USA; New Jersey, USA; Atlanta, GA, USA" + + +def test_detail_without_a_job_posting_fails_the_read() -> None: + row = jobylon.parse(BOARD, "3143")[0] + with pytest.raises(ValueError, match="no JobPosting"): + jobylon.detail(_get({ROLE: b"nothing"}), "3143", row) + page = (b'' + b'') + with pytest.raises(ValueError, match="no JobPosting"): + jobylon.detail(_get({ROLE: page}), "3143", row) + + +@pytest.fixture +def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: + answers: dict[str, Any] = {} + asked: list[str] = [] + + def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None, + www_twin=False): + asked.append(url) + if not ats._allowed_host(host, slug)(httpx.URL(url).host): + raise ats._Refused(url) + answer = answers[url] + if isinstance(answer, BaseException): + raise answer + return answer + + monkeypatch.setattr(ats, "_get", fake_get) + monkeypatch.setattr(ats, "PAGE_DELAY", 0) + return {"answers": answers, "asked": asked} + + +def test_the_board_is_one_request_and_each_role_is_read_once(web: dict[str, Any]) -> None: + web["answers"] |= {FEED: BOARD, ROLE: (FIXTURES / "jobylon-role.html").read_bytes(), + ROLE_2: (FIXTURES / "jobylon-role-2.html").read_bytes()} + with httpx.Client() as client: + rows, complete = ats._fetch_rows(client, "jobylon:3143") + assert len(rows) == 5 and complete and web["asked"] == [FEED] + # The three roles whose pages are not saved answer 404, which counts as read. + for row in rows: + web["answers"].setdefault(row["url"], httpx.HTTPStatusError( + "404", request=httpx.Request("GET", row["url"]), + response=httpx.Response(404, request=httpx.Request("GET", row["url"])))) + filled, done, failed = ats._read_details(client, "jobylon:3143", rows, complete, + set(), set()) + assert done == {f"jobylon:3143:{r['source_id']}" for r in rows} and not failed + assert filled[0]["description"].startswith("Want to be part") and filled[0]["location"] + assert filled[2]["description"] == "" + + +def test_an_empty_board_is_complete(web: dict[str, Any]) -> None: + web["answers"]["https://cdn.jobylon.com/jobs/companies/1056/embed/v2/"] = EMPTY + with httpx.Client() as client: + assert ats._fetch_rows(client, "jobylon:1056") == ([], True) + + +# ------------------------------------------------------- slugs and hosts + +def test_the_module_is_registered_as_a_feed_host_with_detail() -> None: + assert atshosts.HOSTS["jobylon"] is jobylon and ats.HOSTS["jobylon"] == "Jobylon" + assert jobylon.NAME == "jobylon" and not getattr(jobylon, "KEEP_CASE", False) + assert hasattr(jobylon, "detail") and not hasattr(jobylon, "page") + + +def test_the_urls_and_the_robots_page() -> None: + assert jobylon.feed_url("3143") == FEED + assert jobylon.careers_url("3143") == "https://emp.jobylon.com/companies/3143/" + assert jobylon.robots_page("3143") == FEED + allow = jobylon.allowed("3143") + assert allow("cdn.jobylon.com") and allow("emp.jobylon.com") and allow("EMP.JOBYLON.COM") + for host in ("jobylon.com", "feed.jobylon.com", "emp.jobylon.com.evil.example", + "evil.example", "localhost"): + assert not allow(host) + + +@pytest.mark.parametrize("slug", ["3143", "1056", "1"]) +def test_a_company_id_is_a_slug(slug: str) -> None: + assert ats.is_known_target(f"jobylon:{slug}") + + +@pytest.mark.parametrize("slug", [ + "", "acme", "3143-einride", "31 43", "3143\n", " 3143", "3143 ", "3143/x", "3143?x", + "3143#x", "a@b", "..", "%2f", "3143%2f", "3143:443", "localhost", "127.0.0.1", + "1234567890123", "-1", "3.1", +]) +def test_a_hostile_slug_is_refused(slug: str) -> None: + assert not jobylon.slug_ok(slug) + assert not ats.is_known_target(f"jobylon:{slug}") + + +def test_discovery_finds_the_company_id() -> None: + page = ('Jobs' + '' + 'Role' + '' + "") + assert [(h, s) for h, s, _ in boards_in(page)] == [ + ("jobylon", "1056"), ("jobylon", "3143"), ("jobylon", "2222"), ("jobylon", "3333")]