Read Jobylon's boards from the embed page and each role's page
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
68a3858355
commit
a41995da8e
6 files changed
+1694
No files matched your search
@@ -0,0 +1,199 @@
|
||||
"""The Jobylon host module against a saved embed page (Einride's board,
|
||||
trimmed to five roles, and Vipps MobilePay's empty one) and two saved role
|
||||
pages, 2026-09-29. No network."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from app.jobs.discover import boards_in # noqa: E402
|
||||
from plugins import ats, atshosts # noqa: E402
|
||||
from plugins.atshosts import jobylon # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
BOARD = (FIXTURES / "jobylon.html").read_bytes()
|
||||
EMPTY = (FIXTURES / "jobylon-empty.html").read_bytes()
|
||||
FEED = "https://cdn.jobylon.com/jobs/companies/3143/embed/v2/"
|
||||
ROLE = "https://emp.jobylon.com/jobs/385191-einride-financial-controller/"
|
||||
ROLE_2 = "https://emp.jobylon.com/jobs/384238-einride-cdl-driver-recruiter/"
|
||||
|
||||
|
||||
def test_rows_come_from_the_embeds_literal() -> None:
|
||||
rows = jobylon.parse(BOARD, "3143")
|
||||
assert [r["source_id"] for r in rows] == ["385191", "384238", "382124", "380347", "382860"]
|
||||
first = rows[0]
|
||||
assert first["title"] == "Financial Controller"
|
||||
assert first["url"] == ROLE
|
||||
assert first["location"] == "Stockholm"
|
||||
assert first["department"] == "Accounting & Finance" # `&` read as `&`
|
||||
assert first["remote"] == "hybrid"
|
||||
assert first["description"] == "" and first["salary_text"] == ""
|
||||
# A day, with no time of day: the embed says "Sept. 25, 2026".
|
||||
assert first["posted"] is not None and first["posted"].isoformat() == \
|
||||
"2026-09-25T00:00:00+00:00"
|
||||
|
||||
|
||||
def test_places_department_and_remote_follow_the_embeds_words() -> None:
|
||||
rows = {r["source_id"]: r for r in jobylon.parse(BOARD, "3143")}
|
||||
assert rows["384238"]["location"] == "Houston; Dallas; Atlanta"
|
||||
assert rows["384238"]["department"] == "" # "All functions" names none
|
||||
assert rows["382124"]["remote"] == "remote"
|
||||
assert rows["382860"]["remote"] == "onsite"
|
||||
assert rows["380347"]["remote"] is None # a role with no workspace
|
||||
|
||||
|
||||
def test_an_empty_board_is_read_as_no_roles_and_another_page_is_not_a_board() -> None:
|
||||
assert jobylon.parse(EMPTY, "1056") == []
|
||||
with pytest.raises(ValueError, match="not a Jobylon board"):
|
||||
jobylon.parse(b"<html>Not found</html>", "1")
|
||||
|
||||
|
||||
def test_the_literal_reader_takes_quotes_escapes_and_trailing_commas() -> None:
|
||||
assert jobylon._json("[{id: 'a\\u0026b', 'x-y': true, list: ['it\\'s',], n: null,},]") == \
|
||||
[{"id": "a&b", "x-y": True, "list": ["it's"], "n": None}]
|
||||
body = b"JBL.embed_v2['jobs'] = [{title: 'A ] B', url: '/jobs/1-a/', id: '1'}];\n"
|
||||
assert [r["title"] for r in jobylon.parse(body, "9")] == ["A ] B"]
|
||||
|
||||
|
||||
def test_a_role_whose_link_is_not_a_role_page_is_skipped() -> None:
|
||||
body = (b"JBL.embed_v2['jobs'] = [{id: '1', title: 'A', url: 'https://evil.example/jobs/1/'},"
|
||||
b"{id: '2', title: 'B', url: '//evil.example/jobs/2/'},"
|
||||
b"{id: '3', title: 'C', url: '/companies/3/'},"
|
||||
b"{id: '4', title: 'D', url: '/jobs/4-d/'}];\n")
|
||||
assert [r["source_id"] for r in jobylon.parse(body, "9")] == ["4"]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- detail
|
||||
|
||||
def _get(pages: dict[str, bytes], asked: list[str] | None = None) -> Any:
|
||||
def get(url: str, *, method: str = "GET", payload: Any = None,
|
||||
headers: dict[str, str] | None = None) -> bytes:
|
||||
if asked is not None:
|
||||
asked.append(url)
|
||||
return pages[url]
|
||||
return get
|
||||
|
||||
|
||||
def test_detail_reads_the_role_pages_json_ld() -> None:
|
||||
rows = {r["source_id"]: r for r in jobylon.parse(BOARD, "3143")}
|
||||
asked: list[str] = []
|
||||
pages = {ROLE: (FIXTURES / "jobylon-role.html").read_bytes(),
|
||||
ROLE_2: (FIXTURES / "jobylon-role-2.html").read_bytes()}
|
||||
first = jobylon.detail(_get(pages, asked), "3143", rows["385191"])
|
||||
assert asked == [ROLE]
|
||||
assert set(first) <= set(ats.DETAIL_FIELDS)
|
||||
assert first["description"].startswith("Want to be part of transforming road freight?")
|
||||
assert "<" not in first["description"] and "world's leading" in first["description"]
|
||||
assert first["posted"] is not None and first["posted"].isoformat() == \
|
||||
"2026-09-25T13:59:14+00:00"
|
||||
assert first["location"] == "Stockholm, Sweden"
|
||||
# All four places, where the embed named three.
|
||||
second = jobylon.detail(_get(pages), "3143", rows["384238"])
|
||||
assert second["location"] == "Houston, TX, USA; Dallas, TX, USA; New Jersey, USA; Atlanta, GA, USA"
|
||||
|
||||
|
||||
def test_detail_without_a_job_posting_fails_the_read() -> None:
|
||||
row = jobylon.parse(BOARD, "3143")[0]
|
||||
with pytest.raises(ValueError, match="no JobPosting"):
|
||||
jobylon.detail(_get({ROLE: b"<html>nothing</html>"}), "3143", row)
|
||||
page = (b'<script type="application/ld+json">{"@type": "BreadcrumbList"}</script>'
|
||||
b'<script type="application/ld+json">not json</script>')
|
||||
with pytest.raises(ValueError, match="no JobPosting"):
|
||||
jobylon.detail(_get({ROLE: page}), "3143", row)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]:
|
||||
answers: dict[str, Any] = {}
|
||||
asked: list[str] = []
|
||||
|
||||
def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None,
|
||||
www_twin=False):
|
||||
asked.append(url)
|
||||
if not ats._allowed_host(host, slug)(httpx.URL(url).host):
|
||||
raise ats._Refused(url)
|
||||
answer = answers[url]
|
||||
if isinstance(answer, BaseException):
|
||||
raise answer
|
||||
return answer
|
||||
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
|
||||
return {"answers": answers, "asked": asked}
|
||||
|
||||
|
||||
def test_the_board_is_one_request_and_each_role_is_read_once(web: dict[str, Any]) -> None:
|
||||
web["answers"] |= {FEED: BOARD, ROLE: (FIXTURES / "jobylon-role.html").read_bytes(),
|
||||
ROLE_2: (FIXTURES / "jobylon-role-2.html").read_bytes()}
|
||||
with httpx.Client() as client:
|
||||
rows, complete = ats._fetch_rows(client, "jobylon:3143")
|
||||
assert len(rows) == 5 and complete and web["asked"] == [FEED]
|
||||
# The three roles whose pages are not saved answer 404, which counts as read.
|
||||
for row in rows:
|
||||
web["answers"].setdefault(row["url"], httpx.HTTPStatusError(
|
||||
"404", request=httpx.Request("GET", row["url"]),
|
||||
response=httpx.Response(404, request=httpx.Request("GET", row["url"]))))
|
||||
filled, done, failed = ats._read_details(client, "jobylon:3143", rows, complete,
|
||||
set(), set())
|
||||
assert done == {f"jobylon:3143:{r['source_id']}" for r in rows} and not failed
|
||||
assert filled[0]["description"].startswith("Want to be part") and filled[0]["location"]
|
||||
assert filled[2]["description"] == ""
|
||||
|
||||
|
||||
def test_an_empty_board_is_complete(web: dict[str, Any]) -> None:
|
||||
web["answers"]["https://cdn.jobylon.com/jobs/companies/1056/embed/v2/"] = EMPTY
|
||||
with httpx.Client() as client:
|
||||
assert ats._fetch_rows(client, "jobylon:1056") == ([], True)
|
||||
|
||||
|
||||
# ------------------------------------------------------- slugs and hosts
|
||||
|
||||
def test_the_module_is_registered_as_a_feed_host_with_detail() -> None:
|
||||
assert atshosts.HOSTS["jobylon"] is jobylon and ats.HOSTS["jobylon"] == "Jobylon"
|
||||
assert jobylon.NAME == "jobylon" and not getattr(jobylon, "KEEP_CASE", False)
|
||||
assert hasattr(jobylon, "detail") and not hasattr(jobylon, "page")
|
||||
|
||||
|
||||
def test_the_urls_and_the_robots_page() -> None:
|
||||
assert jobylon.feed_url("3143") == FEED
|
||||
assert jobylon.careers_url("3143") == "https://emp.jobylon.com/companies/3143/"
|
||||
assert jobylon.robots_page("3143") == FEED
|
||||
allow = jobylon.allowed("3143")
|
||||
assert allow("cdn.jobylon.com") and allow("emp.jobylon.com") and allow("EMP.JOBYLON.COM")
|
||||
for host in ("jobylon.com", "feed.jobylon.com", "emp.jobylon.com.evil.example",
|
||||
"evil.example", "localhost"):
|
||||
assert not allow(host)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("slug", ["3143", "1056", "1"])
|
||||
def test_a_company_id_is_a_slug(slug: str) -> None:
|
||||
assert ats.is_known_target(f"jobylon:{slug}")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("slug", [
|
||||
"", "acme", "3143-einride", "31 43", "3143\n", " 3143", "3143 ", "3143/x", "3143?x",
|
||||
"3143#x", "a@b", "..", "%2f", "3143%2f", "3143:443", "localhost", "127.0.0.1",
|
||||
"1234567890123", "-1", "3.1",
|
||||
])
|
||||
def test_a_hostile_slug_is_refused(slug: str) -> None:
|
||||
assert not jobylon.slug_ok(slug)
|
||||
assert not ats.is_known_target(f"jobylon:{slug}")
|
||||
|
||||
|
||||
def test_discovery_finds_the_company_id() -> None:
|
||||
page = ('<a href="https://emp.jobylon.com/companies/1056-vipps-mobilepay/">Jobs</a>'
|
||||
'<script src="https://cdn.jobylon.com/jobs/companies/3143/embed/v2/?target=x"></script>'
|
||||
'<a href="https://emp.jobylon.com/jobs/385191-einride-financial-controller/">Role</a>'
|
||||
'<script>var jbl_company_id = 2222;</script>'
|
||||
"<script>jbl_company_id: '3333'</script>")
|
||||
assert [(h, s) for h, s, _ in boards_in(page)] == [
|
||||
("jobylon", "1056"), ("jobylon", "3143"), ("jobylon", "2222"), ("jobylon", "3333")]
|
||||
Reference in new issue
Block a user