Every import, test and live doc follows the new path; behaviour is unchanged. tests/test_atshosts.py becomes tests/test_readers.py. plan-readers.md and review-ingest.md keep the old name, since they describe the state before. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
200 lines
9.0 KiB
Python
200 lines
9.0 KiB
Python
"""The Jobylon host module against a saved embed page (Einride's board,
|
|
trimmed to five roles, and Vipps MobilePay's empty one) and two saved role
|
|
pages, 2026-09-29. No network."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from app.jobs.discover import boards_in # noqa: E402
|
|
from plugins import ats, readers # noqa: E402
|
|
from plugins.readers import jobylon # noqa: E402
|
|
|
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
|
BOARD = (FIXTURES / "jobylon.html").read_bytes()
|
|
EMPTY = (FIXTURES / "jobylon-empty.html").read_bytes()
|
|
FEED = "https://cdn.jobylon.com/jobs/companies/3143/embed/v2/"
|
|
ROLE = "https://emp.jobylon.com/jobs/385191-einride-financial-controller/"
|
|
ROLE_2 = "https://emp.jobylon.com/jobs/384238-einride-cdl-driver-recruiter/"
|
|
|
|
|
|
def test_rows_come_from_the_embeds_literal() -> None:
|
|
rows = jobylon.parse(BOARD, "3143")
|
|
assert [r["source_id"] for r in rows] == ["385191", "384238", "382124", "380347", "382860"]
|
|
first = rows[0]
|
|
assert first["title"] == "Financial Controller"
|
|
assert first["url"] == ROLE
|
|
assert first["location"] == "Stockholm"
|
|
assert first["department"] == "Accounting & Finance" # `&` read as `&`
|
|
assert first["remote"] == "hybrid"
|
|
assert first["description"] == "" and first["salary_text"] == ""
|
|
# A day, with no time of day: the embed says "Sept. 25, 2026".
|
|
assert first["posted"] is not None and first["posted"].isoformat() == \
|
|
"2026-09-25T00:00:00+00:00"
|
|
|
|
|
|
def test_places_department_and_remote_follow_the_embeds_words() -> None:
|
|
rows = {r["source_id"]: r for r in jobylon.parse(BOARD, "3143")}
|
|
assert rows["384238"]["location"] == "Houston; Dallas; Atlanta"
|
|
assert rows["384238"]["department"] == "" # "All functions" names none
|
|
assert rows["382124"]["remote"] == "remote"
|
|
assert rows["382860"]["remote"] == "onsite"
|
|
assert rows["380347"]["remote"] is None # a role with no workspace
|
|
|
|
|
|
def test_an_empty_board_is_read_as_no_roles_and_another_page_is_not_a_board() -> None:
|
|
assert jobylon.parse(EMPTY, "1056") == []
|
|
with pytest.raises(ValueError, match="not a Jobylon board"):
|
|
jobylon.parse(b"<html>Not found</html>", "1")
|
|
|
|
|
|
def test_the_literal_reader_takes_quotes_escapes_and_trailing_commas() -> None:
|
|
assert jobylon._json("[{id: 'a\\u0026b', 'x-y': true, list: ['it\\'s',], n: null,},]") == \
|
|
[{"id": "a&b", "x-y": True, "list": ["it's"], "n": None}]
|
|
body = b"JBL.embed_v2['jobs'] = [{title: 'A ] B', url: '/jobs/1-a/', id: '1'}];\n"
|
|
assert [r["title"] for r in jobylon.parse(body, "9")] == ["A ] B"]
|
|
|
|
|
|
def test_a_role_whose_link_is_not_a_role_page_is_skipped() -> None:
|
|
body = (b"JBL.embed_v2['jobs'] = [{id: '1', title: 'A', url: 'https://evil.example/jobs/1/'},"
|
|
b"{id: '2', title: 'B', url: '//evil.example/jobs/2/'},"
|
|
b"{id: '3', title: 'C', url: '/companies/3/'},"
|
|
b"{id: '4', title: 'D', url: '/jobs/4-d/'}];\n")
|
|
assert [r["source_id"] for r in jobylon.parse(body, "9")] == ["4"]
|
|
|
|
|
|
# ---------------------------------------------------------------- detail
|
|
|
|
def _get(pages: dict[str, bytes], asked: list[str] | None = None) -> Any:
|
|
def get(url: str, *, method: str = "GET", payload: Any = None,
|
|
headers: dict[str, str] | None = None) -> bytes:
|
|
if asked is not None:
|
|
asked.append(url)
|
|
return pages[url]
|
|
return get
|
|
|
|
|
|
def test_detail_reads_the_role_pages_json_ld() -> None:
|
|
rows = {r["source_id"]: r for r in jobylon.parse(BOARD, "3143")}
|
|
asked: list[str] = []
|
|
pages = {ROLE: (FIXTURES / "jobylon-role.html").read_bytes(),
|
|
ROLE_2: (FIXTURES / "jobylon-role-2.html").read_bytes()}
|
|
first = jobylon.detail(_get(pages, asked), "3143", rows["385191"])
|
|
assert asked == [ROLE]
|
|
assert set(first) <= set(ats.DETAIL_FIELDS)
|
|
assert first["description"].startswith("Want to be part of transforming road freight?")
|
|
assert "<" not in first["description"] and "world's leading" in first["description"]
|
|
assert first["posted"] is not None and first["posted"].isoformat() == \
|
|
"2026-09-25T13:59:14+00:00"
|
|
assert first["location"] == "Stockholm, Sweden"
|
|
# All four places, where the embed named three.
|
|
second = jobylon.detail(_get(pages), "3143", rows["384238"])
|
|
assert second["location"] == "Houston, TX, USA; Dallas, TX, USA; New Jersey, USA; Atlanta, GA, USA"
|
|
|
|
|
|
def test_detail_without_a_job_posting_fails_the_read() -> None:
|
|
row = jobylon.parse(BOARD, "3143")[0]
|
|
with pytest.raises(ValueError, match="no JobPosting"):
|
|
jobylon.detail(_get({ROLE: b"<html>nothing</html>"}), "3143", row)
|
|
page = (b'<script type="application/ld+json">{"@type": "BreadcrumbList"}</script>'
|
|
b'<script type="application/ld+json">not json</script>')
|
|
with pytest.raises(ValueError, match="no JobPosting"):
|
|
jobylon.detail(_get({ROLE: page}), "3143", row)
|
|
|
|
|
|
@pytest.fixture
|
|
def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]:
|
|
answers: dict[str, Any] = {}
|
|
asked: list[str] = []
|
|
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None,
|
|
www_twin=False):
|
|
asked.append(url)
|
|
if not ats._allowed_host(host, slug)(httpx.URL(url).host):
|
|
raise ats._Refused(url)
|
|
answer = answers[url]
|
|
if isinstance(answer, BaseException):
|
|
raise answer
|
|
return answer
|
|
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
|
|
return {"answers": answers, "asked": asked}
|
|
|
|
|
|
def test_the_board_is_one_request_and_each_role_is_read_once(web: dict[str, Any]) -> None:
|
|
web["answers"] |= {FEED: BOARD, ROLE: (FIXTURES / "jobylon-role.html").read_bytes(),
|
|
ROLE_2: (FIXTURES / "jobylon-role-2.html").read_bytes()}
|
|
with httpx.Client() as client:
|
|
rows, complete = ats._fetch_rows(client, "jobylon:3143")
|
|
assert len(rows) == 5 and complete and web["asked"] == [FEED]
|
|
# The three roles whose pages are not saved answer 404, which counts as read.
|
|
for row in rows:
|
|
web["answers"].setdefault(row["url"], httpx.HTTPStatusError(
|
|
"404", request=httpx.Request("GET", row["url"]),
|
|
response=httpx.Response(404, request=httpx.Request("GET", row["url"]))))
|
|
filled, done, failed = ats._read_details(client, "jobylon:3143", rows, complete,
|
|
set(), set())
|
|
assert done == {f"jobylon:3143:{r['source_id']}" for r in rows} and not failed
|
|
assert filled[0]["description"].startswith("Want to be part") and filled[0]["location"]
|
|
assert filled[2]["description"] == ""
|
|
|
|
|
|
def test_an_empty_board_is_complete(web: dict[str, Any]) -> None:
|
|
web["answers"]["https://cdn.jobylon.com/jobs/companies/1056/embed/v2/"] = EMPTY
|
|
with httpx.Client() as client:
|
|
assert ats._fetch_rows(client, "jobylon:1056") == ([], True)
|
|
|
|
|
|
# ------------------------------------------------------- slugs and hosts
|
|
|
|
def test_the_module_is_registered_as_a_feed_host_with_detail() -> None:
|
|
assert readers.HOSTS["jobylon"] is jobylon and ats.HOSTS["jobylon"] == "Jobylon"
|
|
assert jobylon.NAME == "jobylon" and not getattr(jobylon, "KEEP_CASE", False)
|
|
assert hasattr(jobylon, "detail") and not hasattr(jobylon, "page")
|
|
|
|
|
|
def test_the_urls_and_the_robots_page() -> None:
|
|
assert jobylon.feed_url("3143") == FEED
|
|
assert jobylon.careers_url("3143") == "https://emp.jobylon.com/companies/3143/"
|
|
assert jobylon.robots_page("3143") == FEED
|
|
allow = jobylon.allowed("3143")
|
|
assert allow("cdn.jobylon.com") and allow("emp.jobylon.com") and allow("EMP.JOBYLON.COM")
|
|
for host in ("jobylon.com", "feed.jobylon.com", "emp.jobylon.com.evil.example",
|
|
"evil.example", "localhost"):
|
|
assert not allow(host)
|
|
|
|
|
|
@pytest.mark.parametrize("slug", ["3143", "1056", "1"])
|
|
def test_a_company_id_is_a_slug(slug: str) -> None:
|
|
assert ats.is_known_target(f"jobylon:{slug}")
|
|
|
|
|
|
@pytest.mark.parametrize("slug", [
|
|
"", "acme", "3143-einride", "31 43", "3143\n", " 3143", "3143 ", "3143/x", "3143?x",
|
|
"3143#x", "a@b", "..", "%2f", "3143%2f", "3143:443", "localhost", "127.0.0.1",
|
|
"1234567890123", "-1", "3.1",
|
|
])
|
|
def test_a_hostile_slug_is_refused(slug: str) -> None:
|
|
assert not jobylon.slug_ok(slug)
|
|
assert not ats.is_known_target(f"jobylon:{slug}")
|
|
|
|
|
|
def test_discovery_finds_the_company_id() -> None:
|
|
page = ('<a href="https://emp.jobylon.com/companies/1056-vipps-mobilepay/">Jobs</a>'
|
|
'<script src="https://cdn.jobylon.com/jobs/companies/3143/embed/v2/?target=x"></script>'
|
|
'<a href="https://emp.jobylon.com/jobs/385191-einride-financial-controller/">Role</a>'
|
|
'<script>var jbl_company_id = 2222;</script>'
|
|
"<script>jbl_company_id: '3333'</script>")
|
|
assert [(h, s) for h, s, _ in boards_in(page)] == [
|
|
("jobylon", "1056"), ("jobylon", "3143"), ("jobylon", "2222"), ("jobylon", "3333")]
|