Files
jobboard/tests/test_ats_phenom.py
T
WolfandClaude Opus 5.5 c497fe965d Keep a description's headings, bullets and paragraphs, and draw them in the reader
The readers flattened a posting's HTML into one line with strip_html, so
sections and bullets ran together. _shared.html_text gives a line per
paragraph, '• ' before a list item and '## ' before a heading; every
reader's description goes through it, sections read per role get their
own headings, and the reader pane draws the lines as h3, ul and p.
Excerpts leave the marks out.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 21:40:12 +02:00

328 lines
15 KiB
Python

"""Phenom: the listing's rows from a saved `POST /widgets` answer (King's,
trimmed), the request body, paging and completeness against a fake `get`, the
description from a saved job page's JSON-LD (King's and Philips's, trimmed),
the slug, and discovery's patterns. No network."""
from __future__ import annotations
import json
import sys
from pathlib import Path
from typing import Any
import httpx
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app.jobs.discover import boards_in # noqa: E402
from plugins import ats # noqa: E402
from plugins.readers import phenom # noqa: E402
from plugins.readers._shared import is_web_url # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
SLUG = "careers.king.com/us/en"
IDS = ["ACMKDSUSR028168EXTERNALENUS", "ACMKDSUSR027656EXTERNALENUS",
"ACMKDSUSR028172EXTERNALENUS", "ACMKDSUSR028147EXTERNALENUS",
"ACMKDSUSR028167EXTERNALENUS"]
def _answer() -> dict[str, Any]:
return json.loads((FIXTURES / "phenom.json").read_bytes())
def _rows() -> list[ats.Row]:
return phenom.parse((FIXTURES / "phenom.json").read_bytes(), SLUG)
# ------------------------------------------------------------------- rows
def test_rows_carry_the_listing_fields() -> None:
rows = _rows()
assert [r["source_id"] for r in rows] == IDS
first = rows[0]
assert first["title"] == "Principal Economy Designer | New Games"
assert first["department"] == "Games Design & Production"
assert first["description"] == "" # the text comes with the role's own page
assert first["teaser"].startswith("Become part of our team and help shape the economic")
assert first["remote"] is None and first["salary_text"] == ""
assert all(is_web_url(r["url"]) for r in rows)
def test_posted_is_the_date_the_role_was_created_not_the_one_the_site_moves() -> None:
"""The second role was created on 15 June and is listed as posted on 28
September; Phenom refreshes `postedDate`, so the creation date is taken, and
`postedDate` only where a site sends no `dateCreated`."""
rows = _rows()
assert rows[0]["posted"].isoformat() == "2026-09-29T09:59:51.253000+00:00"
assert rows[1]["posted"].isoformat() == "2026-06-15T17:02:06.245000+00:00"
data = _answer()
del data["refineSearch"]["data"]["jobs"][1]["dateCreated"]
assert phenom.parse(json.dumps(data).encode(), SLUG)[1]["posted"].isoformat() == \
"2026-09-28T00:00:00+00:00"
def test_the_url_is_the_role_page_on_the_site_not_the_apply_system() -> None:
"""`applyUrl` points at Workday; the role's page is `/job/<jobSeqNo>/<title>`."""
assert _answer()["refineSearch"]["data"]["jobs"][0]["applyUrl"].startswith("https://xboxgaming.")
assert _rows()[0]["url"] == ("https://careers.king.com/us/en/job/ACMKDSUSR028168EXTERNALENUS"
"/Principal-Economy-Designer-New-Games")
assert _rows()[1]["url"].endswith("/Senior-Data-Visualisation-Engineer-12mth-FTC")
def test_the_locations_are_every_place_the_role_names() -> None:
rows = _rows()
assert rows[0]["location"] == ("Barcelona, Spain; London, London, United Kingdom; "
"Stockholm, Stockholm County, Sweden")
assert rows[1]["location"] == "Stockholm, Stockholm County, Sweden"
def test_every_place_is_kept_once_and_the_first_is_the_roles_own() -> None:
rows = _rows()
assert [p["text"] for p in rows[0]["places"]] == [
"Barcelona, Spain", "London, London, United Kingdom", "Stockholm, Stockholm County, Sweden"]
assert rows[2]["places"][0]["text"] == "London, London, United Kingdom"
assert all(p["remote"] is None for r in rows for p in r["places"])
def test_a_role_with_one_place_only_uses_location() -> None:
data = _answer()
job = data["refineSearch"]["data"]["jobs"][0]
job["multi_location"], job["isMultiLocation"] = [], False
assert phenom.parse(json.dumps(data).encode(), SLUG)[0]["location"] == "Barcelona, Spain"
def test_a_title_with_nothing_to_make_a_slug_of_has_a_page_by_id_alone() -> None:
data = _answer()
data["refineSearch"]["data"]["jobs"][0]["title"] = "?!"
assert phenom.parse(json.dumps(data).encode(), SLUG)[0]["url"] == \
"https://careers.king.com/us/en/job/ACMKDSUSR028168EXTERNALENUS"
def test_a_role_without_title_or_id_is_left_out() -> None:
data = _answer()
jobs = data["refineSearch"]["data"]["jobs"]
jobs[0]["title"] = ""
for key in ("jobSeqNo", "jobId", "reqId"):
jobs[1].pop(key, None)
assert [r["source_id"] for r in phenom.parse(json.dumps(data).encode(), SLUG)] == IDS[2:]
# ---------------------------------------------------------- the request
def test_the_body_names_the_locale_and_the_page_and_nothing_secret() -> None:
body = phenom._body("www.careers.philips.com/global/en", 200)
assert (body["lang"], body["country"], body["from"], body["size"]) == \
("en_global", "global", 200, 100)
assert body["ddoKey"] == "refineSearch" and body["jobs"] is True
assert phenom._body(SLUG, 0)["lang"] == "en_us"
assert not {"refNum", "pageId", "token", "key"} & set(body)
# ----------------------------------------------------------------- paging
@pytest.fixture
def no_delay(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
def _fake(monkeypatch: pytest.MonkeyPatch, total: int | None = None
) -> list[tuple[str, str, dict[str, Any] | None]]:
"""`ats._get` answering the search from the saved roles, `from` and `size`
honoured, and a page past the end empty."""
asked: list[tuple[str, str, dict[str, Any] | None]] = []
every = _answer()["refineSearch"]["data"]["jobs"]
def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None,
www_twin=False):
asked.append((url, method, payload))
assert payload is not None
data = _answer()
jobs = every[payload["from"]:payload["from"] + payload["size"]]
data["refineSearch"]["data"]["jobs"] = jobs
data["refineSearch"]["hits"] = len(jobs)
data["refineSearch"]["totalHits"] = len(every) if total is None else total
return json.dumps(data).encode()
monkeypatch.setattr(ats, "_get", fake_get)
return asked
def test_the_listing_is_posted_page_by_page_to_the_stated_total(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
asked = _fake(monkeypatch)
monkeypatch.setattr(phenom, "PAGE_SIZE", 2)
rows, complete = ats._fetch_rows(object(), f"phenom:{SLUG}") # type: ignore[arg-type]
assert [(url, method, (payload or {})["from"], (payload or {})["size"]) for url, method, payload in asked] == [
("https://careers.king.com/widgets", "POST", offset, 2) for offset in (0, 2, 4)]
assert [r["source_id"] for r in rows] == IDS
assert complete
def test_a_board_that_ends_short_of_its_total_is_incomplete(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
_fake(monkeypatch, total=7)
monkeypatch.setattr(phenom, "PAGE_SIZE", 2)
rows, complete = ats._fetch_rows(object(), f"phenom:{SLUG}") # type: ignore[arg-type]
assert (len(rows), complete) == (5, False)
def test_a_small_board_is_one_page(monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
asked = _fake(monkeypatch)
rows, complete = ats._fetch_rows(object(), f"phenom:{SLUG}") # type: ignore[arg-type]
assert len(asked) == 1
assert (len(rows), complete) == (5, True)
def test_an_empty_board_is_complete(monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
empty = {"refineSearch": {"status": 200, "hits": 0, "totalHits": 0, "data": {"jobs": []}}}
monkeypatch.setattr(ats, "_get", lambda *a, **k: json.dumps(empty).encode())
assert ats._fetch_rows(object(), f"phenom:{SLUG}") == ([], True) # type: ignore[arg-type]
def test_an_answer_that_is_not_a_search_fails_the_board(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
monkeypatch.setattr(ats, "_get", lambda *a, **k: b'{"error": "no such page"}')
with pytest.raises(ValueError, match="refineSearch"):
ats._fetch_rows(object(), f"phenom:{SLUG}") # type: ignore[arg-type]
# ----------------------------------------------------------------- detail
def _get_of(name: str, asked: list[tuple[str, dict[str, str] | None]]) -> Any:
body = (FIXTURES / name).read_bytes()
def get(url: str, *, method: str = "GET", payload: Any = None, headers: Any = None) -> bytes:
asked.append((url, headers))
return body
return get
def test_detail_reads_the_description_from_the_json_ld() -> None:
asked: list[tuple[str, dict[str, str] | None]] = []
row = _rows()[0]
got = phenom.detail(_get_of("phenom-detail.html", asked), SLUG, row)
assert asked == [(row["url"], {"Accept": "text/html"})]
assert got["description"].startswith("## Craft:\nData, Analytics & Strategy")
assert "<" not in got["description"] and "&lt;" not in got["description"]
assert "&amp;" not in got["description"]
assert got["posted"].isoformat() == "2026-09-28T00:00:00+00:00"
assert set(got) <= set(ats.DETAIL_FIELDS)
def test_detail_finds_the_job_posting_among_other_blocks_and_takes_its_date() -> None:
got = phenom.detail(_get_of("phenom-detail-2.html", []), "www.careers.philips.com/global/en",
{"url": "https://www.careers.philips.com/global/en/job/1/x"})
assert got["description"].startswith("## Job Title\nZone Business Leader Imaging Nordics")
assert got["posted"].isoformat() == "2026-09-08T00:00:00+00:00"
def test_detail_falls_back_to_the_page_state_where_there_is_no_job_posting() -> None:
"""King's pages of roles posted that day had no `JobPosting` block, only a
`WebPage` one, and the role in the state the page assigns to `phApp.ddo`."""
row = _rows()[0]
got = phenom.detail(_get_of("phenom-detail-3.html", []), SLUG, row)
assert got["description"].startswith("## Craft:\nGames Design & Production\n## Job Description:")
assert "<" not in got["description"] and "&amp;" not in got["description"]
assert got["remote"] == "On-site"
assert "posted" not in got and set(got) <= set(ats.DETAIL_FIELDS)
def test_detail_of_a_page_with_no_description_fails_so_that_it_is_asked_again() -> None:
def get(url: str, **kwargs: Any) -> bytes:
return (b'<html><script type="application/ld+json">{"@type": "BreadcrumbList"}</script>'
b'<script type="application/ld+json">{broken</script>'
b'<script>phApp.ddo = {"jobDetail": {"data": {"job": {"title": "x"}}}};</script></html>')
with pytest.raises(ValueError, match="no description"):
phenom.detail(get, SLUG, {"url": "https://careers.king.com/us/en/job/1"})
@pytest.mark.parametrize("state", ["", "phApp.ddo = {broken", 'phApp.ddo = {"jobDetail": []};',
'phApp.ddo = {"siteConfig": {}};'])
def test_a_page_state_that_is_not_a_role_is_no_description(state: str) -> None:
def get(url: str, **kwargs: Any) -> bytes:
return f"<html><script>{state}</script></html>".encode()
with pytest.raises(ValueError, match="no description"):
phenom.detail(get, SLUG, {"url": "https://careers.king.com/us/en/job/1"})
# ----------------------------------------------------------- slug and URLs
@pytest.mark.parametrize("slug", [
"careers.king.com/us/en", "www.careers.philips.com/global/en",
"careers.justeattakeaway.com/global/en", "jobs.example.co.uk/na/en", "a-b.example.de/de/de",
])
def test_a_host_and_a_locale_root_are_a_slug(slug: str) -> None:
assert phenom.slug_ok(slug) and ats.is_known_target(f"phenom:{slug}")
@pytest.mark.parametrize("slug", [
"", "careers.king.com", "careers.king.com/", "careers.king.com/us", "careers.king.com/us/",
"careers.king.com/us/en/", "careers.king.com/us/en/x", "careers.king.com//en",
"careers.king.com/us/en\n", "careers.king.com/us/en ", " careers.king.com/us/en",
"careers.king.com /us/en", "careers.king.com\n/us/en", "careers.king.com/us/en\r\n",
"careers.king.com@evil.com/us/en", "evil.com@careers.king.com/us/en",
"careers.king.com#x/us/en", "careers.king.com?x=1/us/en", "careers.king.com/us/en?x=1",
"careers.king.com/us/en#x", "careers.king.com/us/%2fen", "careers.king.com/../en",
"careers.king.com/us/..", "careers.king.com:8443/us/en", "careers.king.com:443/us/en",
"localhost/us/en", "127.0.0.1/us/en", "[::1]/us/en", "2130706433/us/en", "10.0.0.1/us/en",
"king/us/en", "careers.king.com./us/en", ".king.com/us/en", "careers..king.com/us/en",
"CAREERS.KING.COM/us/en", "careers.king.com/US/en", "careers.king.com/us/EN",
"careers.king.com\\us/en", "careers.king.com/u/en", "careers.king.com/us/eng",
])
def test_a_hostile_slug_is_refused(slug: str) -> None:
assert not phenom.slug_ok(slug)
assert not ats.is_known_target(f"phenom:{slug}")
def test_only_the_site_named_by_the_slug_is_allowed() -> None:
allow = phenom.allowed(SLUG)
assert allow("careers.king.com") and allow("CAREERS.KING.COM")
assert not allow("king.com") and not allow("careers.king.com.evil.com")
assert not allow("xboxgaming.wd1.myworkdayjobs.com") and not allow("127.0.0.1")
assert ats._allowed_host("phenom", SLUG)("careers.king.com")
def test_the_careers_page_is_the_search_page() -> None:
url = "https://careers.king.com/us/en/search-results"
assert phenom.careers_url(SLUG) == ats.careers_url(f"phenom:{SLUG}") == url
assert phenom.allowed(SLUG)(httpx.URL(url).host)
assert phenom.careers_url("www.careers.philips.com/global/en") == \
"https://www.careers.philips.com/global/en/search-results"
# -------------------------------------------------------------- discovery
def test_discovery_finds_the_site_and_the_locale_in_a_link_to_its_search_page() -> None:
page = ('<a href="https://careers.king.com/us/en/search-results">Jobs</a>'
'<a href="https://www.careers.philips.com/global/en/search-results?q=1">More</a>')
assert [(h, s) for h, s, _ in boards_in(page)] == [
("phenom", "careers.king.com/us/en"), ("phenom", "www.careers.philips.com/global/en")]
def test_discovery_lowercases_what_a_page_writes_in_capitals() -> None:
page = "https://Careers.King.com/US/EN/search-results"
assert [(h, s) for h, s, _ in boards_in(page)] == [("phenom", "careers.king.com/us/en")]
def test_discovery_names_a_site_that_loads_phenoms_assets_without_a_locale_root() -> None:
page = '<script src="https://cdn.phenompeople.com/CareerConnectResources/x.js"></script>'
assert [(h, s) for h, s, _ in boards_in(page)] == [("phenom", "")]
def test_a_page_that_only_looks_like_one_names_nothing() -> None:
"""The search page is the one URL that carries host and locale root, so a
link to any other page of a site is not enough."""
assert boards_in("https://careers.king.com/us/en/job/R028130/Product-Analyst") == []
assert boards_in("https://careers.king.com/search-results") == []
assert boards_in("https://127.0.0.1/us/en/search-results") == []
def test_every_slug_discovery_finds_is_a_slug() -> None:
for page in ("https://careers.king.com/us/en/search-results",
"https://careers.justeattakeaway.com/global/en/search-results#x"):
found = boards_in(page)
assert found and all(phenom.slug_ok(slug) for _h, slug, _ in found), page