Every row carries places (Personio additionalOffices, Workable and Recruitee locations[], every SuccessFactors address, Workday's own read), the list teaser of Phenom and Oracle is kept in teaser, and the level, employment and country a host states are mapped. Workday "N Locations" and "Country - City", and Recruitee "Remote job", are no longer places. Phenom takes dateCreated as posted. robots_page and ROBOTS_ANY_CASE go; READS.robots records the decision. Signatures are feed_url(slug, ask) and page(get, slug, cursor, ask), with an empty Ask as the default so the runner of today still calls them. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
329 lines
15 KiB
Python
329 lines
15 KiB
Python
"""Phenom: the listing's rows from a saved `POST /widgets` answer (King's,
|
|
trimmed), the request body, paging and completeness against a fake `get`, the
|
|
description from a saved job page's JSON-LD (King's and Philips's, trimmed),
|
|
the slug, and discovery's patterns. No network."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from app.jobs.discover import boards_in # noqa: E402
|
|
from plugins import ats # noqa: E402
|
|
from plugins.readers import phenom # noqa: E402
|
|
from plugins.readers._shared import is_web_url # noqa: E402
|
|
|
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
|
SLUG = "careers.king.com/us/en"
|
|
IDS = ["ACMKDSUSR028168EXTERNALENUS", "ACMKDSUSR027656EXTERNALENUS",
|
|
"ACMKDSUSR028172EXTERNALENUS", "ACMKDSUSR028147EXTERNALENUS",
|
|
"ACMKDSUSR028167EXTERNALENUS"]
|
|
|
|
|
|
def _answer() -> dict[str, Any]:
|
|
return json.loads((FIXTURES / "phenom.json").read_bytes())
|
|
|
|
|
|
def _rows() -> list[ats.Row]:
|
|
return phenom.parse((FIXTURES / "phenom.json").read_bytes(), SLUG)
|
|
|
|
|
|
# ------------------------------------------------------------------- rows
|
|
|
|
def test_rows_carry_the_listing_fields() -> None:
|
|
rows = _rows()
|
|
assert [r["source_id"] for r in rows] == IDS
|
|
first = rows[0]
|
|
assert first["title"] == "Principal Economy Designer | New Games"
|
|
assert first["department"] == "Games Design & Production"
|
|
assert first["description"] == "" # the text comes with the role's own page
|
|
assert first["teaser"].startswith("Become part of our team and help shape the economic")
|
|
assert first["remote"] is None and first["salary_text"] == ""
|
|
assert all(is_web_url(r["url"]) for r in rows)
|
|
|
|
|
|
def test_posted_is_the_date_the_role_was_created_not_the_one_the_site_moves() -> None:
|
|
"""The second role was created on 15 June and is listed as posted on 28
|
|
September; Phenom refreshes `postedDate`, so the creation date is taken, and
|
|
`postedDate` only where a site sends no `dateCreated`."""
|
|
rows = _rows()
|
|
assert rows[0]["posted"].isoformat() == "2026-09-29T09:59:51.253000+00:00"
|
|
assert rows[1]["posted"].isoformat() == "2026-06-15T17:02:06.245000+00:00"
|
|
data = _answer()
|
|
del data["refineSearch"]["data"]["jobs"][1]["dateCreated"]
|
|
assert phenom.parse(json.dumps(data).encode(), SLUG)[1]["posted"].isoformat() == \
|
|
"2026-09-28T00:00:00+00:00"
|
|
|
|
|
|
def test_the_url_is_the_role_page_on_the_site_not_the_apply_system() -> None:
|
|
"""`applyUrl` points at Workday; the role's page is `/job/<jobSeqNo>/<title>`."""
|
|
assert _answer()["refineSearch"]["data"]["jobs"][0]["applyUrl"].startswith("https://xboxgaming.")
|
|
assert _rows()[0]["url"] == ("https://careers.king.com/us/en/job/ACMKDSUSR028168EXTERNALENUS"
|
|
"/Principal-Economy-Designer-New-Games")
|
|
assert _rows()[1]["url"].endswith("/Senior-Data-Visualisation-Engineer-12mth-FTC")
|
|
|
|
|
|
def test_the_locations_are_every_place_the_role_names() -> None:
|
|
rows = _rows()
|
|
assert rows[0]["location"] == ("Barcelona, Spain; London, London, United Kingdom; "
|
|
"Stockholm, Stockholm County, Sweden")
|
|
assert rows[1]["location"] == "Stockholm, Stockholm County, Sweden"
|
|
|
|
|
|
def test_every_place_is_kept_once_and_the_first_is_the_roles_own() -> None:
|
|
rows = _rows()
|
|
assert [p["text"] for p in rows[0]["places"]] == [
|
|
"Barcelona, Spain", "London, London, United Kingdom", "Stockholm, Stockholm County, Sweden"]
|
|
assert rows[2]["places"][0]["text"] == "London, London, United Kingdom"
|
|
assert all(p["remote"] is None for r in rows for p in r["places"])
|
|
|
|
|
|
def test_a_role_with_one_place_only_uses_location() -> None:
|
|
data = _answer()
|
|
job = data["refineSearch"]["data"]["jobs"][0]
|
|
job["multi_location"], job["isMultiLocation"] = [], False
|
|
assert phenom.parse(json.dumps(data).encode(), SLUG)[0]["location"] == "Barcelona, Spain"
|
|
|
|
|
|
def test_a_title_with_nothing_to_make_a_slug_of_has_a_page_by_id_alone() -> None:
|
|
data = _answer()
|
|
data["refineSearch"]["data"]["jobs"][0]["title"] = "?!"
|
|
assert phenom.parse(json.dumps(data).encode(), SLUG)[0]["url"] == \
|
|
"https://careers.king.com/us/en/job/ACMKDSUSR028168EXTERNALENUS"
|
|
|
|
|
|
def test_a_role_without_title_or_id_is_left_out() -> None:
|
|
data = _answer()
|
|
jobs = data["refineSearch"]["data"]["jobs"]
|
|
jobs[0]["title"] = ""
|
|
for key in ("jobSeqNo", "jobId", "reqId"):
|
|
jobs[1].pop(key, None)
|
|
assert [r["source_id"] for r in phenom.parse(json.dumps(data).encode(), SLUG)] == IDS[2:]
|
|
|
|
|
|
# ---------------------------------------------------------- the request
|
|
|
|
def test_the_body_names_the_locale_and_the_page_and_nothing_secret() -> None:
|
|
body = phenom._body("www.careers.philips.com/global/en", 200)
|
|
assert (body["lang"], body["country"], body["from"], body["size"]) == \
|
|
("en_global", "global", 200, 100)
|
|
assert body["ddoKey"] == "refineSearch" and body["jobs"] is True
|
|
assert phenom._body(SLUG, 0)["lang"] == "en_us"
|
|
assert not {"refNum", "pageId", "token", "key"} & set(body)
|
|
|
|
|
|
# ----------------------------------------------------------------- paging
|
|
|
|
@pytest.fixture
|
|
def no_delay(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
|
|
|
|
|
|
def _fake(monkeypatch: pytest.MonkeyPatch, total: int | None = None
|
|
) -> list[tuple[str, str, dict[str, Any] | None]]:
|
|
"""`ats._get` answering the search from the saved roles, `from` and `size`
|
|
honoured, and a page past the end empty."""
|
|
asked: list[tuple[str, str, dict[str, Any] | None]] = []
|
|
every = _answer()["refineSearch"]["data"]["jobs"]
|
|
|
|
def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None,
|
|
www_twin=False):
|
|
asked.append((url, method, payload))
|
|
assert payload is not None
|
|
data = _answer()
|
|
jobs = every[payload["from"]:payload["from"] + payload["size"]]
|
|
data["refineSearch"]["data"]["jobs"] = jobs
|
|
data["refineSearch"]["hits"] = len(jobs)
|
|
data["refineSearch"]["totalHits"] = len(every) if total is None else total
|
|
return json.dumps(data).encode()
|
|
|
|
monkeypatch.setattr(ats, "_get", fake_get)
|
|
return asked
|
|
|
|
|
|
def test_the_listing_is_posted_page_by_page_to_the_stated_total(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
asked = _fake(monkeypatch)
|
|
monkeypatch.setattr(phenom, "PAGE_SIZE", 2)
|
|
rows, complete = ats._fetch_rows(object(), f"phenom:{SLUG}") # type: ignore[arg-type]
|
|
assert [(url, method, (payload or {})["from"], (payload or {})["size"]) for url, method, payload in asked] == [
|
|
("https://careers.king.com/widgets", "POST", offset, 2) for offset in (0, 2, 4)]
|
|
assert [r["source_id"] for r in rows] == IDS
|
|
assert complete
|
|
|
|
|
|
def test_a_board_that_ends_short_of_its_total_is_incomplete(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
_fake(monkeypatch, total=7)
|
|
monkeypatch.setattr(phenom, "PAGE_SIZE", 2)
|
|
rows, complete = ats._fetch_rows(object(), f"phenom:{SLUG}") # type: ignore[arg-type]
|
|
assert (len(rows), complete) == (5, False)
|
|
|
|
|
|
def test_a_small_board_is_one_page(monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
asked = _fake(monkeypatch)
|
|
rows, complete = ats._fetch_rows(object(), f"phenom:{SLUG}") # type: ignore[arg-type]
|
|
assert len(asked) == 1
|
|
assert (len(rows), complete) == (5, True)
|
|
|
|
|
|
def test_an_empty_board_is_complete(monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
empty = {"refineSearch": {"status": 200, "hits": 0, "totalHits": 0, "data": {"jobs": []}}}
|
|
monkeypatch.setattr(ats, "_get", lambda *a, **k: json.dumps(empty).encode())
|
|
assert ats._fetch_rows(object(), f"phenom:{SLUG}") == ([], True) # type: ignore[arg-type]
|
|
|
|
|
|
def test_an_answer_that_is_not_a_search_fails_the_board(
|
|
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
|
|
monkeypatch.setattr(ats, "_get", lambda *a, **k: b'{"error": "no such page"}')
|
|
with pytest.raises(ValueError, match="refineSearch"):
|
|
ats._fetch_rows(object(), f"phenom:{SLUG}") # type: ignore[arg-type]
|
|
|
|
|
|
# ----------------------------------------------------------------- detail
|
|
|
|
def _get_of(name: str, asked: list[tuple[str, dict[str, str] | None]]) -> Any:
|
|
body = (FIXTURES / name).read_bytes()
|
|
|
|
def get(url: str, *, method: str = "GET", payload: Any = None, headers: Any = None) -> bytes:
|
|
asked.append((url, headers))
|
|
return body
|
|
return get
|
|
|
|
|
|
def test_detail_reads_the_description_from_the_json_ld() -> None:
|
|
asked: list[tuple[str, dict[str, str] | None]] = []
|
|
row = _rows()[0]
|
|
got = phenom.detail(_get_of("phenom-detail.html", asked), SLUG, row)
|
|
assert asked == [(row["url"], {"Accept": "text/html"})]
|
|
assert got["description"].startswith("Craft: Data, Analytics & Strategy")
|
|
assert "<" not in got["description"] and "<" not in got["description"]
|
|
assert "&" not in got["description"]
|
|
assert got["posted"].isoformat() == "2026-09-28T00:00:00+00:00"
|
|
assert set(got) <= set(ats.DETAIL_FIELDS)
|
|
|
|
|
|
def test_detail_finds_the_job_posting_among_other_blocks_and_takes_its_date() -> None:
|
|
got = phenom.detail(_get_of("phenom-detail-2.html", []), "www.careers.philips.com/global/en",
|
|
{"url": "https://www.careers.philips.com/global/en/job/1/x"})
|
|
assert got["description"].startswith("Job Title Zone Business Leader Imaging Nordics")
|
|
assert got["posted"].isoformat() == "2026-09-08T00:00:00+00:00"
|
|
|
|
|
|
def test_detail_falls_back_to_the_page_state_where_there_is_no_job_posting() -> None:
|
|
"""King's pages of roles posted that day had no `JobPosting` block, only a
|
|
`WebPage` one, and the role in the state the page assigns to `phApp.ddo`."""
|
|
row = _rows()[0]
|
|
got = phenom.detail(_get_of("phenom-detail-3.html", []), SLUG, row)
|
|
assert got["description"].startswith("Craft: Games Design & Production Job Description:")
|
|
assert "<" not in got["description"] and "&" not in got["description"]
|
|
assert got["remote"] == "onsite"
|
|
assert "posted" not in got and set(got) <= set(ats.DETAIL_FIELDS)
|
|
|
|
|
|
def test_detail_of_a_page_with_no_description_fails_so_that_it_is_asked_again() -> None:
|
|
def get(url: str, **kwargs: Any) -> bytes:
|
|
return (b'<html><script type="application/ld+json">{"@type": "BreadcrumbList"}</script>'
|
|
b'<script type="application/ld+json">{broken</script>'
|
|
b'<script>phApp.ddo = {"jobDetail": {"data": {"job": {"title": "x"}}}};</script></html>')
|
|
with pytest.raises(ValueError, match="no description"):
|
|
phenom.detail(get, SLUG, {"url": "https://careers.king.com/us/en/job/1"})
|
|
|
|
|
|
@pytest.mark.parametrize("state", ["", "phApp.ddo = {broken", 'phApp.ddo = {"jobDetail": []};',
|
|
'phApp.ddo = {"siteConfig": {}};'])
|
|
def test_a_page_state_that_is_not_a_role_is_no_description(state: str) -> None:
|
|
def get(url: str, **kwargs: Any) -> bytes:
|
|
return f"<html><script>{state}</script></html>".encode()
|
|
with pytest.raises(ValueError, match="no description"):
|
|
phenom.detail(get, SLUG, {"url": "https://careers.king.com/us/en/job/1"})
|
|
|
|
|
|
# ----------------------------------------------------------- slug and URLs
|
|
|
|
@pytest.mark.parametrize("slug", [
|
|
"careers.king.com/us/en", "www.careers.philips.com/global/en",
|
|
"careers.justeattakeaway.com/global/en", "jobs.example.co.uk/na/en", "a-b.example.de/de/de",
|
|
])
|
|
def test_a_host_and_a_locale_root_are_a_slug(slug: str) -> None:
|
|
assert phenom.slug_ok(slug) and ats.is_known_target(f"phenom:{slug}")
|
|
|
|
|
|
@pytest.mark.parametrize("slug", [
|
|
"", "careers.king.com", "careers.king.com/", "careers.king.com/us", "careers.king.com/us/",
|
|
"careers.king.com/us/en/", "careers.king.com/us/en/x", "careers.king.com//en",
|
|
"careers.king.com/us/en\n", "careers.king.com/us/en ", " careers.king.com/us/en",
|
|
"careers.king.com /us/en", "careers.king.com\n/us/en", "careers.king.com/us/en\r\n",
|
|
"careers.king.com@evil.com/us/en", "evil.com@careers.king.com/us/en",
|
|
"careers.king.com#x/us/en", "careers.king.com?x=1/us/en", "careers.king.com/us/en?x=1",
|
|
"careers.king.com/us/en#x", "careers.king.com/us/%2fen", "careers.king.com/../en",
|
|
"careers.king.com/us/..", "careers.king.com:8443/us/en", "careers.king.com:443/us/en",
|
|
"localhost/us/en", "127.0.0.1/us/en", "[::1]/us/en", "2130706433/us/en", "10.0.0.1/us/en",
|
|
"king/us/en", "careers.king.com./us/en", ".king.com/us/en", "careers..king.com/us/en",
|
|
"CAREERS.KING.COM/us/en", "careers.king.com/US/en", "careers.king.com/us/EN",
|
|
"careers.king.com\\us/en", "careers.king.com/u/en", "careers.king.com/us/eng",
|
|
])
|
|
def test_a_hostile_slug_is_refused(slug: str) -> None:
|
|
assert not phenom.slug_ok(slug)
|
|
assert not ats.is_known_target(f"phenom:{slug}")
|
|
|
|
|
|
def test_only_the_site_named_by_the_slug_is_allowed() -> None:
|
|
allow = phenom.allowed(SLUG)
|
|
assert allow("careers.king.com") and allow("CAREERS.KING.COM")
|
|
assert not allow("king.com") and not allow("careers.king.com.evil.com")
|
|
assert not allow("xboxgaming.wd1.myworkdayjobs.com") and not allow("127.0.0.1")
|
|
assert ats._allowed_host("phenom", SLUG)("careers.king.com")
|
|
|
|
|
|
def test_the_careers_page_is_the_search_page() -> None:
|
|
url = "https://careers.king.com/us/en/search-results"
|
|
assert phenom.careers_url(SLUG) == ats.careers_url(f"phenom:{SLUG}") == url
|
|
assert phenom.allowed(SLUG)(httpx.URL(url).host)
|
|
assert not hasattr(phenom, "robots_page") # decided when built, see READS.robots
|
|
assert phenom.careers_url("www.careers.philips.com/global/en") == \
|
|
"https://www.careers.philips.com/global/en/search-results"
|
|
|
|
|
|
# -------------------------------------------------------------- discovery
|
|
|
|
def test_discovery_finds_the_site_and_the_locale_in_a_link_to_its_search_page() -> None:
|
|
page = ('<a href="https://careers.king.com/us/en/search-results">Jobs</a>'
|
|
'<a href="https://www.careers.philips.com/global/en/search-results?q=1">More</a>')
|
|
assert [(h, s) for h, s, _ in boards_in(page)] == [
|
|
("phenom", "careers.king.com/us/en"), ("phenom", "www.careers.philips.com/global/en")]
|
|
|
|
|
|
def test_discovery_lowercases_what_a_page_writes_in_capitals() -> None:
|
|
page = "https://Careers.King.com/US/EN/search-results"
|
|
assert [(h, s) for h, s, _ in boards_in(page)] == [("phenom", "careers.king.com/us/en")]
|
|
|
|
|
|
def test_discovery_names_a_site_that_loads_phenoms_assets_without_a_locale_root() -> None:
|
|
page = '<script src="https://cdn.phenompeople.com/CareerConnectResources/x.js"></script>'
|
|
assert [(h, s) for h, s, _ in boards_in(page)] == [("phenom", "")]
|
|
|
|
|
|
def test_a_page_that_only_looks_like_one_names_nothing() -> None:
|
|
"""The search page is the one URL that carries host and locale root, so a
|
|
link to any other page of a site is not enough."""
|
|
assert boards_in("https://careers.king.com/us/en/job/R028130/Product-Analyst") == []
|
|
assert boards_in("https://careers.king.com/search-results") == []
|
|
assert boards_in("https://127.0.0.1/us/en/search-results") == []
|
|
|
|
|
|
def test_every_slug_discovery_finds_is_a_slug() -> None:
|
|
for page in ("https://careers.king.com/us/en/search-results",
|
|
"https://careers.justeattakeaway.com/global/en/search-results#x"):
|
|
found = boards_in(page)
|
|
assert found and all(phenom.slug_ok(slug) for _h, slug, _ in found), page
|