Files
jobboard/tests/test_ats_phenom.py
T
WolfandClaude Fable 5.1 2349fc424b Require READS, search params and a dated robots record; record robots.txt per requested path
Every reader declares, so undeclared(), the scope() fallback, robots_page and
ROBOTS_ANY_CASE go. A search provider needs PARAMS and clean_params, a row may
carry company, places gains none. Each robots record names the paths the
reader requests, from robots.txt read again on 2026-09-30. Platsbanken states
remote, HR Manager states no place.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-30 15:51:13 +02:00

328 lines
15 KiB
Python

"""Phenom: the listing's rows from a saved `POST /widgets` answer (King's,
trimmed), the request body, paging and completeness against a fake `get`, the
description from a saved job page's JSON-LD (King's and Philips's, trimmed),
the slug, and discovery's patterns. No network."""
from __future__ import annotations
import json
import sys
from pathlib import Path
from typing import Any
import httpx
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app.jobs.discover import boards_in # noqa: E402
from plugins import ats # noqa: E402
from plugins.readers import phenom # noqa: E402
from plugins.readers._shared import is_web_url # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
SLUG = "careers.king.com/us/en"
IDS = ["ACMKDSUSR028168EXTERNALENUS", "ACMKDSUSR027656EXTERNALENUS",
"ACMKDSUSR028172EXTERNALENUS", "ACMKDSUSR028147EXTERNALENUS",
"ACMKDSUSR028167EXTERNALENUS"]
def _answer() -> dict[str, Any]:
return json.loads((FIXTURES / "phenom.json").read_bytes())
def _rows() -> list[ats.Row]:
return phenom.parse((FIXTURES / "phenom.json").read_bytes(), SLUG)
# ------------------------------------------------------------------- rows
def test_rows_carry_the_listing_fields() -> None:
rows = _rows()
assert [r["source_id"] for r in rows] == IDS
first = rows[0]
assert first["title"] == "Principal Economy Designer | New Games"
assert first["department"] == "Games Design & Production"
assert first["description"] == "" # the text comes with the role's own page
assert first["teaser"].startswith("Become part of our team and help shape the economic")
assert first["remote"] is None and first["salary_text"] == ""
assert all(is_web_url(r["url"]) for r in rows)
def test_posted_is_the_date_the_role_was_created_not_the_one_the_site_moves() -> None:
"""The second role was created on 15 June and is listed as posted on 28
September; Phenom refreshes `postedDate`, so the creation date is taken, and
`postedDate` only where a site sends no `dateCreated`."""
rows = _rows()
assert rows[0]["posted"].isoformat() == "2026-09-29T09:59:51.253000+00:00"
assert rows[1]["posted"].isoformat() == "2026-06-15T17:02:06.245000+00:00"
data = _answer()
del data["refineSearch"]["data"]["jobs"][1]["dateCreated"]
assert phenom.parse(json.dumps(data).encode(), SLUG)[1]["posted"].isoformat() == \
"2026-09-28T00:00:00+00:00"
def test_the_url_is_the_role_page_on_the_site_not_the_apply_system() -> None:
"""`applyUrl` points at Workday; the role's page is `/job/<jobSeqNo>/<title>`."""
assert _answer()["refineSearch"]["data"]["jobs"][0]["applyUrl"].startswith("https://xboxgaming.")
assert _rows()[0]["url"] == ("https://careers.king.com/us/en/job/ACMKDSUSR028168EXTERNALENUS"
"/Principal-Economy-Designer-New-Games")
assert _rows()[1]["url"].endswith("/Senior-Data-Visualisation-Engineer-12mth-FTC")
def test_the_locations_are_every_place_the_role_names() -> None:
rows = _rows()
assert rows[0]["location"] == ("Barcelona, Spain; London, London, United Kingdom; "
"Stockholm, Stockholm County, Sweden")
assert rows[1]["location"] == "Stockholm, Stockholm County, Sweden"
def test_every_place_is_kept_once_and_the_first_is_the_roles_own() -> None:
rows = _rows()
assert [p["text"] for p in rows[0]["places"]] == [
"Barcelona, Spain", "London, London, United Kingdom", "Stockholm, Stockholm County, Sweden"]
assert rows[2]["places"][0]["text"] == "London, London, United Kingdom"
assert all(p["remote"] is None for r in rows for p in r["places"])
def test_a_role_with_one_place_only_uses_location() -> None:
data = _answer()
job = data["refineSearch"]["data"]["jobs"][0]
job["multi_location"], job["isMultiLocation"] = [], False
assert phenom.parse(json.dumps(data).encode(), SLUG)[0]["location"] == "Barcelona, Spain"
def test_a_title_with_nothing_to_make_a_slug_of_has_a_page_by_id_alone() -> None:
data = _answer()
data["refineSearch"]["data"]["jobs"][0]["title"] = "?!"
assert phenom.parse(json.dumps(data).encode(), SLUG)[0]["url"] == \
"https://careers.king.com/us/en/job/ACMKDSUSR028168EXTERNALENUS"
def test_a_role_without_title_or_id_is_left_out() -> None:
data = _answer()
jobs = data["refineSearch"]["data"]["jobs"]
jobs[0]["title"] = ""
for key in ("jobSeqNo", "jobId", "reqId"):
jobs[1].pop(key, None)
assert [r["source_id"] for r in phenom.parse(json.dumps(data).encode(), SLUG)] == IDS[2:]
# ---------------------------------------------------------- the request
def test_the_body_names_the_locale_and_the_page_and_nothing_secret() -> None:
body = phenom._body("www.careers.philips.com/global/en", 200)
assert (body["lang"], body["country"], body["from"], body["size"]) == \
("en_global", "global", 200, 100)
assert body["ddoKey"] == "refineSearch" and body["jobs"] is True
assert phenom._body(SLUG, 0)["lang"] == "en_us"
assert not {"refNum", "pageId", "token", "key"} & set(body)
# ----------------------------------------------------------------- paging
@pytest.fixture
def no_delay(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
def _fake(monkeypatch: pytest.MonkeyPatch, total: int | None = None
) -> list[tuple[str, str, dict[str, Any] | None]]:
"""`ats._get` answering the search from the saved roles, `from` and `size`
honoured, and a page past the end empty."""
asked: list[tuple[str, str, dict[str, Any] | None]] = []
every = _answer()["refineSearch"]["data"]["jobs"]
def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None,
www_twin=False):
asked.append((url, method, payload))
assert payload is not None
data = _answer()
jobs = every[payload["from"]:payload["from"] + payload["size"]]
data["refineSearch"]["data"]["jobs"] = jobs
data["refineSearch"]["hits"] = len(jobs)
data["refineSearch"]["totalHits"] = len(every) if total is None else total
return json.dumps(data).encode()
monkeypatch.setattr(ats, "_get", fake_get)
return asked
def test_the_listing_is_posted_page_by_page_to_the_stated_total(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
asked = _fake(monkeypatch)
monkeypatch.setattr(phenom, "PAGE_SIZE", 2)
rows, complete = ats._fetch_rows(object(), f"phenom:{SLUG}") # type: ignore[arg-type]
assert [(url, method, (payload or {})["from"], (payload or {})["size"]) for url, method, payload in asked] == [
("https://careers.king.com/widgets", "POST", offset, 2) for offset in (0, 2, 4)]
assert [r["source_id"] for r in rows] == IDS
assert complete
def test_a_board_that_ends_short_of_its_total_is_incomplete(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
_fake(monkeypatch, total=7)
monkeypatch.setattr(phenom, "PAGE_SIZE", 2)
rows, complete = ats._fetch_rows(object(), f"phenom:{SLUG}") # type: ignore[arg-type]
assert (len(rows), complete) == (5, False)
def test_a_small_board_is_one_page(monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
asked = _fake(monkeypatch)
rows, complete = ats._fetch_rows(object(), f"phenom:{SLUG}") # type: ignore[arg-type]
assert len(asked) == 1
assert (len(rows), complete) == (5, True)
def test_an_empty_board_is_complete(monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
empty = {"refineSearch": {"status": 200, "hits": 0, "totalHits": 0, "data": {"jobs": []}}}
monkeypatch.setattr(ats, "_get", lambda *a, **k: json.dumps(empty).encode())
assert ats._fetch_rows(object(), f"phenom:{SLUG}") == ([], True) # type: ignore[arg-type]
def test_an_answer_that_is_not_a_search_fails_the_board(
monkeypatch: pytest.MonkeyPatch, no_delay: None) -> None:
monkeypatch.setattr(ats, "_get", lambda *a, **k: b'{"error": "no such page"}')
with pytest.raises(ValueError, match="refineSearch"):
ats._fetch_rows(object(), f"phenom:{SLUG}") # type: ignore[arg-type]
# ----------------------------------------------------------------- detail
def _get_of(name: str, asked: list[tuple[str, dict[str, str] | None]]) -> Any:
body = (FIXTURES / name).read_bytes()
def get(url: str, *, method: str = "GET", payload: Any = None, headers: Any = None) -> bytes:
asked.append((url, headers))
return body
return get
def test_detail_reads_the_description_from_the_json_ld() -> None:
asked: list[tuple[str, dict[str, str] | None]] = []
row = _rows()[0]
got = phenom.detail(_get_of("phenom-detail.html", asked), SLUG, row)
assert asked == [(row["url"], {"Accept": "text/html"})]
assert got["description"].startswith("Craft: Data, Analytics & Strategy")
assert "<" not in got["description"] and "&lt;" not in got["description"]
assert "&amp;" not in got["description"]
assert got["posted"].isoformat() == "2026-09-28T00:00:00+00:00"
assert set(got) <= set(ats.DETAIL_FIELDS)
def test_detail_finds_the_job_posting_among_other_blocks_and_takes_its_date() -> None:
got = phenom.detail(_get_of("phenom-detail-2.html", []), "www.careers.philips.com/global/en",
{"url": "https://www.careers.philips.com/global/en/job/1/x"})
assert got["description"].startswith("Job Title Zone Business Leader Imaging Nordics")
assert got["posted"].isoformat() == "2026-09-08T00:00:00+00:00"
def test_detail_falls_back_to_the_page_state_where_there_is_no_job_posting() -> None:
"""King's pages of roles posted that day had no `JobPosting` block, only a
`WebPage` one, and the role in the state the page assigns to `phApp.ddo`."""
row = _rows()[0]
got = phenom.detail(_get_of("phenom-detail-3.html", []), SLUG, row)
assert got["description"].startswith("Craft: Games Design & Production Job Description:")
assert "<" not in got["description"] and "&amp;" not in got["description"]
assert got["remote"] == "On-site"
assert "posted" not in got and set(got) <= set(ats.DETAIL_FIELDS)
def test_detail_of_a_page_with_no_description_fails_so_that_it_is_asked_again() -> None:
def get(url: str, **kwargs: Any) -> bytes:
return (b'<html><script type="application/ld+json">{"@type": "BreadcrumbList"}</script>'
b'<script type="application/ld+json">{broken</script>'
b'<script>phApp.ddo = {"jobDetail": {"data": {"job": {"title": "x"}}}};</script></html>')
with pytest.raises(ValueError, match="no description"):
phenom.detail(get, SLUG, {"url": "https://careers.king.com/us/en/job/1"})
@pytest.mark.parametrize("state", ["", "phApp.ddo = {broken", 'phApp.ddo = {"jobDetail": []};',
'phApp.ddo = {"siteConfig": {}};'])
def test_a_page_state_that_is_not_a_role_is_no_description(state: str) -> None:
def get(url: str, **kwargs: Any) -> bytes:
return f"<html><script>{state}</script></html>".encode()
with pytest.raises(ValueError, match="no description"):
phenom.detail(get, SLUG, {"url": "https://careers.king.com/us/en/job/1"})
# ----------------------------------------------------------- slug and URLs
@pytest.mark.parametrize("slug", [
"careers.king.com/us/en", "www.careers.philips.com/global/en",
"careers.justeattakeaway.com/global/en", "jobs.example.co.uk/na/en", "a-b.example.de/de/de",
])
def test_a_host_and_a_locale_root_are_a_slug(slug: str) -> None:
assert phenom.slug_ok(slug) and ats.is_known_target(f"phenom:{slug}")
@pytest.mark.parametrize("slug", [
"", "careers.king.com", "careers.king.com/", "careers.king.com/us", "careers.king.com/us/",
"careers.king.com/us/en/", "careers.king.com/us/en/x", "careers.king.com//en",
"careers.king.com/us/en\n", "careers.king.com/us/en ", " careers.king.com/us/en",
"careers.king.com /us/en", "careers.king.com\n/us/en", "careers.king.com/us/en\r\n",
"careers.king.com@evil.com/us/en", "evil.com@careers.king.com/us/en",
"careers.king.com#x/us/en", "careers.king.com?x=1/us/en", "careers.king.com/us/en?x=1",
"careers.king.com/us/en#x", "careers.king.com/us/%2fen", "careers.king.com/../en",
"careers.king.com/us/..", "careers.king.com:8443/us/en", "careers.king.com:443/us/en",
"localhost/us/en", "127.0.0.1/us/en", "[::1]/us/en", "2130706433/us/en", "10.0.0.1/us/en",
"king/us/en", "careers.king.com./us/en", ".king.com/us/en", "careers..king.com/us/en",
"CAREERS.KING.COM/us/en", "careers.king.com/US/en", "careers.king.com/us/EN",
"careers.king.com\\us/en", "careers.king.com/u/en", "careers.king.com/us/eng",
])
def test_a_hostile_slug_is_refused(slug: str) -> None:
assert not phenom.slug_ok(slug)
assert not ats.is_known_target(f"phenom:{slug}")
def test_only_the_site_named_by_the_slug_is_allowed() -> None:
allow = phenom.allowed(SLUG)
assert allow("careers.king.com") and allow("CAREERS.KING.COM")
assert not allow("king.com") and not allow("careers.king.com.evil.com")
assert not allow("xboxgaming.wd1.myworkdayjobs.com") and not allow("127.0.0.1")
assert ats._allowed_host("phenom", SLUG)("careers.king.com")
def test_the_careers_page_is_the_search_page() -> None:
url = "https://careers.king.com/us/en/search-results"
assert phenom.careers_url(SLUG) == ats.careers_url(f"phenom:{SLUG}") == url
assert phenom.allowed(SLUG)(httpx.URL(url).host)
assert phenom.careers_url("www.careers.philips.com/global/en") == \
"https://www.careers.philips.com/global/en/search-results"
# -------------------------------------------------------------- discovery
def test_discovery_finds_the_site_and_the_locale_in_a_link_to_its_search_page() -> None:
page = ('<a href="https://careers.king.com/us/en/search-results">Jobs</a>'
'<a href="https://www.careers.philips.com/global/en/search-results?q=1">More</a>')
assert [(h, s) for h, s, _ in boards_in(page)] == [
("phenom", "careers.king.com/us/en"), ("phenom", "www.careers.philips.com/global/en")]
def test_discovery_lowercases_what_a_page_writes_in_capitals() -> None:
page = "https://Careers.King.com/US/EN/search-results"
assert [(h, s) for h, s, _ in boards_in(page)] == [("phenom", "careers.king.com/us/en")]
def test_discovery_names_a_site_that_loads_phenoms_assets_without_a_locale_root() -> None:
page = '<script src="https://cdn.phenompeople.com/CareerConnectResources/x.js"></script>'
assert [(h, s) for h, s, _ in boards_in(page)] == [("phenom", "")]
def test_a_page_that_only_looks_like_one_names_nothing() -> None:
"""The search page is the one URL that carries host and locale root, so a
link to any other page of a site is not enough."""
assert boards_in("https://careers.king.com/us/en/job/R028130/Product-Analyst") == []
assert boards_in("https://careers.king.com/search-results") == []
assert boards_in("https://127.0.0.1/us/en/search-results") == []
def test_every_slug_discovery_finds_is_a_slug() -> None:
for page in ("https://careers.king.com/us/en/search-results",
"https://careers.justeattakeaway.com/global/en/search-results#x"):
found = boards_in(page)
assert found and all(phenom.slug_ok(slug) for _h, slug, _ in found), page