Ten entries were shadowed by their modules. A test now holds that no entry names a reader, which replaces the per-host checks. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
186 lines
7.8 KiB
Python
186 lines
7.8 KiB
Python
"""The Jibe host module against a saved page of Booking.com's listing (six of
|
|
its 145 roles, descriptions cut). No network."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from app.jobs import discover # noqa: E402
|
|
from plugins import ats, readers # noqa: E402
|
|
from plugins.readers import Ask, jibe # noqa: E402
|
|
|
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
|
SITE = "jobs.booking.com"
|
|
|
|
|
|
def _saved() -> dict[str, Any]:
|
|
return json.loads((FIXTURES / "jibe.json").read_text())
|
|
|
|
|
|
def _board(total: int) -> dict[int, dict[str, Any]]:
|
|
"""`total` roles as the site pages them: 100 to a page, the saved roles
|
|
repeated under new slugs."""
|
|
template = _saved()["jobs"]
|
|
jobs = []
|
|
for n in range(total):
|
|
data = dict(template[n % len(template)]["data"]) | {"slug": str(1000 + n)}
|
|
jobs.append({"data": data})
|
|
return {page + 1: {"jobs": jobs[start:start + 100], "totalCount": total}
|
|
for page, start in enumerate(range(0, total, 100))}
|
|
|
|
|
|
def test_a_page_becomes_rows() -> None:
|
|
rows = jibe.rows(_saved(), SITE)
|
|
assert [r["source_id"] for r in rows] == ["30413", "30262", "30320", "30204", "30222",
|
|
"13402"]
|
|
first, several = rows[0], rows[3]
|
|
assert first["title"].startswith("Booking Holdings Romania - Team Leader")
|
|
assert first["url"] == "https://jobs.booking.com/jobs/30413"
|
|
assert first["location"] == "Bucharest, Romania"
|
|
assert first["department"] == "Legal & Public Affairs"
|
|
assert first["posted"] == datetime(2026, 9, 29, 12, 55, tzinfo=UTC)
|
|
assert (first["remote"], first["salary_text"]) == (None, "")
|
|
assert several["location"] == "Amsterdam, Netherlands; Manchester, United Kingdom"
|
|
# Every place the site lists is kept; a country only for a role at one place.
|
|
assert [p["text"] for p in several["places"]] == ["Amsterdam, Netherlands",
|
|
"Manchester, United Kingdom"]
|
|
assert "country" not in several
|
|
assert first["places"] == [{"text": "Bucharest, Romania", "remote": None}]
|
|
assert (first["country"], first["employment"]) == ("RO", "FULL_TIME")
|
|
assert all(r["description"] and "<" not in r["description"] and " " not in r["description"]
|
|
for r in rows)
|
|
|
|
|
|
def test_qualifications_follow_the_description() -> None:
|
|
text = jibe.rows(_saved(), SITE)[-1]["description"]
|
|
assert "\n\n" in text and text.split("\n\n")[-1].startswith("Java")
|
|
|
|
|
|
def test_the_location_falls_back_to_city_and_country() -> None:
|
|
role = {"slug": "1", "title": "Role", "city": "Riga", "country": "Latvia"}
|
|
assert jibe.rows({"jobs": [{"data": role}]}, SITE)[0]["location"] == "Riga, Latvia"
|
|
|
|
|
|
def test_a_role_without_a_title_or_a_slug_is_left_out() -> None:
|
|
jobs = [{"data": {"slug": "1", "title": " "}}, {"data": {"title": "Role"}}, {}]
|
|
assert jibe.rows({"jobs": jobs}, SITE) == []
|
|
|
|
|
|
def test_a_page_states_its_total_and_the_cursor_is_the_next_number() -> None:
|
|
asked: list[str] = []
|
|
board = _board(145)
|
|
|
|
def get(url: str, **_: Any) -> bytes:
|
|
asked.append(url)
|
|
return json.dumps(board[int(url.rsplit("=", 1)[1])]).encode()
|
|
|
|
decoded, results, total, cursor = jibe.page(get, SITE, None, Ask())
|
|
assert asked == ["https://jobs.booking.com/api/jobs?limit=100&page=1"]
|
|
assert (len(results), total, cursor) == (100, 145, 2)
|
|
assert len(jibe.rows(decoded, SITE)) == 100
|
|
assert jibe.page(get, SITE, 2, Ask())[2:] == (145, 3)
|
|
assert asked[-1].endswith("&page=2")
|
|
|
|
|
|
def _crawl(board: dict[int, dict[str, Any]]) -> tuple[list[dict[str, Any]], bool, list[str]]:
|
|
"""Pages as the runner does: on to the next number until the cursor ends
|
|
or as many distinct ids were read as the total; complete in the latter."""
|
|
seen: list[str] = []
|
|
|
|
def get(url: str, **_: Any) -> bytes:
|
|
seen.append(url)
|
|
return json.dumps(board[int(url.rsplit("=", 1)[1])]).encode()
|
|
|
|
rows: list[dict[str, Any]] = []
|
|
ids: set[str] = set()
|
|
cursor: Any = None
|
|
while True:
|
|
decoded, results, total, cursor = jibe.page(get, SITE, cursor, Ask())
|
|
rows += jibe.rows(decoded, SITE)
|
|
ids |= {r["id"] for r in results}
|
|
if cursor is None or (total is not None and len(ids) >= total):
|
|
return rows, total is not None and len(ids) >= total, seen
|
|
|
|
|
|
def test_the_board_is_read_to_its_total() -> None:
|
|
rows, complete, seen = _crawl(_board(145))
|
|
assert (len(rows), complete) == (145, True)
|
|
assert seen == [f"https://jobs.booking.com/api/jobs?limit=100&page={n}" for n in (1, 2)]
|
|
|
|
|
|
def test_a_board_that_ends_short_of_its_total_is_incomplete() -> None:
|
|
board = _board(145)
|
|
board[2]["totalCount"] = board[1]["totalCount"] = 150 # five roles the site never sends
|
|
board[3] = {"jobs": [], "totalCount": 150}
|
|
rows, complete, _seen = _crawl(board)
|
|
assert (len(rows), complete) == (145, False)
|
|
|
|
|
|
def test_an_empty_board_is_complete_at_zero() -> None:
|
|
assert _crawl({1: {"jobs": [], "totalCount": 0}})[:2] == ([], True)
|
|
|
|
|
|
@pytest.mark.parametrize("body", [b"<html>Not found</html>", b"[]", b'{"jobs": null}',
|
|
b'{"error": "no"}'])
|
|
def test_an_answer_that_is_not_a_listing_raises(body: bytes) -> None:
|
|
with pytest.raises(ValueError):
|
|
jibe.page(lambda url, **_: body, SITE, None, Ask())
|
|
|
|
|
|
def test_a_422_and_a_429_reach_the_shared_code_as_status_errors() -> None:
|
|
"""A module lets them through: 429 after the first page ends the read."""
|
|
request = httpx.Request("GET", f"https://{SITE}/api/jobs")
|
|
|
|
def get(url: str, **_: Any) -> bytes:
|
|
raise httpx.HTTPStatusError("422", request=request,
|
|
response=httpx.Response(422, request=request))
|
|
|
|
with pytest.raises(httpx.HTTPStatusError):
|
|
jibe.page(get, SITE, None, Ask())
|
|
|
|
|
|
def test_the_slug_the_urls_and_the_allow_list() -> None:
|
|
for slug in (SITE, "Jobs.Booking.com", "careers.acme.co.uk", "acme.com", "a-b.c-d.example"):
|
|
assert jibe.slug_ok(slug) and ats.is_known_target(f"jibe:{slug}")
|
|
assert jibe.careers_url("Jobs.Booking.com") == "https://jobs.booking.com/jobs"
|
|
allow = jibe.allowed("Jobs.Booking.com")
|
|
assert allow("jobs.booking.com") and allow("JOBS.BOOKING.COM")
|
|
assert not allow("booking.com") and not allow("evil.example")
|
|
assert not allow("jobs.booking.com.evil.example") and not allow("localhost")
|
|
assert readers.HOSTS["jibe"] is jibe and ats.HOSTS["jibe"] == "Jibe"
|
|
|
|
|
|
@pytest.mark.parametrize("slug", [
|
|
"", "booking", "localhost", "a@b.com", "a.com#x", "a.com?x", "..", "a.com/x", "a.com%2fx",
|
|
"a.com:443", "a.com:8443", " a.com", "a.com ", "a.com\n", "a.com\r\n", "127.0.0.1",
|
|
"2130706433", "[::1]", "0x7f.0x0.0x0.0x1", "a.b", "-a.com", "a-.com", "a..com", "a.com.",
|
|
"app.localhost", "printer.local", "meta.internal", "nas.home.arpa", "router.lan",
|
|
"user@jobs.booking.com", "a_b.com", "évil.com"])
|
|
def test_a_hostile_slug_is_refused(slug: str) -> None:
|
|
assert not jibe.slug_ok(slug)
|
|
assert not ats.is_known_target(f"jibe:{slug}")
|
|
|
|
|
|
def test_discovery_names_jibe_from_its_assets_but_gives_no_host() -> None:
|
|
page = ('<img src="https://cms.jibecdn.com/prod/workingatbooking/assets/logo.png">'
|
|
'<a href="https://workingatbooking.jibeapply.com/jobs">apply</a>')
|
|
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("jibe", "")]
|
|
|
|
|
|
def test_it_declares_a_paged_read_with_a_total_and_many_places() -> None:
|
|
reads = jibe.READS
|
|
assert (reads.listing, reads.page_size, reads.text, reads.complete, reads.places) == (
|
|
"paged", 100, "listing", "stated", "many")
|
|
assert reads.fields.country == "stated" and reads.robots.startswith("read: robots.txt")
|