The readers flattened a posting's HTML into one line with strip_html, so sections and bullets ran together. _shared.html_text gives a line per paragraph, '• ' before a list item and '## ' before a heading; every reader's description goes through it, sections read per role get their own headings, and the reader pane draws the lines as h3, ul and p. Excerpts leave the marks out. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
186 lines
7.8 KiB
Python
186 lines
7.8 KiB
Python
"""The Jibe host module against a saved page of Booking.com's listing (six of
|
|
its 145 roles, descriptions cut). No network."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import sys
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import httpx
|
|
import pytest
|
|
|
|
ROOT = Path(__file__).resolve().parents[1]
|
|
if str(ROOT) not in sys.path:
|
|
sys.path.insert(0, str(ROOT))
|
|
|
|
from app.jobs import discover # noqa: E402
|
|
from plugins import ats, readers # noqa: E402
|
|
from plugins.readers import Ask, jibe # noqa: E402
|
|
|
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
|
SITE = "jobs.booking.com"
|
|
|
|
|
|
def _saved() -> dict[str, Any]:
|
|
return json.loads((FIXTURES / "jibe.json").read_text())
|
|
|
|
|
|
def _board(total: int) -> dict[int, dict[str, Any]]:
|
|
"""`total` roles as the site pages them: 100 to a page, the saved roles
|
|
repeated under new slugs."""
|
|
template = _saved()["jobs"]
|
|
jobs = []
|
|
for n in range(total):
|
|
data = dict(template[n % len(template)]["data"]) | {"slug": str(1000 + n)}
|
|
jobs.append({"data": data})
|
|
return {page + 1: {"jobs": jobs[start:start + 100], "totalCount": total}
|
|
for page, start in enumerate(range(0, total, 100))}
|
|
|
|
|
|
def test_a_page_becomes_rows() -> None:
|
|
rows = jibe.rows(_saved(), SITE)
|
|
assert [r["source_id"] for r in rows] == ["30413", "30262", "30320", "30204", "30222",
|
|
"13402"]
|
|
first, several = rows[0], rows[3]
|
|
assert first["title"].startswith("Booking Holdings Romania - Team Leader")
|
|
assert first["url"] == "https://jobs.booking.com/jobs/30413"
|
|
assert first["location"] == "Bucharest, Romania"
|
|
assert first["department"] == "Legal & Public Affairs"
|
|
assert first["posted"] == datetime(2026, 9, 29, 12, 55, tzinfo=UTC)
|
|
assert (first["remote"], first["salary_text"]) == (None, "")
|
|
assert several["location"] == "Amsterdam, Netherlands; Manchester, United Kingdom"
|
|
# Every place the site lists is kept; a country only for a role at one place.
|
|
assert [p["text"] for p in several["places"]] == ["Amsterdam, Netherlands",
|
|
"Manchester, United Kingdom"]
|
|
assert "country" not in several
|
|
assert first["places"] == [{"text": "Bucharest, Romania", "remote": None}]
|
|
assert (first["country"], first["employment"]) == ("RO", "FULL_TIME")
|
|
assert all(r["description"] and "<" not in r["description"] and " " not in r["description"]
|
|
for r in rows)
|
|
|
|
|
|
def test_qualifications_follow_the_description() -> None:
|
|
text = jibe.rows(_saved(), SITE)[-1]["description"]
|
|
assert any(line.startswith("Java") for line in text.split("\n")[1:])
|
|
|
|
|
|
def test_the_location_falls_back_to_city_and_country() -> None:
|
|
role = {"slug": "1", "title": "Role", "city": "Riga", "country": "Latvia"}
|
|
assert jibe.rows({"jobs": [{"data": role}]}, SITE)[0]["location"] == "Riga, Latvia"
|
|
|
|
|
|
def test_a_role_without_a_title_or_a_slug_is_left_out() -> None:
|
|
jobs = [{"data": {"slug": "1", "title": " "}}, {"data": {"title": "Role"}}, {}]
|
|
assert jibe.rows({"jobs": jobs}, SITE) == []
|
|
|
|
|
|
def test_a_page_states_its_total_and_the_cursor_is_the_next_number() -> None:
|
|
asked: list[str] = []
|
|
board = _board(145)
|
|
|
|
def get(url: str, **_: Any) -> bytes:
|
|
asked.append(url)
|
|
return json.dumps(board[int(url.rsplit("=", 1)[1])]).encode()
|
|
|
|
decoded, results, total, cursor = jibe.page(get, SITE, None, Ask())
|
|
assert asked == ["https://jobs.booking.com/api/jobs?limit=100&page=1"]
|
|
assert (len(results), total, cursor) == (100, 145, 2)
|
|
assert len(jibe.rows(decoded, SITE)) == 100
|
|
assert jibe.page(get, SITE, 2, Ask())[2:] == (145, 3)
|
|
assert asked[-1].endswith("&page=2")
|
|
|
|
|
|
def _crawl(board: dict[int, dict[str, Any]]) -> tuple[list[dict[str, Any]], bool, list[str]]:
|
|
"""Pages as the runner does: on to the next number until the cursor ends
|
|
or as many distinct ids were read as the total; complete in the latter."""
|
|
seen: list[str] = []
|
|
|
|
def get(url: str, **_: Any) -> bytes:
|
|
seen.append(url)
|
|
return json.dumps(board[int(url.rsplit("=", 1)[1])]).encode()
|
|
|
|
rows: list[dict[str, Any]] = []
|
|
ids: set[str] = set()
|
|
cursor: Any = None
|
|
while True:
|
|
decoded, results, total, cursor = jibe.page(get, SITE, cursor, Ask())
|
|
rows += jibe.rows(decoded, SITE)
|
|
ids |= {r["id"] for r in results}
|
|
if cursor is None or (total is not None and len(ids) >= total):
|
|
return rows, total is not None and len(ids) >= total, seen
|
|
|
|
|
|
def test_the_board_is_read_to_its_total() -> None:
|
|
rows, complete, seen = _crawl(_board(145))
|
|
assert (len(rows), complete) == (145, True)
|
|
assert seen == [f"https://jobs.booking.com/api/jobs?limit=100&page={n}" for n in (1, 2)]
|
|
|
|
|
|
def test_a_board_that_ends_short_of_its_total_is_incomplete() -> None:
|
|
board = _board(145)
|
|
board[2]["totalCount"] = board[1]["totalCount"] = 150 # five roles the site never sends
|
|
board[3] = {"jobs": [], "totalCount": 150}
|
|
rows, complete, _seen = _crawl(board)
|
|
assert (len(rows), complete) == (145, False)
|
|
|
|
|
|
def test_an_empty_board_is_complete_at_zero() -> None:
|
|
assert _crawl({1: {"jobs": [], "totalCount": 0}})[:2] == ([], True)
|
|
|
|
|
|
@pytest.mark.parametrize("body", [b"<html>Not found</html>", b"[]", b'{"jobs": null}',
|
|
b'{"error": "no"}'])
|
|
def test_an_answer_that_is_not_a_listing_raises(body: bytes) -> None:
|
|
with pytest.raises(ValueError):
|
|
jibe.page(lambda url, **_: body, SITE, None, Ask())
|
|
|
|
|
|
def test_a_422_and_a_429_reach_the_shared_code_as_status_errors() -> None:
|
|
"""A module lets them through: 429 after the first page ends the read."""
|
|
request = httpx.Request("GET", f"https://{SITE}/api/jobs")
|
|
|
|
def get(url: str, **_: Any) -> bytes:
|
|
raise httpx.HTTPStatusError("422", request=request,
|
|
response=httpx.Response(422, request=request))
|
|
|
|
with pytest.raises(httpx.HTTPStatusError):
|
|
jibe.page(get, SITE, None, Ask())
|
|
|
|
|
|
def test_the_slug_the_urls_and_the_allow_list() -> None:
|
|
for slug in (SITE, "Jobs.Booking.com", "careers.acme.co.uk", "acme.com", "a-b.c-d.example"):
|
|
assert jibe.slug_ok(slug) and ats.is_known_target(f"jibe:{slug}")
|
|
assert jibe.careers_url("Jobs.Booking.com") == "https://jobs.booking.com/jobs"
|
|
allow = jibe.allowed("Jobs.Booking.com")
|
|
assert allow("jobs.booking.com") and allow("JOBS.BOOKING.COM")
|
|
assert not allow("booking.com") and not allow("evil.example")
|
|
assert not allow("jobs.booking.com.evil.example") and not allow("localhost")
|
|
assert readers.HOSTS["jibe"] is jibe and ats.HOSTS["jibe"] == "Jibe"
|
|
|
|
|
|
@pytest.mark.parametrize("slug", [
|
|
"", "booking", "localhost", "a@b.com", "a.com#x", "a.com?x", "..", "a.com/x", "a.com%2fx",
|
|
"a.com:443", "a.com:8443", " a.com", "a.com ", "a.com\n", "a.com\r\n", "127.0.0.1",
|
|
"2130706433", "[::1]", "0x7f.0x0.0x0.0x1", "a.b", "-a.com", "a-.com", "a..com", "a.com.",
|
|
"app.localhost", "printer.local", "meta.internal", "nas.home.arpa", "router.lan",
|
|
"user@jobs.booking.com", "a_b.com", "évil.com"])
|
|
def test_a_hostile_slug_is_refused(slug: str) -> None:
|
|
assert not jibe.slug_ok(slug)
|
|
assert not ats.is_known_target(f"jibe:{slug}")
|
|
|
|
|
|
def test_discovery_names_jibe_from_its_assets_but_gives_no_host() -> None:
|
|
page = ('<img src="https://cms.jibecdn.com/prod/workingatbooking/assets/logo.png">'
|
|
'<a href="https://workingatbooking.jibeapply.com/jobs">apply</a>')
|
|
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("jibe", "")]
|
|
|
|
|
|
def test_it_declares_a_paged_read_with_a_total_and_many_places() -> None:
|
|
reads = jibe.READS
|
|
assert (reads.listing, reads.page_size, reads.text, reads.complete, reads.places) == (
|
|
"paged", 100, "listing", "stated", "many")
|
|
assert reads.fields.country == "stated" and reads.robots.startswith("read: robots.txt")
|