Files
jobboard/tests/test_ats_jibe.py
T
WolfandClaude Opus 5.5 c497fe965d Keep a description's headings, bullets and paragraphs, and draw them in the reader
The readers flattened a posting's HTML into one line with strip_html, so
sections and bullets ran together. _shared.html_text gives a line per
paragraph, '• ' before a list item and '## ' before a heading; every
reader's description goes through it, sections read per role get their
own headings, and the reader pane draws the lines as h3, ul and p.
Excerpts leave the marks out.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-30 21:40:12 +02:00

186 lines
7.8 KiB
Python

"""The Jibe host module against a saved page of Booking.com's listing (six of
its 145 roles, descriptions cut). No network."""
from __future__ import annotations
import json
import sys
from datetime import UTC, datetime
from pathlib import Path
from typing import Any
import httpx
import pytest
ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from app.jobs import discover # noqa: E402
from plugins import ats, readers # noqa: E402
from plugins.readers import Ask, jibe # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
SITE = "jobs.booking.com"
def _saved() -> dict[str, Any]:
return json.loads((FIXTURES / "jibe.json").read_text())
def _board(total: int) -> dict[int, dict[str, Any]]:
"""`total` roles as the site pages them: 100 to a page, the saved roles
repeated under new slugs."""
template = _saved()["jobs"]
jobs = []
for n in range(total):
data = dict(template[n % len(template)]["data"]) | {"slug": str(1000 + n)}
jobs.append({"data": data})
return {page + 1: {"jobs": jobs[start:start + 100], "totalCount": total}
for page, start in enumerate(range(0, total, 100))}
def test_a_page_becomes_rows() -> None:
rows = jibe.rows(_saved(), SITE)
assert [r["source_id"] for r in rows] == ["30413", "30262", "30320", "30204", "30222",
"13402"]
first, several = rows[0], rows[3]
assert first["title"].startswith("Booking Holdings Romania - Team Leader")
assert first["url"] == "https://jobs.booking.com/jobs/30413"
assert first["location"] == "Bucharest, Romania"
assert first["department"] == "Legal & Public Affairs"
assert first["posted"] == datetime(2026, 9, 29, 12, 55, tzinfo=UTC)
assert (first["remote"], first["salary_text"]) == (None, "")
assert several["location"] == "Amsterdam, Netherlands; Manchester, United Kingdom"
# Every place the site lists is kept; a country only for a role at one place.
assert [p["text"] for p in several["places"]] == ["Amsterdam, Netherlands",
"Manchester, United Kingdom"]
assert "country" not in several
assert first["places"] == [{"text": "Bucharest, Romania", "remote": None}]
assert (first["country"], first["employment"]) == ("RO", "FULL_TIME")
assert all(r["description"] and "<" not in r["description"] and "&nbsp;" not in r["description"]
for r in rows)
def test_qualifications_follow_the_description() -> None:
text = jibe.rows(_saved(), SITE)[-1]["description"]
assert any(line.startswith("Java") for line in text.split("\n")[1:])
def test_the_location_falls_back_to_city_and_country() -> None:
role = {"slug": "1", "title": "Role", "city": "Riga", "country": "Latvia"}
assert jibe.rows({"jobs": [{"data": role}]}, SITE)[0]["location"] == "Riga, Latvia"
def test_a_role_without_a_title_or_a_slug_is_left_out() -> None:
jobs = [{"data": {"slug": "1", "title": " "}}, {"data": {"title": "Role"}}, {}]
assert jibe.rows({"jobs": jobs}, SITE) == []
def test_a_page_states_its_total_and_the_cursor_is_the_next_number() -> None:
asked: list[str] = []
board = _board(145)
def get(url: str, **_: Any) -> bytes:
asked.append(url)
return json.dumps(board[int(url.rsplit("=", 1)[1])]).encode()
decoded, results, total, cursor = jibe.page(get, SITE, None, Ask())
assert asked == ["https://jobs.booking.com/api/jobs?limit=100&page=1"]
assert (len(results), total, cursor) == (100, 145, 2)
assert len(jibe.rows(decoded, SITE)) == 100
assert jibe.page(get, SITE, 2, Ask())[2:] == (145, 3)
assert asked[-1].endswith("&page=2")
def _crawl(board: dict[int, dict[str, Any]]) -> tuple[list[dict[str, Any]], bool, list[str]]:
"""Pages as the runner does: on to the next number until the cursor ends
or as many distinct ids were read as the total; complete in the latter."""
seen: list[str] = []
def get(url: str, **_: Any) -> bytes:
seen.append(url)
return json.dumps(board[int(url.rsplit("=", 1)[1])]).encode()
rows: list[dict[str, Any]] = []
ids: set[str] = set()
cursor: Any = None
while True:
decoded, results, total, cursor = jibe.page(get, SITE, cursor, Ask())
rows += jibe.rows(decoded, SITE)
ids |= {r["id"] for r in results}
if cursor is None or (total is not None and len(ids) >= total):
return rows, total is not None and len(ids) >= total, seen
def test_the_board_is_read_to_its_total() -> None:
rows, complete, seen = _crawl(_board(145))
assert (len(rows), complete) == (145, True)
assert seen == [f"https://jobs.booking.com/api/jobs?limit=100&page={n}" for n in (1, 2)]
def test_a_board_that_ends_short_of_its_total_is_incomplete() -> None:
board = _board(145)
board[2]["totalCount"] = board[1]["totalCount"] = 150 # five roles the site never sends
board[3] = {"jobs": [], "totalCount": 150}
rows, complete, _seen = _crawl(board)
assert (len(rows), complete) == (145, False)
def test_an_empty_board_is_complete_at_zero() -> None:
assert _crawl({1: {"jobs": [], "totalCount": 0}})[:2] == ([], True)
@pytest.mark.parametrize("body", [b"<html>Not found</html>", b"[]", b'{"jobs": null}',
b'{"error": "no"}'])
def test_an_answer_that_is_not_a_listing_raises(body: bytes) -> None:
with pytest.raises(ValueError):
jibe.page(lambda url, **_: body, SITE, None, Ask())
def test_a_422_and_a_429_reach_the_shared_code_as_status_errors() -> None:
"""A module lets them through: 429 after the first page ends the read."""
request = httpx.Request("GET", f"https://{SITE}/api/jobs")
def get(url: str, **_: Any) -> bytes:
raise httpx.HTTPStatusError("422", request=request,
response=httpx.Response(422, request=request))
with pytest.raises(httpx.HTTPStatusError):
jibe.page(get, SITE, None, Ask())
def test_the_slug_the_urls_and_the_allow_list() -> None:
for slug in (SITE, "Jobs.Booking.com", "careers.acme.co.uk", "acme.com", "a-b.c-d.example"):
assert jibe.slug_ok(slug) and ats.is_known_target(f"jibe:{slug}")
assert jibe.careers_url("Jobs.Booking.com") == "https://jobs.booking.com/jobs"
allow = jibe.allowed("Jobs.Booking.com")
assert allow("jobs.booking.com") and allow("JOBS.BOOKING.COM")
assert not allow("booking.com") and not allow("evil.example")
assert not allow("jobs.booking.com.evil.example") and not allow("localhost")
assert readers.HOSTS["jibe"] is jibe and ats.HOSTS["jibe"] == "Jibe"
@pytest.mark.parametrize("slug", [
"", "booking", "localhost", "a@b.com", "a.com#x", "a.com?x", "..", "a.com/x", "a.com%2fx",
"a.com:443", "a.com:8443", " a.com", "a.com ", "a.com\n", "a.com\r\n", "127.0.0.1",
"2130706433", "[::1]", "0x7f.0x0.0x0.0x1", "a.b", "-a.com", "a-.com", "a..com", "a.com.",
"app.localhost", "printer.local", "meta.internal", "nas.home.arpa", "router.lan",
"user@jobs.booking.com", "a_b.com", "évil.com"])
def test_a_hostile_slug_is_refused(slug: str) -> None:
assert not jibe.slug_ok(slug)
assert not ats.is_known_target(f"jibe:{slug}")
def test_discovery_names_jibe_from_its_assets_but_gives_no_host() -> None:
page = ('<img src="https://cms.jibecdn.com/prod/workingatbooking/assets/logo.png">'
'<a href="https://workingatbooking.jibeapply.com/jobs">apply</a>')
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("jibe", "")]
def test_it_declares_a_paged_read_with_a_total_and_many_places() -> None:
reads = jibe.READS
assert (reads.listing, reads.page_size, reads.text, reads.complete, reads.places) == (
"paged", 100, "listing", "stated", "many")
assert reads.fields.country == "stated" and reads.robots.startswith("read: robots.txt")