Hosts a to l declare READS: places, teaser, stated seniority and employment
Ashby, BambooHR, Deel, Eightfold, Gem, Greenhouse, HiBob, HR Manager, Jibe, Jobylon and Lever move to feed_url(slug, ask) and page(get, slug, cursor, ask), declare READS with the robots decision, and drop robots_page. Rows carry places (Ashby secondaryLocations, Lever allLocations, Greenhouse offices in place of a Hybrid placeholder), Jobylon keeps its summary as teaser and maps experience and employment, Eightfold reads creationTs as the posting date and Ashby reads workplaceType. Fixtures <name>-places.* and jobylon-teaser.html drive the conformance checks; the eleven names leave UNDECLARED. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
4c0da58da7
commit
f2e652a8da
35 files changed
+1290
-391
No files matched your search
+41
-27
@@ -18,7 +18,7 @@ if str(ROOT) not in sys.path:
|
||||
|
||||
from app.jobs import discover # noqa: E402
|
||||
from plugins import ats, readers # noqa: E402
|
||||
from plugins.readers import jibe # noqa: E402
|
||||
from plugins.readers import Ask, jibe # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
SITE = "jobs.booking.com"
|
||||
@@ -52,6 +52,12 @@ def test_a_page_becomes_rows() -> None:
|
||||
assert first["posted"] == datetime(2026, 9, 29, 12, 55, tzinfo=UTC)
|
||||
assert (first["remote"], first["salary_text"]) == (None, "")
|
||||
assert several["location"] == "Amsterdam, Netherlands; Manchester, United Kingdom"
|
||||
# Every place the site lists is kept; a country only for a role at one place.
|
||||
assert [p["text"] for p in several["places"]] == ["Amsterdam, Netherlands",
|
||||
"Manchester, United Kingdom"]
|
||||
assert "country" not in several
|
||||
assert first["places"] == [{"text": "Bucharest, Romania", "remote": None}]
|
||||
assert (first["country"], first["employment"]) == ("RO", "full_time")
|
||||
assert all(r["description"] and "<" not in r["description"] and " " not in r["description"]
|
||||
for r in rows)
|
||||
|
||||
@@ -79,56 +85,57 @@ def test_a_page_states_its_total_and_the_cursor_is_the_next_number() -> None:
|
||||
asked.append(url)
|
||||
return json.dumps(board[int(url.rsplit("=", 1)[1])]).encode()
|
||||
|
||||
decoded, results, total, cursor = jibe.page(get, SITE, None)
|
||||
decoded, results, total, cursor = jibe.page(get, SITE, None, Ask())
|
||||
assert asked == ["https://jobs.booking.com/api/jobs?limit=100&page=1"]
|
||||
assert (len(results), total, cursor) == (100, 145, 2)
|
||||
assert len(jibe.rows(decoded, SITE)) == 100
|
||||
assert jibe.page(get, SITE, 2)[2:] == (145, 3)
|
||||
assert jibe.page(get, SITE, 2, Ask())[2:] == (145, 3)
|
||||
assert asked[-1].endswith("&page=2")
|
||||
|
||||
|
||||
def test_the_board_is_read_to_its_total_through_the_app(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
|
||||
board = _board(145)
|
||||
def _crawl(board: dict[int, dict[str, Any]]) -> tuple[list[dict[str, Any]], bool, list[str]]:
|
||||
"""Pages as the runner does: on to the next number until the cursor ends
|
||||
or as many distinct ids were read as the total; complete in the latter."""
|
||||
seen: list[str] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
seen.append(str(request.url))
|
||||
return httpx.Response(200, json=board[int(request.url.params["page"])])
|
||||
def get(url: str, **_: Any) -> bytes:
|
||||
seen.append(url)
|
||||
return json.dumps(board[int(url.rsplit("=", 1)[1])]).encode()
|
||||
|
||||
with httpx.Client(transport=httpx.MockTransport(handler)) as client:
|
||||
rows, complete = ats._fetch_rows(client, f"jibe:{SITE}")
|
||||
rows: list[dict[str, Any]] = []
|
||||
ids: set[str] = set()
|
||||
cursor: Any = None
|
||||
while True:
|
||||
decoded, results, total, cursor = jibe.page(get, SITE, cursor, Ask())
|
||||
rows += jibe.rows(decoded, SITE)
|
||||
ids |= {r["id"] for r in results}
|
||||
if cursor is None or (total is not None and len(ids) >= total):
|
||||
return rows, total is not None and len(ids) >= total, seen
|
||||
|
||||
|
||||
def test_the_board_is_read_to_its_total() -> None:
|
||||
rows, complete, seen = _crawl(_board(145))
|
||||
assert (len(rows), complete) == (145, True)
|
||||
assert seen == [f"https://jobs.booking.com/api/jobs?limit=100&page={n}" for n in (1, 2)]
|
||||
|
||||
|
||||
def test_a_board_that_ends_short_of_its_total_is_incomplete(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
|
||||
def test_a_board_that_ends_short_of_its_total_is_incomplete() -> None:
|
||||
board = _board(145)
|
||||
board[2]["totalCount"] = board[1]["totalCount"] = 150 # five roles the site never sends
|
||||
board[3] = {"jobs": [], "totalCount": 150}
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, json=board[int(request.url.params["page"])])
|
||||
|
||||
with httpx.Client(transport=httpx.MockTransport(handler)) as client:
|
||||
rows, complete = ats._fetch_rows(client, f"jibe:{SITE}")
|
||||
rows, complete, _seen = _crawl(board)
|
||||
assert (len(rows), complete) == (145, False)
|
||||
|
||||
|
||||
def test_an_empty_board_is_complete_at_zero() -> None:
|
||||
body = {"jobs": [], "totalCount": 0}
|
||||
with httpx.Client(transport=httpx.MockTransport(
|
||||
lambda request: httpx.Response(200, json=body))) as client:
|
||||
assert ats._fetch_rows(client, f"jibe:{SITE}") == ([], True)
|
||||
assert _crawl({1: {"jobs": [], "totalCount": 0}})[:2] == ([], True)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("body", [b"<html>Not found</html>", b"[]", b'{"jobs": null}',
|
||||
b'{"error": "no"}'])
|
||||
def test_an_answer_that_is_not_a_listing_raises(body: bytes) -> None:
|
||||
with pytest.raises(ValueError):
|
||||
jibe.page(lambda url, **_: body, SITE, None)
|
||||
jibe.page(lambda url, **_: body, SITE, None, Ask())
|
||||
|
||||
|
||||
def test_a_422_and_a_429_reach_the_shared_code_as_status_errors() -> None:
|
||||
@@ -140,14 +147,13 @@ def test_a_422_and_a_429_reach_the_shared_code_as_status_errors() -> None:
|
||||
response=httpx.Response(422, request=request))
|
||||
|
||||
with pytest.raises(httpx.HTTPStatusError):
|
||||
jibe.page(get, SITE, None)
|
||||
jibe.page(get, SITE, None, Ask())
|
||||
|
||||
|
||||
def test_the_slug_the_urls_and_the_allow_list() -> None:
|
||||
for slug in (SITE, "Jobs.Booking.com", "careers.acme.co.uk", "acme.com", "a-b.c-d.example"):
|
||||
assert jibe.slug_ok(slug) and ats.is_known_target(f"jibe:{slug}")
|
||||
assert jibe.careers_url("Jobs.Booking.com") == "https://jobs.booking.com/jobs"
|
||||
assert jibe.robots_page(SITE) == "https://jobs.booking.com/jobs"
|
||||
allow = jibe.allowed("Jobs.Booking.com")
|
||||
assert allow("jobs.booking.com") and allow("JOBS.BOOKING.COM")
|
||||
assert not allow("booking.com") and not allow("evil.example")
|
||||
@@ -171,3 +177,11 @@ def test_discovery_names_jibe_from_its_assets_but_gives_no_host() -> None:
|
||||
'<a href="https://workingatbooking.jibeapply.com/jobs">apply</a>')
|
||||
assert [(h, s) for h, s, _ in discover.boards_in(page)] == [("jibe", "")]
|
||||
assert "jibe" not in {h for h, _ in discover._UNREADABLE}
|
||||
|
||||
|
||||
def test_it_declares_a_paged_read_with_a_total_and_many_places() -> None:
|
||||
reads = jibe.READS
|
||||
assert (reads.listing, reads.page_size, reads.text, reads.complete, reads.places) == (
|
||||
"paged", 100, "listing", "stated", "many")
|
||||
assert reads.fields.country == "stated" and reads.robots.startswith("read: robots.txt")
|
||||
assert not hasattr(jibe, "robots_page")
|
||||
Reference in new issue
Block a user