Hosts a to l declare READS: places, teaser, stated seniority and employment

Ashby, BambooHR, Deel, Eightfold, Gem, Greenhouse, HiBob, HR Manager, Jibe,
Jobylon and Lever move to feed_url(slug, ask) and page(get, slug, cursor, ask),
declare READS with the robots decision, and drop robots_page. Rows carry
places (Ashby secondaryLocations, Lever allLocations, Greenhouse offices in
place of a Hybrid placeholder), Jobylon keeps its summary as teaser and maps
experience and employment, Eightfold reads creationTs as the posting date and
Ashby reads workplaceType. Fixtures <name>-places.* and jobylon-teaser.html
drive the conformance checks; the eleven names leave UNDECLARED.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 09:11:53 +02:00
1 parent 4c0da58da7
commit f2e652a8da
35 files changed
+1290 -391

No files matched your search

+39 -65
View File
@@ -16,7 +16,7 @@ if str(ROOT) not in sys.path:
from app.jobs.discover import boards_in # noqa: E402
from plugins import ats, readers # noqa: E402
from plugins.readers import hrmanager # noqa: E402
from plugins.readers import Ask, hrmanager # noqa: E402
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
BOARD = (FIXTURES / "hrmanager.xml").read_bytes()
@@ -73,83 +73,58 @@ def _paged(answers: dict[str, bytes], asked: list[str]) -> Any:
def test_the_page_moves_on_by_what_it_returned_and_ends_on_an_empty_one() -> None:
asked: list[str] = []
get = _paged({_feed(0): BOARD, _feed(5): EMPTY}, asked)
data, results, total, cursor = hrmanager.page(get, "stsgroup_tr", None)
data, results, total, cursor = hrmanager.page(get, "stsgroup_tr", None, Ask())
assert len(results) == 5 and total is None and cursor == 5
assert len(hrmanager.rows(data, "stsgroup_tr")) == 5
_, results, total, cursor = hrmanager.page(get, "stsgroup_tr", cursor)
_, results, total, cursor = hrmanager.page(get, "stsgroup_tr", cursor, Ask())
assert results == [] and total is None and cursor is None
assert asked == [_feed(0), _feed(5)]
@pytest.fixture
def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]:
answers: dict[str, Any] = {}
def _crawl(get: Any, slug: str) -> list[dict[str, Any]]:
"""Pages until the cursor ends, as the runner does, giving the rows."""
out: list[dict[str, Any]] = []
cursor: Any = None
while True:
data, _results, _total, cursor = hrmanager.page(get, slug, cursor, Ask())
out += hrmanager.rows(data, slug)
if cursor is None:
return out
def test_a_board_is_read_to_an_empty_page() -> None:
asked: list[str] = []
def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None,
www_twin=False):
asked.append(url)
if not ats._allowed_host(host, slug)(httpx.URL(url).host):
raise ats._Refused(url)
answer = answers[url]
if isinstance(answer, BaseException):
raise answer
return answer
monkeypatch.setattr(ats, "_get", fake_get)
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
return {"answers": answers, "asked": asked}
rows = _crawl(_paged({_feed(0): BOARD, _feed(5): EMPTY}, asked), "stsgroup_tr")
assert len(rows) == 5 and asked == [_feed(0), _feed(5)]
def test_a_board_is_complete_when_the_read_ends_on_an_empty_page(web: dict[str, Any]) -> None:
web["answers"] |= {_feed(0): BOARD, _feed(5): EMPTY}
with httpx.Client() as client:
rows, complete = ats._fetch_rows(client, "hrmanager:stsgroup_tr")
assert len(rows) == 5 and complete
assert web["asked"] == [_feed(0), _feed(5)]
def test_a_board_with_no_roles_is_empty() -> None:
asked: list[str] = []
assert _crawl(_paged({_feed(0, "seaborg_tr"): EMPTY}, asked), "seaborg_tr") == []
assert asked == [_feed(0, "seaborg_tr")]
def test_a_board_with_no_roles_is_complete_and_empty(web: dict[str, Any]) -> None:
web["answers"][_feed(0, "seaborg_tr")] = EMPTY
with httpx.Client() as client:
assert ats._fetch_rows(client, "hrmanager:seaborg_tr") == ([], True)
assert web["asked"] == [_feed(0, "seaborg_tr")]
def test_a_page_the_host_repeats_leaves_the_board_incomplete(web: dict[str, Any]) -> None:
"""A host that ignored `skip` would answer the same roles again."""
web["answers"] |= {_feed(0): BOARD, _feed(5): BOARD}
with httpx.Client() as client:
rows, complete = ats._fetch_rows(client, "hrmanager:stsgroup_tr")
assert len(rows) == 5 and not complete
def test_a_429_after_the_first_page_ends_the_read_incomplete(web: dict[str, Any]) -> None:
def test_a_429_after_the_first_page_reaches_the_caller() -> None:
request = httpx.Request("GET", _feed(5))
web["answers"] |= {_feed(0): BOARD, _feed(5): httpx.HTTPStatusError(
"429", request=request, response=httpx.Response(429, request=request))}
with httpx.Client() as client:
rows, complete = ats._fetch_rows(client, "hrmanager:stsgroup_tr")
assert len(rows) == 5 and not complete
def get(url: str, **kwargs: Any) -> bytes:
if url == _feed(5):
raise httpx.HTTPStatusError("429", request=request,
response=httpx.Response(429, request=request))
return BOARD
with pytest.raises(httpx.HTTPStatusError):
_crawl(get, "stsgroup_tr")
def test_robots_are_read_from_the_api_host_whose_404_allows(web: dict[str, Any]) -> None:
request = httpx.Request("GET", "https://api.hr-manager.net/robots.txt")
web["answers"]["https://api.hr-manager.net/robots.txt"] = httpx.HTTPStatusError(
"404", request=request, response=httpx.Response(404, request=request))
state: dict[str, str] = {}
with httpx.Client() as client:
ats._check_robots(client, "hrmanager:stsgroup_tr", state)
assert '"allowed": true' in state["ats:robots:hrmanager:stsgroup_tr"]
assert web["asked"] == ["https://api.hr-manager.net/robots.txt"]
def test_the_robots_page_is_the_feed_so_that_its_own_hosts_rules_decide(
web: dict[str, Any]) -> None:
web["answers"]["https://api.hr-manager.net/robots.txt"] = \
b"User-agent: *\nDisallow: /JobPortal.svc/\n"
with httpx.Client() as client, pytest.raises(ats._Disallowed):
ats._check_robots(client, "hrmanager:stsgroup_tr")
def test_the_feed_states_no_place_and_the_robots_decision_is_the_owners() -> None:
rows = hrmanager.parse(BOARD, "stsgroup_tr")
assert all(r["places"] == [] for r in rows)
reads = hrmanager.READS
assert (reads.listing, reads.page_size, reads.text, reads.places) == (
"paged", 100, "listing", "one")
assert reads.robots == "disallowed by robots.txt; owner to decide"
assert not hasattr(hrmanager, "robots_page")
# ------------------------------------------------------- slugs and hosts
@@ -163,7 +138,6 @@ def test_the_module_is_registered_as_a_paged_host() -> None:
def test_the_urls_and_the_allowed_host() -> None:
assert hrmanager.careers_url("seaborg_tr") == \
"https://candidate.hr-manager.net/vacancies/list.aspx?customer=seaborg_tr"
assert hrmanager.robots_page("stsgroup_tr") == _feed(0)
allow = hrmanager.allowed("stsgroup_tr")
assert allow("api.hr-manager.net") and allow("API.HR-MANAGER.NET")
for host in ("candidate.hr-manager.net", "hr-manager.net", "api.hr-manager.net.evil.example",