Require READS, search params and a dated robots record; record robots.txt per requested path

Every reader declares, so undeclared(), the scope() fallback, robots_page and
ROBOTS_ANY_CASE go. A search provider needs PARAMS and clean_params, a row may
carry company, places gains none. Each robots record names the paths the
reader requests, from robots.txt read again on 2026-09-30. Platsbanken states
remote, HR Manager states no place.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-09-30 15:51:13 +02:00
1 parent 4e827665dd
commit 2349fc424b
46 files changed
+323 -100

No files matched your search

+5 -6
View File
@@ -87,8 +87,8 @@ def _get(pages: dict[str, bytes], asked: list[str] | None = None) -> Any:
def test_detail_reads_the_role_pages_json_ld() -> None:
rows = {r["source_id"]: r for r in jobylon.parse(BOARD, "3143")}
asked: list[str] = []
pages = {ROLE: (FIXTURES / "jobylon-role.html").read_bytes(),
ROLE_2: (FIXTURES / "jobylon-role-2.html").read_bytes()}
pages = {ROLE: (FIXTURES / "jobylon-detail.html").read_bytes(),
ROLE_2: (FIXTURES / "jobylon-detail-2.html").read_bytes()}
first = jobylon.detail(_get(pages, asked), "3143", rows["385191"])
assert asked == [ROLE]
assert set(first) <= set(ats.DETAIL_FIELDS) | {"places"}
@@ -116,8 +116,8 @@ def test_detail_without_a_job_posting_fails_the_read() -> None:
def test_the_board_is_one_request_and_each_role_is_read_on_its_own_page() -> None:
asked: list[str] = []
pages = {FEED: BOARD, ROLE: (FIXTURES / "jobylon-role.html").read_bytes(),
ROLE_2: (FIXTURES / "jobylon-role-2.html").read_bytes()}
pages = {FEED: BOARD, ROLE: (FIXTURES / "jobylon-detail.html").read_bytes(),
ROLE_2: (FIXTURES / "jobylon-detail-2.html").read_bytes()}
get = _get(pages, asked)
rows = jobylon.parse(get(jobylon.feed_url("3143", Ask())), "3143")
assert len(rows) == 5 and asked == [FEED]
@@ -150,8 +150,7 @@ def test_it_declares_a_whole_listing_with_a_teaser_and_detail() -> None:
assert (reads.listing, reads.text, reads.teaser, reads.places) == (
"whole", "detail", True, "many")
assert (reads.fields.posted, reads.fields.seniority) == ("trusted", "stated")
assert reads.robots.startswith("read: robots.txt allows /jobs/companies/")
assert not hasattr(jobylon, "robots_page")
assert "/jobs/companies/<id>/embed/v2/" in reads.robots
# ------------------------------------------------------- slugs and hosts