Runner: readers read from their declaration, ats and search thin over it
plugins/readers/_runner.py takes the guarded fetch, the listing (whole, paged, one series per filter value unioned by id), completeness from READS, the sweep of a board over the page cap across pulls, the 10% guard on a filtered or empty="suspect" read, facets kept a day, the detail reads, the raw an Item carries and the failure bookkeeping out of plugins/ats.py. Ask comes from a subscription's params (ask_for, open for the reach), with a declared key resolved into Ask.params["key"] and refused when missing. The runtime robots.txt check goes (decided at build time, READS.robots). plugins/search.py reads the providers of readers.PROVIDERS through the runner and applies the saved pages param. A filtered board's close keeps roles with a vote or an application. DETAIL_FIELDS gain places, teaser, country and employment. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
779b136757
commit
902089a5e2
14 files changed
+1608
-1492
No files matched your search
@@ -133,25 +133,6 @@ def test_a_429_after_the_first_page_ends_the_read_incomplete(web: dict[str, Any]
|
||||
assert len(rows) == 5 and not complete
|
||||
|
||||
|
||||
def test_robots_are_read_from_the_api_host_whose_404_allows(web: dict[str, Any]) -> None:
|
||||
request = httpx.Request("GET", "https://api.hr-manager.net/robots.txt")
|
||||
web["answers"]["https://api.hr-manager.net/robots.txt"] = httpx.HTTPStatusError(
|
||||
"404", request=request, response=httpx.Response(404, request=request))
|
||||
state: dict[str, str] = {}
|
||||
with httpx.Client() as client:
|
||||
ats._check_robots(client, "hrmanager:stsgroup_tr", state)
|
||||
assert '"allowed": true' in state["ats:robots:hrmanager:stsgroup_tr"]
|
||||
assert web["asked"] == ["https://api.hr-manager.net/robots.txt"]
|
||||
|
||||
|
||||
def test_the_robots_page_is_the_feed_so_that_its_own_hosts_rules_decide(
|
||||
web: dict[str, Any]) -> None:
|
||||
web["answers"]["https://api.hr-manager.net/robots.txt"] = \
|
||||
b"User-agent: *\nDisallow: /JobPortal.svc/\n"
|
||||
with httpx.Client() as client, pytest.raises(ats._Disallowed):
|
||||
ats._check_robots(client, "hrmanager:stsgroup_tr")
|
||||
|
||||
|
||||
# ------------------------------------------------------- slugs and hosts
|
||||
|
||||
def test_the_module_is_registered_as_a_paged_host() -> None:
|
||||
|
||||
Reference in new issue
Block a user