Runner: readers read from their declaration, ats and search thin over it
plugins/readers/_runner.py takes the guarded fetch, the listing (whole, paged, one series per filter value unioned by id), completeness from READS, the sweep of a board over the page cap across pulls, the 10% guard on a filtered or empty="suspect" read, facets kept a day, the detail reads, the raw an Item carries and the failure bookkeeping out of plugins/ats.py. Ask comes from a subscription's params (ask_for, open for the reach), with a declared key resolved into Ask.params["key"] and refused when missing. The runtime robots.txt check goes (decided at build time, READS.robots). plugins/search.py reads the providers of readers.PROVIDERS through the runner and applies the saved pages param. A filtered board's close keeps roles with a vote or an application. DETAIL_FIELDS gain places, teaser, country and employment. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
779b136757
commit
902089a5e2
14 files changed
+1608
-1492
No files matched your search
+11
-193
@@ -1,6 +1,6 @@
|
||||
"""The host modules of `plugins/readers/` and what `plugins/ats.py` does for
|
||||
every host: the registry and its contract check, the fetch a host is handed,
|
||||
the per-role detail reads, robots.txt, and Lever's EU instance. No network —
|
||||
the per-role detail reads, and Lever's EU instance. No network —
|
||||
a fake host stands in where a behaviour belongs to no real host yet."""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -11,7 +11,7 @@ import subprocess
|
||||
import sys
|
||||
import types
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
@@ -23,6 +23,7 @@ if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from plugins import ats, readers # noqa: E402
|
||||
from plugins.readers import _runner # noqa: E402
|
||||
from plugins.readers import lever # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
@@ -242,11 +243,9 @@ def test_the_bound_get_reaches_a_replaced_ats_get(monkeypatch: pytest.MonkeyPatc
|
||||
|
||||
# ------------------------------------------------------- a fake host
|
||||
|
||||
def _fake_host(*, detail: Callable[..., Any] | None = None,
|
||||
robots: bool = False) -> types.ModuleType:
|
||||
def _fake_host(*, detail: Callable[..., Any] | None = None) -> types.ModuleType:
|
||||
"""A host `fake` whose feed is a JSON list of ids, each role with a
|
||||
teaser for a description, and, with `detail`, a per-role page; with
|
||||
`robots`, its listing is the page robots.txt must allow."""
|
||||
teaser for a description, and, with `detail`, a per-role page."""
|
||||
module: Any = types.ModuleType("fake")
|
||||
module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", ()
|
||||
module.slug_ok = lambda slug: slug.isalnum()
|
||||
@@ -260,8 +259,6 @@ def _fake_host(*, detail: Callable[..., Any] | None = None,
|
||||
for i in json.loads(body)]
|
||||
if detail is not None:
|
||||
module.detail = detail
|
||||
if robots:
|
||||
module.robots_page = lambda slug: f"https://jobs.fake.example/{slug}/list"
|
||||
return module
|
||||
|
||||
|
||||
@@ -411,8 +408,8 @@ def test_an_incomplete_listing_keeps_the_ids_it_did_not_list(
|
||||
ids stay, and the next pull does not read them in detail again."""
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["2"]'})
|
||||
monkeypatch.setattr(ats, "_fetch_rows", lambda client, target: (
|
||||
readers.HOSTS["fake"].parse(b'["2"]', "acme"), False))
|
||||
monkeypatch.setattr(_runner, "read_listing", lambda *args, **kwargs: _runner.Listing(
|
||||
readers.HOSTS["fake"].parse(b'["2"]', "acme"), False, ["2"]))
|
||||
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
|
||||
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
|
||||
@@ -511,7 +508,7 @@ def test_detail_fills_only_the_fields_a_later_pull_keeps(
|
||||
"salary_text": "EUR 60,000 to 70,000 per year", "department": "Sales",
|
||||
"title": "Another title"}
|
||||
|
||||
caplog.set_level("DEBUG", logger=ats.log.name)
|
||||
caplog.set_level("DEBUG", logger=_runner.log.name)
|
||||
[item], _state = _run(monkeypatch, _fake_host(detail=detail))
|
||||
assert (item.body, item.title, item.published_at) == (
|
||||
"in full", "Role 1", datetime(2026, 9, 1, tzinfo=UTC))
|
||||
@@ -519,8 +516,9 @@ def test_detail_fills_only_the_fields_a_later_pull_keeps(
|
||||
item.raw["salary_max"]) == ("hybrid", "EUR", 60000, 70000)
|
||||
assert (item.raw["salary_text"], item.raw["department"]) == ("", "")
|
||||
assert "department, salary_text, title left out" in caplog.text
|
||||
assert set(ats.DETAIL_FIELDS) == {"description", "location", "remote", "currency",
|
||||
"salary_min", "salary_max", "posted"}
|
||||
assert set(ats.DETAIL_FIELDS) == {"description", "location", "places", "teaser", "remote",
|
||||
"country", "employment", "currency", "salary_min",
|
||||
"salary_max", "posted"}
|
||||
|
||||
|
||||
def _paged_host() -> types.ModuleType:
|
||||
@@ -583,186 +581,6 @@ def test_a_pull_stopped_half_way_through_a_board_keeps_no_detail_ids(
|
||||
assert "ats:detailed:fake:acme" not in source.state
|
||||
|
||||
|
||||
# ------------------------------------------------------------ robots.txt
|
||||
|
||||
ROBOTS = "https://jobs.fake.example/robots.txt"
|
||||
|
||||
|
||||
def test_robots_that_allow_the_page_let_the_board_be_read(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /admin/\n",
|
||||
"https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
items, state = _run(monkeypatch, _fake_host(robots=True))
|
||||
assert len(items) == 1
|
||||
assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("robots", [
|
||||
b"User-agent: *\nDisallow: /acme/\n",
|
||||
# A rule for this app by name counts as well as the one for everyone.
|
||||
f"User-agent: {ats.USER_AGENT.split('/')[0]}\nDisallow: /\n\nUser-agent: *\nAllow: /\n"
|
||||
.encode(),
|
||||
])
|
||||
def test_robots_that_disallow_the_page_fail_the_board(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], robots: bytes) -> None:
|
||||
web["answers"].update({ROBOTS: robots, "https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
items, state = _run(monkeypatch, _fake_host(robots=True))
|
||||
assert items == []
|
||||
assert web["asked"] == [ROBOTS] # the listing is never fetched
|
||||
assert json.loads(state["ats:failing:fake:acme"])["reason"] == "robots.txt disallows it"
|
||||
assert "ats:ok:fake:acme" not in state
|
||||
|
||||
|
||||
def test_a_host_that_serves_any_case_is_forbidden_by_a_rule_in_any_case(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /ACME/\n",
|
||||
"https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
host = _fake_host(robots=True)
|
||||
items, _state = _run(monkeypatch, host)
|
||||
assert len(items) == 1 # paths are case-sensitive by default
|
||||
host.ROBOTS_ANY_CASE = True
|
||||
web["asked"].clear()
|
||||
items, state = _run(monkeypatch, host)
|
||||
assert items == []
|
||||
assert json.loads(state["ats:failing:fake:acme"])["reason"] == "robots.txt disallows it"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("robots", "page", "allowed"), [
|
||||
# The longest matching rule wins, whatever the order of the lines.
|
||||
("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n",
|
||||
"https://jobs.infineon.com/api/pcsx/search?domain=infineon.com", True),
|
||||
("User-agent: *\nAllow: /careers\nDisallow: /\n", "https://x.example/careers/list", True),
|
||||
("User-agent: *\nDisallow: /\nAllow: /api/pcsx\n", "https://x.example/other", False),
|
||||
# An allow wins a tie.
|
||||
("User-agent: *\nDisallow: /jobs\nAllow: /jobs\n", "https://x.example/jobs", True),
|
||||
# `*` inside a path, and the query counts as part of it.
|
||||
("User-agent: *\nDisallow: /*company\n",
|
||||
"https://career5.successfactors.eu/career?company=x", False),
|
||||
# `$` anchors the end.
|
||||
("User-agent: *\nDisallow: /*.json$\n", "https://x.example/a/jobs.json", False),
|
||||
("User-agent: *\nDisallow: /*.json$\n", "https://x.example/a/jobs.json?page=2", True),
|
||||
# An empty Disallow allows.
|
||||
("User-agent: *\nDisallow:\n", "https://x.example/anything", True),
|
||||
# A byte order mark and CRLF line ends.
|
||||
("\ufeffUser-agent: *\r\nDisallow: /\r\n", "https://x.example/jobs", False),
|
||||
# Paths are case-sensitive.
|
||||
("User-agent: *\nDisallow: /Novonesis_Careers/\n",
|
||||
"https://novonesis.wd103.myworkdayjobs.com/Novonesis_Careers/", False),
|
||||
("User-agent: *\nDisallow: /Novonesis_Careers/\n",
|
||||
"https://novonesis.wd103.myworkdayjobs.com/novonesis_careers/", True),
|
||||
# A group for this app by name counts, a group for someone else does
|
||||
# not, and a token matches whole, not as a substring.
|
||||
(f"User-agent: {ats.ROBOTS_TOKEN.upper()}\nDisallow: /\n\nUser-agent: *\nAllow: /\n",
|
||||
"https://x.example/jobs", False),
|
||||
("User-agent: Googlebot\nDisallow: /\n", "https://x.example/jobs", True),
|
||||
("User-agent: sk\nDisallow: /\n\nUser-agent: *\nAllow: /\n", "https://x.example/jobs", True),
|
||||
# Two agents share one group, and a comment is no rule.
|
||||
("User-agent: a\nUser-agent: *\nDisallow: /jobs # not these\n",
|
||||
"https://x.example/jobs/1", False),
|
||||
# A group for this app that allows does not undo the `*` group's refusal.
|
||||
(f"User-agent: {ats.ROBOTS_TOKEN}\nAllow: /\n\nUser-agent: *\nDisallow: /\n",
|
||||
"https://x.example/jobs", False),
|
||||
# Rules before any user-agent line belong to no group.
|
||||
("Disallow: /\nUser-agent: *\nAllow: /\n", "https://x.example/jobs", True),
|
||||
])
|
||||
def test_robots_rules_follow_rfc_9309(robots: str, page: str, allowed: bool) -> None:
|
||||
assert ats._robots_allows(robots.encode(), page) is allowed
|
||||
|
||||
|
||||
def test_a_robots_verdict_is_kept_for_a_day(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n",
|
||||
"https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
module = _fake_host(robots=True)
|
||||
_items, state = _run(monkeypatch, module)
|
||||
assert web["asked"] == [ROBOTS]
|
||||
|
||||
# Within the day the kept verdict holds, even though the site changed.
|
||||
web["answers"][ROBOTS] = b"User-agent: *\nAllow: /\n"
|
||||
web["asked"].clear()
|
||||
items, state = _run(monkeypatch, module, state)
|
||||
assert (items, web["asked"]) == ([], [])
|
||||
|
||||
# A day on, robots.txt is read again.
|
||||
kept = json.loads(state["ats:robots:fake:acme"])
|
||||
kept["checked"] = (datetime.now(UTC) - timedelta(days=1, minutes=1)).isoformat()
|
||||
state["ats:robots:fake:acme"] = json.dumps(kept)
|
||||
items, state = _run(monkeypatch, module, state)
|
||||
assert web["asked"] == [ROBOTS, "https://jobs.fake.example/acme/list"]
|
||||
assert len(items) == 1 and "ats:failing:fake:acme" not in state
|
||||
|
||||
|
||||
@pytest.mark.parametrize("code", [404, 410, 401, 403])
|
||||
def test_a_robots_txt_answering_4xx_allows_and_is_kept(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], code: int) -> None:
|
||||
web["answers"].update({ROBOTS: _status(code), "https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
items, state = _run(monkeypatch, _fake_host(robots=True))
|
||||
assert len(items) == 1
|
||||
assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failure", [
|
||||
_status(500), _status(503), _status(429), httpx.ConnectTimeout("timed out"),
|
||||
httpx.ConnectError("refused"), ats._Refused("https://elsewhere.example/robots.txt"),
|
||||
])
|
||||
def test_a_robots_txt_that_cannot_be_read_fails_the_board_and_is_asked_again(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], failure: Exception) -> None:
|
||||
web["answers"].update({ROBOTS: failure, "https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
items, state = _run(monkeypatch, _fake_host(robots=True))
|
||||
assert items == [] and web["asked"] == [ROBOTS]
|
||||
assert json.loads(state["ats:failing:fake:acme"])["reason"] == \
|
||||
"robots.txt could not be read"
|
||||
assert "ats:robots:fake:acme" not in state
|
||||
|
||||
|
||||
def _robots_site(location: str) -> Callable[[httpx.Request], httpx.Response]:
|
||||
"""A site whose robots.txt on jobs.fake.example redirects to `location`,
|
||||
where a robots.txt that disallows everything is served."""
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if str(request.url) == ROBOTS:
|
||||
return httpx.Response(301, headers={"location": location})
|
||||
if request.url.path == "/robots.txt":
|
||||
return httpx.Response(200, text="User-agent: *\nDisallow: /\n")
|
||||
return httpx.Response(200, text='["1"]')
|
||||
return handler
|
||||
|
||||
|
||||
def test_a_robots_txt_redirect_to_the_www_twin_is_followed(
|
||||
monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(robots=True))
|
||||
with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \
|
||||
pytest.raises(ats._Disallowed):
|
||||
ats._check_robots(client, "fake:acme", {})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("location", [
|
||||
"https://evil.example/robots.txt", "https://other.fake.example/robots.txt",
|
||||
"http://www.jobs.fake.example/robots.txt",
|
||||
])
|
||||
def test_a_robots_txt_redirect_anywhere_else_is_unreachable(
|
||||
monkeypatch: pytest.MonkeyPatch, location: str) -> None:
|
||||
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(robots=True))
|
||||
with _client(_robots_site(location)) as client, pytest.raises(ats._RobotsUnreachable):
|
||||
ats._check_robots(client, "fake:acme", {})
|
||||
|
||||
|
||||
def test_the_www_twin_is_for_robots_txt_only(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(robots=True))
|
||||
with _client(_robots_site("https://www.jobs.fake.example/robots.txt")) as client, \
|
||||
pytest.raises(ats._Refused):
|
||||
ats._get(client, ROBOTS, "fake", "acme")
|
||||
|
||||
|
||||
def test_fetch_board_honours_robots_txt(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n",
|
||||
"https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
monkeypatch.setitem(readers.HOSTS, "fake", _fake_host(robots=True))
|
||||
with pytest.raises(ats._Disallowed) as caught:
|
||||
ats.fetch_board("fake:acme")
|
||||
assert ats.failure_reason(caught.value) == "robots.txt disallows it"
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- Lever EU
|
||||
|
||||
def test_a_lever_slug_may_name_the_eu_instance() -> None:
|
||||
|
||||
Reference in new issue
Block a user