Read a role's own page once for a host with detail, and fail a board whose robots.txt disallows it
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b02b72b4aa
commit
8b14d8575b
2 files changed
+578
-6
No files matched your search
+153
-6
@@ -6,7 +6,8 @@ unauthenticated JSON, XML or RSS — first-party, no scraping, and a role that
|
||||
drops off the board is *closed*, which is the cheapest staleness check there
|
||||
is. What a host says and how its feed reads is its module's; this file holds
|
||||
what every host shares: the fetch and its allow-list, paging and
|
||||
completeness, the classifier, and the plugin itself.
|
||||
completeness, the per-role detail reads, robots.txt, the classifier, and the
|
||||
plugin itself.
|
||||
|
||||
The allow-list and the classifier are ported from `openings.go` and
|
||||
`cmd/seedgen/main.go` in
|
||||
@@ -32,9 +33,10 @@ import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Callable, Iterator
|
||||
from datetime import UTC, datetime
|
||||
from collections.abc import Callable, Iterator, MutableMapping
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
from urllib.robotparser import RobotFileParser
|
||||
|
||||
import httpx
|
||||
|
||||
@@ -110,6 +112,11 @@ class _TooLarge(ValueError):
|
||||
retry at the host's `fallback_url`."""
|
||||
|
||||
|
||||
class _Disallowed(Exception):
|
||||
"""The site's robots.txt disallows the page a board is read from (see
|
||||
`_check_robots`)."""
|
||||
|
||||
|
||||
def _get(client: httpx.Client, url: str, host: str, slug: str, *,
|
||||
method: str = "GET", payload: dict[str, Any] | None = None,
|
||||
headers: dict[str, str] | None = None) -> bytes:
|
||||
@@ -329,7 +336,8 @@ def careers_url(target: str) -> str:
|
||||
#: Pages read from one paged board at most, and the pause between two of
|
||||
#: them. Workable answers ten roles a page, SmartRecruiters a hundred and
|
||||
#: Teamtailor fifty, so the cap is 1,000, 10,000 and 5,000 roles; the pause
|
||||
#: keeps Workable, which answers a burst with 429, from seeing one.
|
||||
#: keeps Workable, which answers a burst with 429, from seeing one. The same
|
||||
#: pause separates two detail reads.
|
||||
MAX_PAGES = 100
|
||||
PAGE_DELAY = 0.5
|
||||
|
||||
@@ -408,18 +416,142 @@ def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
|
||||
return module.parse(body, slug), True
|
||||
|
||||
|
||||
# -------------------------------------------------------------- robots.txt
|
||||
|
||||
#: How long a robots.txt answer holds for one board before it is read again.
|
||||
ROBOTS_TTL = timedelta(days=1)
|
||||
|
||||
|
||||
def _robots_allows(body: bytes, page: str) -> bool:
|
||||
"""Whether a robots.txt lets `page` be fetched: by its rules for every
|
||||
agent (`*`), and by any it has for this app's own User-Agent token, so a
|
||||
site that shuts out this app by name is honoured as well."""
|
||||
parser = RobotFileParser()
|
||||
parser.parse(body.decode("utf-8", "replace").splitlines())
|
||||
return parser.can_fetch("*", page) and parser.can_fetch(USER_AGENT, page)
|
||||
|
||||
|
||||
def _check_robots(client: httpx.Client, target: str,
|
||||
state: MutableMapping[str, str] | None = None) -> None:
|
||||
"""Raise `_Disallowed` when the site of a host with a `robots_page` has a
|
||||
robots.txt that disallows that page. Read through `_get`, so from the
|
||||
board's own allowed hosts only, and at most once per `ROBOTS_TTL` for a
|
||||
board when `state` is given: the verdict and when it was reached are kept
|
||||
under `ats:robots:<target>`. A robots.txt that is missing (404 or 410)
|
||||
allows, and that verdict is kept like any other; one that cannot be read
|
||||
for another reason allows too, but is not kept, so the next pull asks
|
||||
again."""
|
||||
host, _, slug = target.partition(":")
|
||||
module = atshosts.HOSTS.get(host)
|
||||
if module is None or not hasattr(module, "robots_page") or not is_known_target(target):
|
||||
return
|
||||
page = module.robots_page(slug)
|
||||
key = f"ats:robots:{target}"
|
||||
now = datetime.now(UTC)
|
||||
if state is not None:
|
||||
try:
|
||||
kept = json.loads(state.get(key) or "{}")
|
||||
fresh = now - datetime.fromisoformat(kept["checked"]) < ROBOTS_TTL
|
||||
except (ValueError, KeyError, TypeError):
|
||||
kept, fresh = {}, False
|
||||
if fresh:
|
||||
if not kept.get("allowed", True):
|
||||
raise _Disallowed(f"robots.txt disallows {page}")
|
||||
return
|
||||
url = f"https://{httpx.URL(page).host}/robots.txt"
|
||||
try:
|
||||
body = _getter(client, host, slug)(url, headers={"Accept": "text/plain"})
|
||||
except httpx.HTTPStatusError as exc:
|
||||
if exc.response.status_code not in (404, 410):
|
||||
log.warning("ats: %s: robots.txt answered %s; allowed until the next pull",
|
||||
target, exc.response.status_code)
|
||||
return
|
||||
allowed = True
|
||||
except Exception as exc:
|
||||
log.warning("ats: %s: robots.txt not read (%s); allowed until the next pull",
|
||||
target, exc)
|
||||
return
|
||||
else:
|
||||
allowed = _robots_allows(body, page)
|
||||
if state is not None:
|
||||
state[key] = json.dumps({"checked": now.isoformat(timespec="seconds"),
|
||||
"allowed": allowed})
|
||||
if not allowed:
|
||||
raise _Disallowed(f"robots.txt disallows {page}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- details
|
||||
|
||||
#: Roles read in detail from one board in one pull at most. The rest of a
|
||||
#: large new board is read over the next pulls, 200 at a time.
|
||||
DETAIL_CAP = 200
|
||||
|
||||
|
||||
def _read_details(client: httpx.Client, target: str, rows: list[Row], complete: bool,
|
||||
detailed: set[str]) -> tuple[list[Row], set[str]]:
|
||||
"""The rows of a board whose host has `detail`, each role not yet in
|
||||
`detailed` (compound ids) filled from its own page, at most `DETAIL_CAP`
|
||||
of them and `PAGE_DELAY` apart. Returns the rows and the ids read in
|
||||
detail so far, for `ats:detailed:<target>`.
|
||||
|
||||
Every row not read in detail in this call goes out with an empty
|
||||
description: a role read on an earlier pull keeps the one it has, since
|
||||
`upsert_role` never overwrites a filled column with an empty value, and a
|
||||
teaser from the listing must not replace it. A detail that raises is
|
||||
logged and the role left for the next pull; a 429 stops the reads for this
|
||||
board until then. None of this touches completeness, which is the
|
||||
listing's alone. The ids of roles no longer listed are dropped, but only
|
||||
after a complete listing, so one cut short does not make the next pull
|
||||
read its missing roles again."""
|
||||
host, _, slug = target.partition(":")
|
||||
module = atshosts.HOSTS[host]
|
||||
get = _getter(client, host, slug)
|
||||
done = set(detailed)
|
||||
out: list[Row] = []
|
||||
tried = 0
|
||||
stopped = False
|
||||
for row in rows:
|
||||
compound = f"{host}:{slug}:{row['source_id']}"
|
||||
if compound in done or stopped or tried >= DETAIL_CAP:
|
||||
out.append(row | {"description": ""})
|
||||
continue
|
||||
time.sleep(PAGE_DELAY)
|
||||
tried += 1
|
||||
try:
|
||||
filled = module.detail(get, slug, row)
|
||||
except Exception as exc:
|
||||
if isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code == 429:
|
||||
log.warning("ats: %s: 429 on a detail read; the rest wait for the next pull",
|
||||
target)
|
||||
stopped = True
|
||||
else:
|
||||
log.warning("ats: %s: detail of %s failed (%s)", target, row["source_id"], exc)
|
||||
out.append(row | {"description": ""})
|
||||
continue
|
||||
out.append(row | filled | {"source_id": row["source_id"]})
|
||||
done.add(compound)
|
||||
if complete:
|
||||
done &= {f"{host}:{slug}:{row['source_id']}" for row in rows}
|
||||
return out, done
|
||||
|
||||
|
||||
def fetch_board(target: str) -> tuple[list[Row], bool]:
|
||||
"""Fetch and parse one board directly: no `Source`, no `self.state`, just
|
||||
"does this board answer, what does it list, and is that all of it". For
|
||||
`app companies add|check`, which ask that question about one board at a
|
||||
time outside any pull. Raises the same way `_fetch_rows` does."""
|
||||
time outside any pull. The listing only, with no detail reads, and
|
||||
robots.txt read afresh. Raises the same way `_fetch_rows` does, and
|
||||
`_Disallowed` for a board robots.txt shuts out."""
|
||||
with httpx.Client() as client:
|
||||
_check_robots(client, target)
|
||||
return _fetch_rows(client, target)
|
||||
|
||||
|
||||
def failure_reason(exc: BaseException) -> str:
|
||||
"""One short line for why a board did not answer: the status of an HTTP
|
||||
error, else the exception's class and the first line of its message."""
|
||||
if isinstance(exc, _Disallowed):
|
||||
return "robots.txt disallows it"
|
||||
if isinstance(exc, httpx.HTTPStatusError):
|
||||
return f"{exc.response.status_code} {exc.response.reason_phrase}".strip()
|
||||
lines = str(exc).strip().splitlines()
|
||||
@@ -465,6 +597,7 @@ class ATSSource(Source):
|
||||
host, _, slug = target.partition(":")
|
||||
failing = f"ats:failing:{target}"
|
||||
try:
|
||||
_check_robots(client, target, self.state)
|
||||
rows, complete = _fetch_rows(client, target)
|
||||
except Exception as exc:
|
||||
log.warning("ats: %s failed (%s)", target, exc)
|
||||
@@ -481,6 +614,15 @@ class ATSSource(Source):
|
||||
continue
|
||||
self.state.pop(failing, None)
|
||||
|
||||
detailed_key = f"ats:detailed:{target}"
|
||||
detailed: set[str] | None = None
|
||||
if hasattr(atshosts.HOSTS.get(host), "detail"):
|
||||
try:
|
||||
detailed = set(json.loads(self.state.get(detailed_key) or "[]"))
|
||||
except (ValueError, TypeError):
|
||||
detailed = set()
|
||||
rows, detailed = _read_details(client, target, rows, complete, detailed)
|
||||
|
||||
seen_ids = []
|
||||
for row in rows:
|
||||
# The compound id, matching `roles.source_id` — what
|
||||
@@ -520,7 +662,12 @@ class ATSSource(Source):
|
||||
# board that blew up half way through is neither marked ok nor
|
||||
# has a partial id list treated as the whole of it. `complete`
|
||||
# says whether the ids are the whole board, which the pull
|
||||
# needs before it closes what they leave out.
|
||||
# needs before it closes what they leave out. The ids read in
|
||||
# detail are kept here too, after every row went out, so a
|
||||
# pull stopped half way reads the rest again rather than
|
||||
# losing their descriptions.
|
||||
self.state[f"ats:ok:{target}"] = run_marker
|
||||
self.state[f"ats:complete:{target}"] = "1" if complete else ""
|
||||
self.state[f"ats:ids:{target}"] = json.dumps(seen_ids)
|
||||
if detailed is not None:
|
||||
self.state[detailed_key] = json.dumps(sorted(detailed))
|
||||
@@ -0,0 +1,425 @@
|
||||
"""The host modules of `plugins/atshosts/` and what `plugins/ats.py` does for
|
||||
every host: the registry and its contract check, the fetch a host is handed,
|
||||
the per-role detail reads, robots.txt, and Lever's EU instance. No network —
|
||||
a fake host stands in where a behaviour belongs to no real host yet."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import sys
|
||||
import types
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from plugins import ats, atshosts # noqa: E402
|
||||
from plugins.atshosts import lever # noqa: E402
|
||||
|
||||
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||
|
||||
|
||||
# --------------------------------------------------------------- registry
|
||||
|
||||
def test_the_registry_finds_every_module_in_order_of_name() -> None:
|
||||
names = sorted(p.stem for p in (ROOT / "plugins" / "atshosts").glob("*.py")
|
||||
if not p.stem.startswith("_"))
|
||||
assert list(atshosts.HOSTS) == names
|
||||
assert set(names) >= {"greenhouse", "lever", "ashby", "workable", "smartrecruiters",
|
||||
"recruitee", "personio", "teamtailor"}
|
||||
assert all(not atshosts.problems(module, name) for name, module in atshosts.HOSTS.items())
|
||||
# The Sources page's select and the plugin's description follow the modules.
|
||||
assert {name: module.LABEL for name, module in atshosts.HOSTS.items()} == ats.HOSTS
|
||||
assert "Teamtailor" in ats.ATSSource.description
|
||||
|
||||
|
||||
def _package(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, name: str,
|
||||
modules: dict[str, str]) -> list[str]:
|
||||
package = tmp_path / name
|
||||
package.mkdir()
|
||||
(package / "__init__.py").write_text("")
|
||||
for module, source in modules.items():
|
||||
(package / f"{module}.py").write_text(source)
|
||||
monkeypatch.syspath_prepend(str(tmp_path))
|
||||
return [str(package)]
|
||||
|
||||
|
||||
_GOOD = """
|
||||
NAME = "{name}"
|
||||
LABEL = "Good"
|
||||
PATTERNS = (r"good\\.example/([a-z]+)",)
|
||||
def slug_ok(slug): return True
|
||||
def allowed(slug): return lambda h: h == "good.example"
|
||||
def careers_url(slug): return "https://good.example/" + slug
|
||||
def feed_url(slug): return "https://good.example/" + slug + ".json"
|
||||
def parse(body, slug): return []
|
||||
"""
|
||||
|
||||
|
||||
def test_the_registry_loads_a_package_by_name_and_skips_private_modules(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
path = _package(tmp_path, monkeypatch, "hostsok", {
|
||||
"zeta": _GOOD.format(name="zeta"), "alpha": _GOOD.format(name="alpha"),
|
||||
"_helpers": "X = 1\n"})
|
||||
assert list(atshosts.load(path, "hostsok")) == ["alpha", "zeta"]
|
||||
|
||||
|
||||
def test_the_registry_refuses_a_module_missing_a_required_name(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
broken = _GOOD.format(name="broken").replace("def careers_url", "def not_careers_url")
|
||||
path = _package(tmp_path, monkeypatch, "hostsbad", {"broken": broken})
|
||||
with pytest.raises(ImportError, match=r"hostsbad\.broken .*no careers_url"):
|
||||
atshosts.load(path, "hostsbad")
|
||||
|
||||
|
||||
def test_the_contract_check_names_each_problem() -> None:
|
||||
module = types.ModuleType("odd")
|
||||
module.NAME, module.LABEL, module.PATTERNS = "other", "", ("(",)
|
||||
module.slug_ok = module.allowed = module.careers_url = lambda slug: ""
|
||||
module.page = lambda get, slug, cursor: None
|
||||
module.probe_ok = lambda body: True
|
||||
found = atshosts.problems(module, "odd")
|
||||
assert "NAME is 'other', not the module's name 'odd'" in found
|
||||
assert "LABEL is not a non-empty string" in found
|
||||
assert any(p.startswith("PATTERNS has '('") for p in found)
|
||||
assert "page without rows" in found
|
||||
assert "probe_ok without probe_url" in found
|
||||
|
||||
|
||||
# ------------------------------------------------------------ the fetch
|
||||
|
||||
def _client(handler: Callable[[httpx.Request], httpx.Response]) -> httpx.Client:
|
||||
return httpx.Client(transport=httpx.MockTransport(handler))
|
||||
|
||||
|
||||
def test_get_sends_extra_headers_on_top_of_its_own() -> None:
|
||||
seen: list[httpx.Headers] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
seen.append(request.headers)
|
||||
return httpx.Response(200, content=b"ok")
|
||||
|
||||
with _client(handler) as client:
|
||||
get = ats._getter(client, "lever", "acme")
|
||||
assert get("https://api.lever.co/v0/postings/acme",
|
||||
headers={"Accept": "text/plain", "X-Board": "acme"}) == b"ok"
|
||||
get("https://api.lever.co/v0/postings/acme")
|
||||
assert seen[0]["accept"] == "text/plain" and seen[0]["x-board"] == "acme"
|
||||
assert seen[0]["user-agent"] == seen[1]["user-agent"] == ats.USER_AGENT
|
||||
assert seen[1]["accept"] == "application/json, text/xml"
|
||||
assert "x-board" not in seen[1]
|
||||
|
||||
|
||||
def test_the_bound_get_reaches_a_replaced_ats_get(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Bound before the replacement, and still reaching it: `_get` is looked
|
||||
up when a request is made."""
|
||||
client = object()
|
||||
get = ats._getter(client, "lever", "acme") # type: ignore[arg-type]
|
||||
calls: list[tuple[Any, ...]] = []
|
||||
|
||||
def replaced(*args: Any, **kwargs: Any) -> bytes:
|
||||
calls.append((args, kwargs))
|
||||
return b""
|
||||
|
||||
monkeypatch.setattr(ats, "_get", replaced)
|
||||
get("https://api.lever.co/v0/postings/acme", headers={"X": "1"})
|
||||
get("https://api.lever.co/v0/postings/acme", method="POST", payload={})
|
||||
assert calls == [
|
||||
((client, "https://api.lever.co/v0/postings/acme", "lever", "acme"),
|
||||
{"method": "GET", "headers": {"X": "1"}}),
|
||||
((client, "https://api.lever.co/v0/postings/acme", "lever", "acme"),
|
||||
{"method": "POST", "payload": {}})]
|
||||
|
||||
|
||||
# ------------------------------------------------------- a fake host
|
||||
|
||||
def _fake_host(*, detail: Callable[..., Any] | None = None,
|
||||
robots: bool = False) -> types.ModuleType:
|
||||
"""A host `fake` whose feed is a JSON list of ids, each role with a
|
||||
teaser for a description, and, with `detail`, a per-role page; with
|
||||
`robots`, its listing is the page robots.txt must allow."""
|
||||
module = types.ModuleType("fake")
|
||||
module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", ()
|
||||
module.slug_ok = lambda slug: slug.isalnum()
|
||||
module.allowed = lambda slug: lambda h: h == "jobs.fake.example"
|
||||
module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}"
|
||||
module.feed_url = lambda slug: f"https://jobs.fake.example/{slug}/list"
|
||||
module.parse = lambda body, slug: [
|
||||
{"source_id": i, "title": f"Role {i}", "location": "", "department": "",
|
||||
"url": f"https://jobs.fake.example/{slug}/{i}", "posted": None,
|
||||
"description": "teaser", "remote": None, "salary_text": ""}
|
||||
for i in json.loads(body)]
|
||||
if detail is not None:
|
||||
module.detail = detail
|
||||
if robots:
|
||||
module.robots_page = lambda slug: f"https://jobs.fake.example/{slug}/list"
|
||||
return module
|
||||
|
||||
|
||||
def _status(code: int, url: str = "https://jobs.fake.example/") -> httpx.HTTPStatusError:
|
||||
request = httpx.Request("GET", url)
|
||||
return httpx.HTTPStatusError(str(code), request=request,
|
||||
response=httpx.Response(code, request=request))
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]:
|
||||
"""`ats._get` answering from a dict of URL -> body or exception, and
|
||||
remembering what it was asked. No delay between requests."""
|
||||
answers: dict[str, Any] = {}
|
||||
asked: list[str] = []
|
||||
|
||||
def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None):
|
||||
asked.append(url)
|
||||
if not ats._allowed_host(host, slug)(httpx.URL(url).host):
|
||||
raise ats._Refused(url)
|
||||
answer = answers.get(url, _status(404, url))
|
||||
if isinstance(answer, BaseException):
|
||||
raise answer
|
||||
return answer
|
||||
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
|
||||
return {"answers": answers, "asked": asked}
|
||||
|
||||
|
||||
def _run(monkeypatch: pytest.MonkeyPatch, module: types.ModuleType,
|
||||
state: dict[str, str] | None = None) -> tuple[list[Any], dict[str, str]]:
|
||||
monkeypatch.setitem(atshosts.HOSTS, "fake", module)
|
||||
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"})
|
||||
if state is not None:
|
||||
source.state = state
|
||||
return list(source.fetch()), source.state
|
||||
|
||||
|
||||
def _detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]:
|
||||
"""The fake host's detail: the role's own page is its description."""
|
||||
return {"description": get(f"https://jobs.fake.example/{slug}/{row['source_id']}").decode(),
|
||||
"location": "Copenhagen"}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- detail
|
||||
|
||||
def test_detail_is_read_for_unread_roles_only(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one in full",
|
||||
f"{base}/2": b"two in full", f"{base}/3": b"three in full"})
|
||||
module = _fake_host(detail=_detail)
|
||||
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"])}
|
||||
items, state = _run(monkeypatch, module, state)
|
||||
|
||||
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/3"]
|
||||
assert [(i.source_id, i.body) for i in items] == [
|
||||
("fake:acme:1", "one in full"), ("fake:acme:2", ""), ("fake:acme:3", "three in full")]
|
||||
# What detail returns replaces the listing's field; a role not read keeps the listing's.
|
||||
assert items[0].raw["place"] == "Copenhagen" and items[1].raw["place"] == ""
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == [
|
||||
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
|
||||
assert state["ats:complete:fake:acme"] == "1"
|
||||
|
||||
|
||||
def test_detail_reads_stop_at_the_cap_and_the_rest_wait(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
ids = [str(n) for n in range(5)]
|
||||
web["answers"].update({f"{base}/list": json.dumps(ids).encode()}
|
||||
| {f"{base}/{i}": f"full {i}".encode() for i in ids})
|
||||
monkeypatch.setattr(ats, "DETAIL_CAP", 2)
|
||||
module = _fake_host(detail=_detail)
|
||||
items, state = _run(monkeypatch, module)
|
||||
assert [i.body for i in items] == ["full 0", "full 1", "", "", ""]
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:0", "fake:acme:1"]
|
||||
|
||||
web["asked"].clear()
|
||||
items, state = _run(monkeypatch, module, state)
|
||||
assert web["asked"] == [f"{base}/list", f"{base}/2", f"{base}/3"]
|
||||
assert [i.body for i in items] == ["", "", "full 2", "full 3", ""]
|
||||
|
||||
|
||||
def test_a_detail_that_fails_skips_the_role_and_not_the_board(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one",
|
||||
f"{base}/2": ValueError("no such page shape"), f"{base}/3": b"three"})
|
||||
items, state = _run(monkeypatch, _fake_host(detail=_detail))
|
||||
assert [(i.source_id, i.body) for i in items] == [
|
||||
("fake:acme:1", "one"), ("fake:acme:2", ""), ("fake:acme:3", "three")]
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:3"]
|
||||
assert "ats:failing:fake:acme" not in state and state["ats:ok:fake:acme"] == "r"
|
||||
assert state["ats:complete:fake:acme"] == "1"
|
||||
|
||||
|
||||
def test_a_429_on_a_detail_stops_the_reads_for_the_pull(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one",
|
||||
f"{base}/2": _status(429), f"{base}/3": b"three"})
|
||||
items, state = _run(monkeypatch, _fake_host(detail=_detail))
|
||||
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/2"]
|
||||
assert [i.body for i in items] == ["one", "", ""]
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
|
||||
assert state["ats:complete:fake:acme"] == "1"
|
||||
|
||||
|
||||
def test_ids_no_longer_listed_leave_the_detailed_list(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["2"]'})
|
||||
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
|
||||
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:2"]
|
||||
|
||||
|
||||
def test_an_incomplete_listing_keeps_the_ids_it_did_not_list(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
"""A listing cut short says nothing about the roles it left out, so their
|
||||
ids stay, and the next pull does not read them in detail again."""
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["2"]'})
|
||||
monkeypatch.setattr(ats, "_fetch_rows", lambda client, target: (
|
||||
atshosts.HOSTS["fake"].parse(b'["2"]', "acme"), False))
|
||||
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
|
||||
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
|
||||
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
|
||||
assert state["ats:complete:fake:acme"] == ""
|
||||
|
||||
|
||||
def test_fetch_board_reads_the_listing_only(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
base = "https://jobs.fake.example/acme"
|
||||
web["answers"].update({f"{base}/list": b'["1"]', f"{base}/1": b"one"})
|
||||
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(detail=_detail))
|
||||
rows, complete = ats.fetch_board("fake:acme")
|
||||
assert ([r["description"] for r in rows], complete) == (["teaser"], True)
|
||||
assert web["asked"] == [f"{base}/list"]
|
||||
|
||||
|
||||
def test_a_host_without_detail_keeps_the_listings_description(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
web["answers"]["https://jobs.fake.example/acme/list"] = b'["1"]'
|
||||
items, state = _run(monkeypatch, _fake_host())
|
||||
assert [i.body for i in items] == ["teaser"]
|
||||
assert "ats:detailed:fake:acme" not in state
|
||||
|
||||
|
||||
# ------------------------------------------------------------ robots.txt
|
||||
|
||||
ROBOTS = "https://jobs.fake.example/robots.txt"
|
||||
|
||||
|
||||
def test_robots_that_allow_the_page_let_the_board_be_read(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /admin/\n",
|
||||
"https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
items, state = _run(monkeypatch, _fake_host(robots=True))
|
||||
assert len(items) == 1
|
||||
assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("robots", [
|
||||
b"User-agent: *\nDisallow: /acme/\n",
|
||||
# A rule for this app by name counts as well as the one for everyone.
|
||||
f"User-agent: {ats.USER_AGENT.split('/')[0]}\nDisallow: /\n\nUser-agent: *\nAllow: /\n"
|
||||
.encode(),
|
||||
])
|
||||
def test_robots_that_disallow_the_page_fail_the_board(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], robots: bytes) -> None:
|
||||
web["answers"].update({ROBOTS: robots, "https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
items, state = _run(monkeypatch, _fake_host(robots=True))
|
||||
assert items == []
|
||||
assert web["asked"] == [ROBOTS] # the listing is never fetched
|
||||
assert json.loads(state["ats:failing:fake:acme"])["reason"] == "robots.txt disallows it"
|
||||
assert "ats:ok:fake:acme" not in state
|
||||
|
||||
|
||||
def test_a_robots_verdict_is_kept_for_a_day(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n",
|
||||
"https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
module = _fake_host(robots=True)
|
||||
_items, state = _run(monkeypatch, module)
|
||||
assert web["asked"] == [ROBOTS]
|
||||
|
||||
# Within the day the kept verdict holds, even though the site changed.
|
||||
web["answers"][ROBOTS] = b"User-agent: *\nAllow: /\n"
|
||||
web["asked"].clear()
|
||||
items, state = _run(monkeypatch, module, state)
|
||||
assert (items, web["asked"]) == ([], [])
|
||||
|
||||
# A day on, robots.txt is read again.
|
||||
kept = json.loads(state["ats:robots:fake:acme"])
|
||||
kept["checked"] = (datetime.now(UTC) - timedelta(days=1, minutes=1)).isoformat()
|
||||
state["ats:robots:fake:acme"] = json.dumps(kept)
|
||||
items, state = _run(monkeypatch, module, state)
|
||||
assert web["asked"] == [ROBOTS, "https://jobs.fake.example/acme/list"]
|
||||
assert len(items) == 1 and "ats:failing:fake:acme" not in state
|
||||
|
||||
|
||||
def test_a_missing_robots_txt_allows_and_is_kept(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
web["answers"]["https://jobs.fake.example/acme/list"] = b'["1"]' # robots.txt: 404
|
||||
items, state = _run(monkeypatch, _fake_host(robots=True))
|
||||
assert len(items) == 1
|
||||
assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True
|
||||
|
||||
|
||||
def test_a_robots_txt_that_cannot_be_read_allows_and_is_asked_again(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
web["answers"].update({ROBOTS: _status(503), "https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
items, state = _run(monkeypatch, _fake_host(robots=True))
|
||||
assert len(items) == 1 and "ats:robots:fake:acme" not in state
|
||||
|
||||
|
||||
def test_fetch_board_honours_robots_txt(
|
||||
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n",
|
||||
"https://jobs.fake.example/acme/list": b'["1"]'})
|
||||
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True))
|
||||
with pytest.raises(ats._Disallowed) as caught:
|
||||
ats.fetch_board("fake:acme")
|
||||
assert ats.failure_reason(caught.value) == "robots.txt disallows it"
|
||||
|
||||
|
||||
# ----------------------------------------------------------------- Lever EU
|
||||
|
||||
def test_a_lever_slug_may_name_the_eu_instance() -> None:
|
||||
assert ats.is_known_target("lever:acme") and ats.is_known_target("lever:eu/tomtom")
|
||||
assert not ats.is_known_target("lever:us/acme")
|
||||
assert not ats.is_known_target("lever:eu/")
|
||||
assert not ats.is_known_target("lever:eu/a/b")
|
||||
assert ats.careers_url("lever:acme") == "https://jobs.lever.co/acme"
|
||||
assert ats.careers_url("lever:eu/tomtom") == "https://jobs.eu.lever.co/tomtom"
|
||||
assert lever.feed_url("eu/tomtom") == "https://api.eu.lever.co/v0/postings/tomtom?mode=json"
|
||||
eu = ats._allowed_host("lever", "eu/tomtom")
|
||||
assert eu("api.eu.lever.co") and not eu("api.lever.co")
|
||||
assert not ats._allowed_host("lever", "tomtom")("api.eu.lever.co")
|
||||
|
||||
|
||||
def test_a_lever_eu_board_is_read_from_the_eu_api(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
urls: list[str] = []
|
||||
|
||||
def fake_get(client, url, host, slug, *, method="GET"):
|
||||
urls.append(url)
|
||||
assert ats._allowed_host(host, slug)(httpx.URL(url).host)
|
||||
return (FIXTURES / "lever.json").read_bytes()
|
||||
|
||||
monkeypatch.setattr(ats, "_get", fake_get)
|
||||
rows, complete = ats._fetch_rows(object(), "lever:eu/tomtom") # type: ignore[arg-type]
|
||||
assert urls == ["https://api.eu.lever.co/v0/postings/tomtom?mode=json"]
|
||||
assert rows and complete
|
||||
|
||||
|
||||
def test_the_eu_api_is_refused_for_a_us_board_and_the_reverse() -> None:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(302, headers={"location": "https://api.lever.co/v0/postings/x"})
|
||||
|
||||
with _client(handler) as client, pytest.raises(ats._Refused):
|
||||
ats._get(client, "https://api.eu.lever.co/v0/postings/x", "lever", "eu/x")
|
||||
Reference in new issue
Block a user