Read a role's own page once for a host with detail, and fail a board whose robots.txt disallows it
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b02b72b4aa
commit
8b14d8575b
2 files changed
+578
-6
No files matched your search
+153
-6
@@ -6,7 +6,8 @@ unauthenticated JSON, XML or RSS — first-party, no scraping, and a role that
|
|||||||
drops off the board is *closed*, which is the cheapest staleness check there
|
drops off the board is *closed*, which is the cheapest staleness check there
|
||||||
is. What a host says and how its feed reads is its module's; this file holds
|
is. What a host says and how its feed reads is its module's; this file holds
|
||||||
what every host shares: the fetch and its allow-list, paging and
|
what every host shares: the fetch and its allow-list, paging and
|
||||||
completeness, the classifier, and the plugin itself.
|
completeness, the per-role detail reads, robots.txt, the classifier, and the
|
||||||
|
plugin itself.
|
||||||
|
|
||||||
The allow-list and the classifier are ported from `openings.go` and
|
The allow-list and the classifier are ported from `openings.go` and
|
||||||
`cmd/seedgen/main.go` in
|
`cmd/seedgen/main.go` in
|
||||||
@@ -32,9 +33,10 @@ import json
|
|||||||
import logging
|
import logging
|
||||||
import re
|
import re
|
||||||
import time
|
import time
|
||||||
from collections.abc import Callable, Iterator
|
from collections.abc import Callable, Iterator, MutableMapping
|
||||||
from datetime import UTC, datetime
|
from datetime import UTC, datetime, timedelta
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
from urllib.robotparser import RobotFileParser
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
@@ -110,6 +112,11 @@ class _TooLarge(ValueError):
|
|||||||
retry at the host's `fallback_url`."""
|
retry at the host's `fallback_url`."""
|
||||||
|
|
||||||
|
|
||||||
|
class _Disallowed(Exception):
|
||||||
|
"""The site's robots.txt disallows the page a board is read from (see
|
||||||
|
`_check_robots`)."""
|
||||||
|
|
||||||
|
|
||||||
def _get(client: httpx.Client, url: str, host: str, slug: str, *,
|
def _get(client: httpx.Client, url: str, host: str, slug: str, *,
|
||||||
method: str = "GET", payload: dict[str, Any] | None = None,
|
method: str = "GET", payload: dict[str, Any] | None = None,
|
||||||
headers: dict[str, str] | None = None) -> bytes:
|
headers: dict[str, str] | None = None) -> bytes:
|
||||||
@@ -329,7 +336,8 @@ def careers_url(target: str) -> str:
|
|||||||
#: Pages read from one paged board at most, and the pause between two of
|
#: Pages read from one paged board at most, and the pause between two of
|
||||||
#: them. Workable answers ten roles a page, SmartRecruiters a hundred and
|
#: them. Workable answers ten roles a page, SmartRecruiters a hundred and
|
||||||
#: Teamtailor fifty, so the cap is 1,000, 10,000 and 5,000 roles; the pause
|
#: Teamtailor fifty, so the cap is 1,000, 10,000 and 5,000 roles; the pause
|
||||||
#: keeps Workable, which answers a burst with 429, from seeing one.
|
#: keeps Workable, which answers a burst with 429, from seeing one. The same
|
||||||
|
#: pause separates two detail reads.
|
||||||
MAX_PAGES = 100
|
MAX_PAGES = 100
|
||||||
PAGE_DELAY = 0.5
|
PAGE_DELAY = 0.5
|
||||||
|
|
||||||
@@ -408,18 +416,142 @@ def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
|
|||||||
return module.parse(body, slug), True
|
return module.parse(body, slug), True
|
||||||
|
|
||||||
|
|
||||||
|
# -------------------------------------------------------------- robots.txt
|
||||||
|
|
||||||
|
#: How long a robots.txt answer holds for one board before it is read again.
|
||||||
|
ROBOTS_TTL = timedelta(days=1)
|
||||||
|
|
||||||
|
|
||||||
|
def _robots_allows(body: bytes, page: str) -> bool:
|
||||||
|
"""Whether a robots.txt lets `page` be fetched: by its rules for every
|
||||||
|
agent (`*`), and by any it has for this app's own User-Agent token, so a
|
||||||
|
site that shuts out this app by name is honoured as well."""
|
||||||
|
parser = RobotFileParser()
|
||||||
|
parser.parse(body.decode("utf-8", "replace").splitlines())
|
||||||
|
return parser.can_fetch("*", page) and parser.can_fetch(USER_AGENT, page)
|
||||||
|
|
||||||
|
|
||||||
|
def _check_robots(client: httpx.Client, target: str,
|
||||||
|
state: MutableMapping[str, str] | None = None) -> None:
|
||||||
|
"""Raise `_Disallowed` when the site of a host with a `robots_page` has a
|
||||||
|
robots.txt that disallows that page. Read through `_get`, so from the
|
||||||
|
board's own allowed hosts only, and at most once per `ROBOTS_TTL` for a
|
||||||
|
board when `state` is given: the verdict and when it was reached are kept
|
||||||
|
under `ats:robots:<target>`. A robots.txt that is missing (404 or 410)
|
||||||
|
allows, and that verdict is kept like any other; one that cannot be read
|
||||||
|
for another reason allows too, but is not kept, so the next pull asks
|
||||||
|
again."""
|
||||||
|
host, _, slug = target.partition(":")
|
||||||
|
module = atshosts.HOSTS.get(host)
|
||||||
|
if module is None or not hasattr(module, "robots_page") or not is_known_target(target):
|
||||||
|
return
|
||||||
|
page = module.robots_page(slug)
|
||||||
|
key = f"ats:robots:{target}"
|
||||||
|
now = datetime.now(UTC)
|
||||||
|
if state is not None:
|
||||||
|
try:
|
||||||
|
kept = json.loads(state.get(key) or "{}")
|
||||||
|
fresh = now - datetime.fromisoformat(kept["checked"]) < ROBOTS_TTL
|
||||||
|
except (ValueError, KeyError, TypeError):
|
||||||
|
kept, fresh = {}, False
|
||||||
|
if fresh:
|
||||||
|
if not kept.get("allowed", True):
|
||||||
|
raise _Disallowed(f"robots.txt disallows {page}")
|
||||||
|
return
|
||||||
|
url = f"https://{httpx.URL(page).host}/robots.txt"
|
||||||
|
try:
|
||||||
|
body = _getter(client, host, slug)(url, headers={"Accept": "text/plain"})
|
||||||
|
except httpx.HTTPStatusError as exc:
|
||||||
|
if exc.response.status_code not in (404, 410):
|
||||||
|
log.warning("ats: %s: robots.txt answered %s; allowed until the next pull",
|
||||||
|
target, exc.response.status_code)
|
||||||
|
return
|
||||||
|
allowed = True
|
||||||
|
except Exception as exc:
|
||||||
|
log.warning("ats: %s: robots.txt not read (%s); allowed until the next pull",
|
||||||
|
target, exc)
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
allowed = _robots_allows(body, page)
|
||||||
|
if state is not None:
|
||||||
|
state[key] = json.dumps({"checked": now.isoformat(timespec="seconds"),
|
||||||
|
"allowed": allowed})
|
||||||
|
if not allowed:
|
||||||
|
raise _Disallowed(f"robots.txt disallows {page}")
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- details
|
||||||
|
|
||||||
|
#: Roles read in detail from one board in one pull at most. The rest of a
|
||||||
|
#: large new board is read over the next pulls, 200 at a time.
|
||||||
|
DETAIL_CAP = 200
|
||||||
|
|
||||||
|
|
||||||
|
def _read_details(client: httpx.Client, target: str, rows: list[Row], complete: bool,
|
||||||
|
detailed: set[str]) -> tuple[list[Row], set[str]]:
|
||||||
|
"""The rows of a board whose host has `detail`, each role not yet in
|
||||||
|
`detailed` (compound ids) filled from its own page, at most `DETAIL_CAP`
|
||||||
|
of them and `PAGE_DELAY` apart. Returns the rows and the ids read in
|
||||||
|
detail so far, for `ats:detailed:<target>`.
|
||||||
|
|
||||||
|
Every row not read in detail in this call goes out with an empty
|
||||||
|
description: a role read on an earlier pull keeps the one it has, since
|
||||||
|
`upsert_role` never overwrites a filled column with an empty value, and a
|
||||||
|
teaser from the listing must not replace it. A detail that raises is
|
||||||
|
logged and the role left for the next pull; a 429 stops the reads for this
|
||||||
|
board until then. None of this touches completeness, which is the
|
||||||
|
listing's alone. The ids of roles no longer listed are dropped, but only
|
||||||
|
after a complete listing, so one cut short does not make the next pull
|
||||||
|
read its missing roles again."""
|
||||||
|
host, _, slug = target.partition(":")
|
||||||
|
module = atshosts.HOSTS[host]
|
||||||
|
get = _getter(client, host, slug)
|
||||||
|
done = set(detailed)
|
||||||
|
out: list[Row] = []
|
||||||
|
tried = 0
|
||||||
|
stopped = False
|
||||||
|
for row in rows:
|
||||||
|
compound = f"{host}:{slug}:{row['source_id']}"
|
||||||
|
if compound in done or stopped or tried >= DETAIL_CAP:
|
||||||
|
out.append(row | {"description": ""})
|
||||||
|
continue
|
||||||
|
time.sleep(PAGE_DELAY)
|
||||||
|
tried += 1
|
||||||
|
try:
|
||||||
|
filled = module.detail(get, slug, row)
|
||||||
|
except Exception as exc:
|
||||||
|
if isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code == 429:
|
||||||
|
log.warning("ats: %s: 429 on a detail read; the rest wait for the next pull",
|
||||||
|
target)
|
||||||
|
stopped = True
|
||||||
|
else:
|
||||||
|
log.warning("ats: %s: detail of %s failed (%s)", target, row["source_id"], exc)
|
||||||
|
out.append(row | {"description": ""})
|
||||||
|
continue
|
||||||
|
out.append(row | filled | {"source_id": row["source_id"]})
|
||||||
|
done.add(compound)
|
||||||
|
if complete:
|
||||||
|
done &= {f"{host}:{slug}:{row['source_id']}" for row in rows}
|
||||||
|
return out, done
|
||||||
|
|
||||||
|
|
||||||
def fetch_board(target: str) -> tuple[list[Row], bool]:
|
def fetch_board(target: str) -> tuple[list[Row], bool]:
|
||||||
"""Fetch and parse one board directly: no `Source`, no `self.state`, just
|
"""Fetch and parse one board directly: no `Source`, no `self.state`, just
|
||||||
"does this board answer, what does it list, and is that all of it". For
|
"does this board answer, what does it list, and is that all of it". For
|
||||||
`app companies add|check`, which ask that question about one board at a
|
`app companies add|check`, which ask that question about one board at a
|
||||||
time outside any pull. Raises the same way `_fetch_rows` does."""
|
time outside any pull. The listing only, with no detail reads, and
|
||||||
|
robots.txt read afresh. Raises the same way `_fetch_rows` does, and
|
||||||
|
`_Disallowed` for a board robots.txt shuts out."""
|
||||||
with httpx.Client() as client:
|
with httpx.Client() as client:
|
||||||
|
_check_robots(client, target)
|
||||||
return _fetch_rows(client, target)
|
return _fetch_rows(client, target)
|
||||||
|
|
||||||
|
|
||||||
def failure_reason(exc: BaseException) -> str:
|
def failure_reason(exc: BaseException) -> str:
|
||||||
"""One short line for why a board did not answer: the status of an HTTP
|
"""One short line for why a board did not answer: the status of an HTTP
|
||||||
error, else the exception's class and the first line of its message."""
|
error, else the exception's class and the first line of its message."""
|
||||||
|
if isinstance(exc, _Disallowed):
|
||||||
|
return "robots.txt disallows it"
|
||||||
if isinstance(exc, httpx.HTTPStatusError):
|
if isinstance(exc, httpx.HTTPStatusError):
|
||||||
return f"{exc.response.status_code} {exc.response.reason_phrase}".strip()
|
return f"{exc.response.status_code} {exc.response.reason_phrase}".strip()
|
||||||
lines = str(exc).strip().splitlines()
|
lines = str(exc).strip().splitlines()
|
||||||
@@ -465,6 +597,7 @@ class ATSSource(Source):
|
|||||||
host, _, slug = target.partition(":")
|
host, _, slug = target.partition(":")
|
||||||
failing = f"ats:failing:{target}"
|
failing = f"ats:failing:{target}"
|
||||||
try:
|
try:
|
||||||
|
_check_robots(client, target, self.state)
|
||||||
rows, complete = _fetch_rows(client, target)
|
rows, complete = _fetch_rows(client, target)
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
log.warning("ats: %s failed (%s)", target, exc)
|
log.warning("ats: %s failed (%s)", target, exc)
|
||||||
@@ -481,6 +614,15 @@ class ATSSource(Source):
|
|||||||
continue
|
continue
|
||||||
self.state.pop(failing, None)
|
self.state.pop(failing, None)
|
||||||
|
|
||||||
|
detailed_key = f"ats:detailed:{target}"
|
||||||
|
detailed: set[str] | None = None
|
||||||
|
if hasattr(atshosts.HOSTS.get(host), "detail"):
|
||||||
|
try:
|
||||||
|
detailed = set(json.loads(self.state.get(detailed_key) or "[]"))
|
||||||
|
except (ValueError, TypeError):
|
||||||
|
detailed = set()
|
||||||
|
rows, detailed = _read_details(client, target, rows, complete, detailed)
|
||||||
|
|
||||||
seen_ids = []
|
seen_ids = []
|
||||||
for row in rows:
|
for row in rows:
|
||||||
# The compound id, matching `roles.source_id` — what
|
# The compound id, matching `roles.source_id` — what
|
||||||
@@ -520,7 +662,12 @@ class ATSSource(Source):
|
|||||||
# board that blew up half way through is neither marked ok nor
|
# board that blew up half way through is neither marked ok nor
|
||||||
# has a partial id list treated as the whole of it. `complete`
|
# has a partial id list treated as the whole of it. `complete`
|
||||||
# says whether the ids are the whole board, which the pull
|
# says whether the ids are the whole board, which the pull
|
||||||
# needs before it closes what they leave out.
|
# needs before it closes what they leave out. The ids read in
|
||||||
|
# detail are kept here too, after every row went out, so a
|
||||||
|
# pull stopped half way reads the rest again rather than
|
||||||
|
# losing their descriptions.
|
||||||
self.state[f"ats:ok:{target}"] = run_marker
|
self.state[f"ats:ok:{target}"] = run_marker
|
||||||
self.state[f"ats:complete:{target}"] = "1" if complete else ""
|
self.state[f"ats:complete:{target}"] = "1" if complete else ""
|
||||||
self.state[f"ats:ids:{target}"] = json.dumps(seen_ids)
|
self.state[f"ats:ids:{target}"] = json.dumps(seen_ids)
|
||||||
|
if detailed is not None:
|
||||||
|
self.state[detailed_key] = json.dumps(sorted(detailed))
|
||||||
@@ -0,0 +1,425 @@
|
|||||||
|
"""The host modules of `plugins/atshosts/` and what `plugins/ats.py` does for
|
||||||
|
every host: the registry and its contract check, the fetch a host is handed,
|
||||||
|
the per-role detail reads, robots.txt, and Lever's EU instance. No network —
|
||||||
|
a fake host stands in where a behaviour belongs to no real host yet."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
import types
|
||||||
|
from collections.abc import Callable
|
||||||
|
from datetime import UTC, datetime, timedelta
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT = Path(__file__).resolve().parents[1]
|
||||||
|
if str(ROOT) not in sys.path:
|
||||||
|
sys.path.insert(0, str(ROOT))
|
||||||
|
|
||||||
|
from plugins import ats, atshosts # noqa: E402
|
||||||
|
from plugins.atshosts import lever # noqa: E402
|
||||||
|
|
||||||
|
FIXTURES = ROOT / "tests" / "fixtures" / "ats"
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------- registry
|
||||||
|
|
||||||
|
def test_the_registry_finds_every_module_in_order_of_name() -> None:
|
||||||
|
names = sorted(p.stem for p in (ROOT / "plugins" / "atshosts").glob("*.py")
|
||||||
|
if not p.stem.startswith("_"))
|
||||||
|
assert list(atshosts.HOSTS) == names
|
||||||
|
assert set(names) >= {"greenhouse", "lever", "ashby", "workable", "smartrecruiters",
|
||||||
|
"recruitee", "personio", "teamtailor"}
|
||||||
|
assert all(not atshosts.problems(module, name) for name, module in atshosts.HOSTS.items())
|
||||||
|
# The Sources page's select and the plugin's description follow the modules.
|
||||||
|
assert {name: module.LABEL for name, module in atshosts.HOSTS.items()} == ats.HOSTS
|
||||||
|
assert "Teamtailor" in ats.ATSSource.description
|
||||||
|
|
||||||
|
|
||||||
|
def _package(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, name: str,
|
||||||
|
modules: dict[str, str]) -> list[str]:
|
||||||
|
package = tmp_path / name
|
||||||
|
package.mkdir()
|
||||||
|
(package / "__init__.py").write_text("")
|
||||||
|
for module, source in modules.items():
|
||||||
|
(package / f"{module}.py").write_text(source)
|
||||||
|
monkeypatch.syspath_prepend(str(tmp_path))
|
||||||
|
return [str(package)]
|
||||||
|
|
||||||
|
|
||||||
|
_GOOD = """
|
||||||
|
NAME = "{name}"
|
||||||
|
LABEL = "Good"
|
||||||
|
PATTERNS = (r"good\\.example/([a-z]+)",)
|
||||||
|
def slug_ok(slug): return True
|
||||||
|
def allowed(slug): return lambda h: h == "good.example"
|
||||||
|
def careers_url(slug): return "https://good.example/" + slug
|
||||||
|
def feed_url(slug): return "https://good.example/" + slug + ".json"
|
||||||
|
def parse(body, slug): return []
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_registry_loads_a_package_by_name_and_skips_private_modules(
|
||||||
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
path = _package(tmp_path, monkeypatch, "hostsok", {
|
||||||
|
"zeta": _GOOD.format(name="zeta"), "alpha": _GOOD.format(name="alpha"),
|
||||||
|
"_helpers": "X = 1\n"})
|
||||||
|
assert list(atshosts.load(path, "hostsok")) == ["alpha", "zeta"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_registry_refuses_a_module_missing_a_required_name(
|
||||||
|
tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
broken = _GOOD.format(name="broken").replace("def careers_url", "def not_careers_url")
|
||||||
|
path = _package(tmp_path, monkeypatch, "hostsbad", {"broken": broken})
|
||||||
|
with pytest.raises(ImportError, match=r"hostsbad\.broken .*no careers_url"):
|
||||||
|
atshosts.load(path, "hostsbad")
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_contract_check_names_each_problem() -> None:
|
||||||
|
module = types.ModuleType("odd")
|
||||||
|
module.NAME, module.LABEL, module.PATTERNS = "other", "", ("(",)
|
||||||
|
module.slug_ok = module.allowed = module.careers_url = lambda slug: ""
|
||||||
|
module.page = lambda get, slug, cursor: None
|
||||||
|
module.probe_ok = lambda body: True
|
||||||
|
found = atshosts.problems(module, "odd")
|
||||||
|
assert "NAME is 'other', not the module's name 'odd'" in found
|
||||||
|
assert "LABEL is not a non-empty string" in found
|
||||||
|
assert any(p.startswith("PATTERNS has '('") for p in found)
|
||||||
|
assert "page without rows" in found
|
||||||
|
assert "probe_ok without probe_url" in found
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------ the fetch
|
||||||
|
|
||||||
|
def _client(handler: Callable[[httpx.Request], httpx.Response]) -> httpx.Client:
|
||||||
|
return httpx.Client(transport=httpx.MockTransport(handler))
|
||||||
|
|
||||||
|
|
||||||
|
def test_get_sends_extra_headers_on_top_of_its_own() -> None:
|
||||||
|
seen: list[httpx.Headers] = []
|
||||||
|
|
||||||
|
def handler(request: httpx.Request) -> httpx.Response:
|
||||||
|
seen.append(request.headers)
|
||||||
|
return httpx.Response(200, content=b"ok")
|
||||||
|
|
||||||
|
with _client(handler) as client:
|
||||||
|
get = ats._getter(client, "lever", "acme")
|
||||||
|
assert get("https://api.lever.co/v0/postings/acme",
|
||||||
|
headers={"Accept": "text/plain", "X-Board": "acme"}) == b"ok"
|
||||||
|
get("https://api.lever.co/v0/postings/acme")
|
||||||
|
assert seen[0]["accept"] == "text/plain" and seen[0]["x-board"] == "acme"
|
||||||
|
assert seen[0]["user-agent"] == seen[1]["user-agent"] == ats.USER_AGENT
|
||||||
|
assert seen[1]["accept"] == "application/json, text/xml"
|
||||||
|
assert "x-board" not in seen[1]
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_bound_get_reaches_a_replaced_ats_get(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
"""Bound before the replacement, and still reaching it: `_get` is looked
|
||||||
|
up when a request is made."""
|
||||||
|
client = object()
|
||||||
|
get = ats._getter(client, "lever", "acme") # type: ignore[arg-type]
|
||||||
|
calls: list[tuple[Any, ...]] = []
|
||||||
|
|
||||||
|
def replaced(*args: Any, **kwargs: Any) -> bytes:
|
||||||
|
calls.append((args, kwargs))
|
||||||
|
return b""
|
||||||
|
|
||||||
|
monkeypatch.setattr(ats, "_get", replaced)
|
||||||
|
get("https://api.lever.co/v0/postings/acme", headers={"X": "1"})
|
||||||
|
get("https://api.lever.co/v0/postings/acme", method="POST", payload={})
|
||||||
|
assert calls == [
|
||||||
|
((client, "https://api.lever.co/v0/postings/acme", "lever", "acme"),
|
||||||
|
{"method": "GET", "headers": {"X": "1"}}),
|
||||||
|
((client, "https://api.lever.co/v0/postings/acme", "lever", "acme"),
|
||||||
|
{"method": "POST", "payload": {}})]
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------- a fake host
|
||||||
|
|
||||||
|
def _fake_host(*, detail: Callable[..., Any] | None = None,
|
||||||
|
robots: bool = False) -> types.ModuleType:
|
||||||
|
"""A host `fake` whose feed is a JSON list of ids, each role with a
|
||||||
|
teaser for a description, and, with `detail`, a per-role page; with
|
||||||
|
`robots`, its listing is the page robots.txt must allow."""
|
||||||
|
module = types.ModuleType("fake")
|
||||||
|
module.NAME, module.LABEL, module.PATTERNS = "fake", "Fake", ()
|
||||||
|
module.slug_ok = lambda slug: slug.isalnum()
|
||||||
|
module.allowed = lambda slug: lambda h: h == "jobs.fake.example"
|
||||||
|
module.careers_url = lambda slug: f"https://jobs.fake.example/{slug}"
|
||||||
|
module.feed_url = lambda slug: f"https://jobs.fake.example/{slug}/list"
|
||||||
|
module.parse = lambda body, slug: [
|
||||||
|
{"source_id": i, "title": f"Role {i}", "location": "", "department": "",
|
||||||
|
"url": f"https://jobs.fake.example/{slug}/{i}", "posted": None,
|
||||||
|
"description": "teaser", "remote": None, "salary_text": ""}
|
||||||
|
for i in json.loads(body)]
|
||||||
|
if detail is not None:
|
||||||
|
module.detail = detail
|
||||||
|
if robots:
|
||||||
|
module.robots_page = lambda slug: f"https://jobs.fake.example/{slug}/list"
|
||||||
|
return module
|
||||||
|
|
||||||
|
|
||||||
|
def _status(code: int, url: str = "https://jobs.fake.example/") -> httpx.HTTPStatusError:
|
||||||
|
request = httpx.Request("GET", url)
|
||||||
|
return httpx.HTTPStatusError(str(code), request=request,
|
||||||
|
response=httpx.Response(code, request=request))
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def web(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]:
|
||||||
|
"""`ats._get` answering from a dict of URL -> body or exception, and
|
||||||
|
remembering what it was asked. No delay between requests."""
|
||||||
|
answers: dict[str, Any] = {}
|
||||||
|
asked: list[str] = []
|
||||||
|
|
||||||
|
def fake_get(client, url, host, slug, *, method="GET", payload=None, headers=None):
|
||||||
|
asked.append(url)
|
||||||
|
if not ats._allowed_host(host, slug)(httpx.URL(url).host):
|
||||||
|
raise ats._Refused(url)
|
||||||
|
answer = answers.get(url, _status(404, url))
|
||||||
|
if isinstance(answer, BaseException):
|
||||||
|
raise answer
|
||||||
|
return answer
|
||||||
|
|
||||||
|
monkeypatch.setattr(ats, "_get", fake_get)
|
||||||
|
monkeypatch.setattr(ats, "PAGE_DELAY", 0)
|
||||||
|
return {"answers": answers, "asked": asked}
|
||||||
|
|
||||||
|
|
||||||
|
def _run(monkeypatch: pytest.MonkeyPatch, module: types.ModuleType,
|
||||||
|
state: dict[str, str] | None = None) -> tuple[list[Any], dict[str, str]]:
|
||||||
|
monkeypatch.setitem(atshosts.HOSTS, "fake", module)
|
||||||
|
source = ats.ATSSource({"boards": [{"url": "fake:acme", "label": "Acme"}], "run": "r"})
|
||||||
|
if state is not None:
|
||||||
|
source.state = state
|
||||||
|
return list(source.fetch()), source.state
|
||||||
|
|
||||||
|
|
||||||
|
def _detail(get: Any, slug: str, row: dict[str, Any]) -> dict[str, Any]:
|
||||||
|
"""The fake host's detail: the role's own page is its description."""
|
||||||
|
return {"description": get(f"https://jobs.fake.example/{slug}/{row['source_id']}").decode(),
|
||||||
|
"location": "Copenhagen"}
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------- detail
|
||||||
|
|
||||||
|
def test_detail_is_read_for_unread_roles_only(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||||
|
base = "https://jobs.fake.example/acme"
|
||||||
|
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one in full",
|
||||||
|
f"{base}/2": b"two in full", f"{base}/3": b"three in full"})
|
||||||
|
module = _fake_host(detail=_detail)
|
||||||
|
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:2"])}
|
||||||
|
items, state = _run(monkeypatch, module, state)
|
||||||
|
|
||||||
|
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/3"]
|
||||||
|
assert [(i.source_id, i.body) for i in items] == [
|
||||||
|
("fake:acme:1", "one in full"), ("fake:acme:2", ""), ("fake:acme:3", "three in full")]
|
||||||
|
# What detail returns replaces the listing's field; a role not read keeps the listing's.
|
||||||
|
assert items[0].raw["place"] == "Copenhagen" and items[1].raw["place"] == ""
|
||||||
|
assert json.loads(state["ats:detailed:fake:acme"]) == [
|
||||||
|
"fake:acme:1", "fake:acme:2", "fake:acme:3"]
|
||||||
|
assert state["ats:complete:fake:acme"] == "1"
|
||||||
|
|
||||||
|
|
||||||
|
def test_detail_reads_stop_at_the_cap_and_the_rest_wait(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||||
|
base = "https://jobs.fake.example/acme"
|
||||||
|
ids = [str(n) for n in range(5)]
|
||||||
|
web["answers"].update({f"{base}/list": json.dumps(ids).encode()}
|
||||||
|
| {f"{base}/{i}": f"full {i}".encode() for i in ids})
|
||||||
|
monkeypatch.setattr(ats, "DETAIL_CAP", 2)
|
||||||
|
module = _fake_host(detail=_detail)
|
||||||
|
items, state = _run(monkeypatch, module)
|
||||||
|
assert [i.body for i in items] == ["full 0", "full 1", "", "", ""]
|
||||||
|
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:0", "fake:acme:1"]
|
||||||
|
|
||||||
|
web["asked"].clear()
|
||||||
|
items, state = _run(monkeypatch, module, state)
|
||||||
|
assert web["asked"] == [f"{base}/list", f"{base}/2", f"{base}/3"]
|
||||||
|
assert [i.body for i in items] == ["", "", "full 2", "full 3", ""]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_detail_that_fails_skips_the_role_and_not_the_board(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||||
|
base = "https://jobs.fake.example/acme"
|
||||||
|
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one",
|
||||||
|
f"{base}/2": ValueError("no such page shape"), f"{base}/3": b"three"})
|
||||||
|
items, state = _run(monkeypatch, _fake_host(detail=_detail))
|
||||||
|
assert [(i.source_id, i.body) for i in items] == [
|
||||||
|
("fake:acme:1", "one"), ("fake:acme:2", ""), ("fake:acme:3", "three")]
|
||||||
|
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:3"]
|
||||||
|
assert "ats:failing:fake:acme" not in state and state["ats:ok:fake:acme"] == "r"
|
||||||
|
assert state["ats:complete:fake:acme"] == "1"
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_429_on_a_detail_stops_the_reads_for_the_pull(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||||
|
base = "https://jobs.fake.example/acme"
|
||||||
|
web["answers"].update({f"{base}/list": b'["1", "2", "3"]', f"{base}/1": b"one",
|
||||||
|
f"{base}/2": _status(429), f"{base}/3": b"three"})
|
||||||
|
items, state = _run(monkeypatch, _fake_host(detail=_detail))
|
||||||
|
assert web["asked"] == [f"{base}/list", f"{base}/1", f"{base}/2"]
|
||||||
|
assert [i.body for i in items] == ["one", "", ""]
|
||||||
|
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1"]
|
||||||
|
assert state["ats:complete:fake:acme"] == "1"
|
||||||
|
|
||||||
|
|
||||||
|
def test_ids_no_longer_listed_leave_the_detailed_list(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||||
|
base = "https://jobs.fake.example/acme"
|
||||||
|
web["answers"].update({f"{base}/list": b'["2"]'})
|
||||||
|
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
|
||||||
|
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
|
||||||
|
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:2"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_an_incomplete_listing_keeps_the_ids_it_did_not_list(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||||
|
"""A listing cut short says nothing about the roles it left out, so their
|
||||||
|
ids stay, and the next pull does not read them in detail again."""
|
||||||
|
base = "https://jobs.fake.example/acme"
|
||||||
|
web["answers"].update({f"{base}/list": b'["2"]'})
|
||||||
|
monkeypatch.setattr(ats, "_fetch_rows", lambda client, target: (
|
||||||
|
atshosts.HOSTS["fake"].parse(b'["2"]', "acme"), False))
|
||||||
|
state = {"ats:detailed:fake:acme": json.dumps(["fake:acme:1", "fake:acme:2"])}
|
||||||
|
_items, state = _run(monkeypatch, _fake_host(detail=_detail), state)
|
||||||
|
assert json.loads(state["ats:detailed:fake:acme"]) == ["fake:acme:1", "fake:acme:2"]
|
||||||
|
assert state["ats:complete:fake:acme"] == ""
|
||||||
|
|
||||||
|
|
||||||
|
def test_fetch_board_reads_the_listing_only(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||||
|
base = "https://jobs.fake.example/acme"
|
||||||
|
web["answers"].update({f"{base}/list": b'["1"]', f"{base}/1": b"one"})
|
||||||
|
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(detail=_detail))
|
||||||
|
rows, complete = ats.fetch_board("fake:acme")
|
||||||
|
assert ([r["description"] for r in rows], complete) == (["teaser"], True)
|
||||||
|
assert web["asked"] == [f"{base}/list"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_host_without_detail_keeps_the_listings_description(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||||
|
web["answers"]["https://jobs.fake.example/acme/list"] = b'["1"]'
|
||||||
|
items, state = _run(monkeypatch, _fake_host())
|
||||||
|
assert [i.body for i in items] == ["teaser"]
|
||||||
|
assert "ats:detailed:fake:acme" not in state
|
||||||
|
|
||||||
|
|
||||||
|
# ------------------------------------------------------------ robots.txt
|
||||||
|
|
||||||
|
ROBOTS = "https://jobs.fake.example/robots.txt"
|
||||||
|
|
||||||
|
|
||||||
|
def test_robots_that_allow_the_page_let_the_board_be_read(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||||
|
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /admin/\n",
|
||||||
|
"https://jobs.fake.example/acme/list": b'["1"]'})
|
||||||
|
items, state = _run(monkeypatch, _fake_host(robots=True))
|
||||||
|
assert len(items) == 1
|
||||||
|
assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("robots", [
|
||||||
|
b"User-agent: *\nDisallow: /acme/\n",
|
||||||
|
# A rule for this app by name counts as well as the one for everyone.
|
||||||
|
f"User-agent: {ats.USER_AGENT.split('/')[0]}\nDisallow: /\n\nUser-agent: *\nAllow: /\n"
|
||||||
|
.encode(),
|
||||||
|
])
|
||||||
|
def test_robots_that_disallow_the_page_fail_the_board(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any], robots: bytes) -> None:
|
||||||
|
web["answers"].update({ROBOTS: robots, "https://jobs.fake.example/acme/list": b'["1"]'})
|
||||||
|
items, state = _run(monkeypatch, _fake_host(robots=True))
|
||||||
|
assert items == []
|
||||||
|
assert web["asked"] == [ROBOTS] # the listing is never fetched
|
||||||
|
assert json.loads(state["ats:failing:fake:acme"])["reason"] == "robots.txt disallows it"
|
||||||
|
assert "ats:ok:fake:acme" not in state
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_robots_verdict_is_kept_for_a_day(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||||
|
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n",
|
||||||
|
"https://jobs.fake.example/acme/list": b'["1"]'})
|
||||||
|
module = _fake_host(robots=True)
|
||||||
|
_items, state = _run(monkeypatch, module)
|
||||||
|
assert web["asked"] == [ROBOTS]
|
||||||
|
|
||||||
|
# Within the day the kept verdict holds, even though the site changed.
|
||||||
|
web["answers"][ROBOTS] = b"User-agent: *\nAllow: /\n"
|
||||||
|
web["asked"].clear()
|
||||||
|
items, state = _run(monkeypatch, module, state)
|
||||||
|
assert (items, web["asked"]) == ([], [])
|
||||||
|
|
||||||
|
# A day on, robots.txt is read again.
|
||||||
|
kept = json.loads(state["ats:robots:fake:acme"])
|
||||||
|
kept["checked"] = (datetime.now(UTC) - timedelta(days=1, minutes=1)).isoformat()
|
||||||
|
state["ats:robots:fake:acme"] = json.dumps(kept)
|
||||||
|
items, state = _run(monkeypatch, module, state)
|
||||||
|
assert web["asked"] == [ROBOTS, "https://jobs.fake.example/acme/list"]
|
||||||
|
assert len(items) == 1 and "ats:failing:fake:acme" not in state
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_missing_robots_txt_allows_and_is_kept(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||||
|
web["answers"]["https://jobs.fake.example/acme/list"] = b'["1"]' # robots.txt: 404
|
||||||
|
items, state = _run(monkeypatch, _fake_host(robots=True))
|
||||||
|
assert len(items) == 1
|
||||||
|
assert json.loads(state["ats:robots:fake:acme"])["allowed"] is True
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_robots_txt_that_cannot_be_read_allows_and_is_asked_again(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||||
|
web["answers"].update({ROBOTS: _status(503), "https://jobs.fake.example/acme/list": b'["1"]'})
|
||||||
|
items, state = _run(monkeypatch, _fake_host(robots=True))
|
||||||
|
assert len(items) == 1 and "ats:robots:fake:acme" not in state
|
||||||
|
|
||||||
|
|
||||||
|
def test_fetch_board_honours_robots_txt(
|
||||||
|
monkeypatch: pytest.MonkeyPatch, web: dict[str, Any]) -> None:
|
||||||
|
web["answers"].update({ROBOTS: b"User-agent: *\nDisallow: /\n",
|
||||||
|
"https://jobs.fake.example/acme/list": b'["1"]'})
|
||||||
|
monkeypatch.setitem(atshosts.HOSTS, "fake", _fake_host(robots=True))
|
||||||
|
with pytest.raises(ats._Disallowed) as caught:
|
||||||
|
ats.fetch_board("fake:acme")
|
||||||
|
assert ats.failure_reason(caught.value) == "robots.txt disallows it"
|
||||||
|
|
||||||
|
|
||||||
|
# ----------------------------------------------------------------- Lever EU
|
||||||
|
|
||||||
|
def test_a_lever_slug_may_name_the_eu_instance() -> None:
|
||||||
|
assert ats.is_known_target("lever:acme") and ats.is_known_target("lever:eu/tomtom")
|
||||||
|
assert not ats.is_known_target("lever:us/acme")
|
||||||
|
assert not ats.is_known_target("lever:eu/")
|
||||||
|
assert not ats.is_known_target("lever:eu/a/b")
|
||||||
|
assert ats.careers_url("lever:acme") == "https://jobs.lever.co/acme"
|
||||||
|
assert ats.careers_url("lever:eu/tomtom") == "https://jobs.eu.lever.co/tomtom"
|
||||||
|
assert lever.feed_url("eu/tomtom") == "https://api.eu.lever.co/v0/postings/tomtom?mode=json"
|
||||||
|
eu = ats._allowed_host("lever", "eu/tomtom")
|
||||||
|
assert eu("api.eu.lever.co") and not eu("api.lever.co")
|
||||||
|
assert not ats._allowed_host("lever", "tomtom")("api.eu.lever.co")
|
||||||
|
|
||||||
|
|
||||||
|
def test_a_lever_eu_board_is_read_from_the_eu_api(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
urls: list[str] = []
|
||||||
|
|
||||||
|
def fake_get(client, url, host, slug, *, method="GET"):
|
||||||
|
urls.append(url)
|
||||||
|
assert ats._allowed_host(host, slug)(httpx.URL(url).host)
|
||||||
|
return (FIXTURES / "lever.json").read_bytes()
|
||||||
|
|
||||||
|
monkeypatch.setattr(ats, "_get", fake_get)
|
||||||
|
rows, complete = ats._fetch_rows(object(), "lever:eu/tomtom") # type: ignore[arg-type]
|
||||||
|
assert urls == ["https://api.eu.lever.co/v0/postings/tomtom?mode=json"]
|
||||||
|
assert rows and complete
|
||||||
|
|
||||||
|
|
||||||
|
def test_the_eu_api_is_refused_for_a_us_board_and_the_reverse() -> None:
|
||||||
|
def handler(request: httpx.Request) -> httpx.Response:
|
||||||
|
return httpx.Response(302, headers={"location": "https://api.lever.co/v0/postings/x"})
|
||||||
|
|
||||||
|
with _client(handler) as client, pytest.raises(ats._Refused):
|
||||||
|
ats._get(client, "https://api.eu.lever.co/v0/postings/x", "lever", "eu/x")
|
||||||
Reference in new issue
Block a user