Read a role's own page once for a host with detail, and fail a board whose robots.txt disallows it

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-29 17:05:44 +02:00
1 parent b02b72b4aa
commit 8b14d8575b
2 files changed
+578 -6

No files matched your search

+153 -6
View File
@@ -6,7 +6,8 @@ unauthenticated JSON, XML or RSS — first-party, no scraping, and a role that
drops off the board is *closed*, which is the cheapest staleness check there
is. What a host says and how its feed reads is its module's; this file holds
what every host shares: the fetch and its allow-list, paging and
completeness, the classifier, and the plugin itself.
completeness, the per-role detail reads, robots.txt, the classifier, and the
plugin itself.
The allow-list and the classifier are ported from `openings.go` and
`cmd/seedgen/main.go` in
@@ -32,9 +33,10 @@ import json
import logging
import re
import time
from collections.abc import Callable, Iterator
from datetime import UTC, datetime
from collections.abc import Callable, Iterator, MutableMapping
from datetime import UTC, datetime, timedelta
from typing import Any
from urllib.robotparser import RobotFileParser
import httpx
@@ -110,6 +112,11 @@ class _TooLarge(ValueError):
retry at the host's `fallback_url`."""
class _Disallowed(Exception):
"""The site's robots.txt disallows the page a board is read from (see
`_check_robots`)."""
def _get(client: httpx.Client, url: str, host: str, slug: str, *,
method: str = "GET", payload: dict[str, Any] | None = None,
headers: dict[str, str] | None = None) -> bytes:
@@ -329,7 +336,8 @@ def careers_url(target: str) -> str:
#: Pages read from one paged board at most, and the pause between two of
#: them. Workable answers ten roles a page, SmartRecruiters a hundred and
#: Teamtailor fifty, so the cap is 1,000, 10,000 and 5,000 roles; the pause
#: keeps Workable, which answers a burst with 429, from seeing one.
#: keeps Workable, which answers a burst with 429, from seeing one. The same
#: pause separates two detail reads.
MAX_PAGES = 100
PAGE_DELAY = 0.5
@@ -408,18 +416,142 @@ def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
return module.parse(body, slug), True
# -------------------------------------------------------------- robots.txt
#: How long a robots.txt answer holds for one board before it is read again.
ROBOTS_TTL = timedelta(days=1)
def _robots_allows(body: bytes, page: str) -> bool:
"""Whether a robots.txt lets `page` be fetched: by its rules for every
agent (`*`), and by any it has for this app's own User-Agent token, so a
site that shuts out this app by name is honoured as well."""
parser = RobotFileParser()
parser.parse(body.decode("utf-8", "replace").splitlines())
return parser.can_fetch("*", page) and parser.can_fetch(USER_AGENT, page)
def _check_robots(client: httpx.Client, target: str,
state: MutableMapping[str, str] | None = None) -> None:
"""Raise `_Disallowed` when the site of a host with a `robots_page` has a
robots.txt that disallows that page. Read through `_get`, so from the
board's own allowed hosts only, and at most once per `ROBOTS_TTL` for a
board when `state` is given: the verdict and when it was reached are kept
under `ats:robots:<target>`. A robots.txt that is missing (404 or 410)
allows, and that verdict is kept like any other; one that cannot be read
for another reason allows too, but is not kept, so the next pull asks
again."""
host, _, slug = target.partition(":")
module = atshosts.HOSTS.get(host)
if module is None or not hasattr(module, "robots_page") or not is_known_target(target):
return
page = module.robots_page(slug)
key = f"ats:robots:{target}"
now = datetime.now(UTC)
if state is not None:
try:
kept = json.loads(state.get(key) or "{}")
fresh = now - datetime.fromisoformat(kept["checked"]) < ROBOTS_TTL
except (ValueError, KeyError, TypeError):
kept, fresh = {}, False
if fresh:
if not kept.get("allowed", True):
raise _Disallowed(f"robots.txt disallows {page}")
return
url = f"https://{httpx.URL(page).host}/robots.txt"
try:
body = _getter(client, host, slug)(url, headers={"Accept": "text/plain"})
except httpx.HTTPStatusError as exc:
if exc.response.status_code not in (404, 410):
log.warning("ats: %s: robots.txt answered %s; allowed until the next pull",
target, exc.response.status_code)
return
allowed = True
except Exception as exc:
log.warning("ats: %s: robots.txt not read (%s); allowed until the next pull",
target, exc)
return
else:
allowed = _robots_allows(body, page)
if state is not None:
state[key] = json.dumps({"checked": now.isoformat(timespec="seconds"),
"allowed": allowed})
if not allowed:
raise _Disallowed(f"robots.txt disallows {page}")
# ---------------------------------------------------------------- details
#: Roles read in detail from one board in one pull at most. The rest of a
#: large new board is read over the next pulls, 200 at a time.
DETAIL_CAP = 200
def _read_details(client: httpx.Client, target: str, rows: list[Row], complete: bool,
detailed: set[str]) -> tuple[list[Row], set[str]]:
"""The rows of a board whose host has `detail`, each role not yet in
`detailed` (compound ids) filled from its own page, at most `DETAIL_CAP`
of them and `PAGE_DELAY` apart. Returns the rows and the ids read in
detail so far, for `ats:detailed:<target>`.
Every row not read in detail in this call goes out with an empty
description: a role read on an earlier pull keeps the one it has, since
`upsert_role` never overwrites a filled column with an empty value, and a
teaser from the listing must not replace it. A detail that raises is
logged and the role left for the next pull; a 429 stops the reads for this
board until then. None of this touches completeness, which is the
listing's alone. The ids of roles no longer listed are dropped, but only
after a complete listing, so one cut short does not make the next pull
read its missing roles again."""
host, _, slug = target.partition(":")
module = atshosts.HOSTS[host]
get = _getter(client, host, slug)
done = set(detailed)
out: list[Row] = []
tried = 0
stopped = False
for row in rows:
compound = f"{host}:{slug}:{row['source_id']}"
if compound in done or stopped or tried >= DETAIL_CAP:
out.append(row | {"description": ""})
continue
time.sleep(PAGE_DELAY)
tried += 1
try:
filled = module.detail(get, slug, row)
except Exception as exc:
if isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code == 429:
log.warning("ats: %s: 429 on a detail read; the rest wait for the next pull",
target)
stopped = True
else:
log.warning("ats: %s: detail of %s failed (%s)", target, row["source_id"], exc)
out.append(row | {"description": ""})
continue
out.append(row | filled | {"source_id": row["source_id"]})
done.add(compound)
if complete:
done &= {f"{host}:{slug}:{row['source_id']}" for row in rows}
return out, done
def fetch_board(target: str) -> tuple[list[Row], bool]:
"""Fetch and parse one board directly: no `Source`, no `self.state`, just
"does this board answer, what does it list, and is that all of it". For
`app companies add|check`, which ask that question about one board at a
time outside any pull. Raises the same way `_fetch_rows` does."""
time outside any pull. The listing only, with no detail reads, and
robots.txt read afresh. Raises the same way `_fetch_rows` does, and
`_Disallowed` for a board robots.txt shuts out."""
with httpx.Client() as client:
_check_robots(client, target)
return _fetch_rows(client, target)
def failure_reason(exc: BaseException) -> str:
"""One short line for why a board did not answer: the status of an HTTP
error, else the exception's class and the first line of its message."""
if isinstance(exc, _Disallowed):
return "robots.txt disallows it"
if isinstance(exc, httpx.HTTPStatusError):
return f"{exc.response.status_code} {exc.response.reason_phrase}".strip()
lines = str(exc).strip().splitlines()
@@ -465,6 +597,7 @@ class ATSSource(Source):
host, _, slug = target.partition(":")
failing = f"ats:failing:{target}"
try:
_check_robots(client, target, self.state)
rows, complete = _fetch_rows(client, target)
except Exception as exc:
log.warning("ats: %s failed (%s)", target, exc)
@@ -481,6 +614,15 @@ class ATSSource(Source):
continue
self.state.pop(failing, None)
detailed_key = f"ats:detailed:{target}"
detailed: set[str] | None = None
if hasattr(atshosts.HOSTS.get(host), "detail"):
try:
detailed = set(json.loads(self.state.get(detailed_key) or "[]"))
except (ValueError, TypeError):
detailed = set()
rows, detailed = _read_details(client, target, rows, complete, detailed)
seen_ids = []
for row in rows:
# The compound id, matching `roles.source_id` — what
@@ -520,7 +662,12 @@ class ATSSource(Source):
# board that blew up half way through is neither marked ok nor
# has a partial id list treated as the whole of it. `complete`
# says whether the ids are the whole board, which the pull
# needs before it closes what they leave out.
# needs before it closes what they leave out. The ids read in
# detail are kept here too, after every row went out, so a
# pull stopped half way reads the rest again rather than
# losing their descriptions.
self.state[f"ats:ok:{target}"] = run_marker
self.state[f"ats:complete:{target}"] = "1" if complete else ""
self.state[f"ats:ids:{target}"] = json.dumps(seen_ids)
if detailed is not None:
self.state[detailed_key] = json.dumps(sorted(detailed))