Read a role's own page once for a host with detail, and fail a board whose robots.txt disallows it
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b02b72b4aa
commit
8b14d8575b
2 files changed
+578
-6
No files matched your search
+153
-6
@@ -6,7 +6,8 @@ unauthenticated JSON, XML or RSS — first-party, no scraping, and a role that
|
||||
drops off the board is *closed*, which is the cheapest staleness check there
|
||||
is. What a host says and how its feed reads is its module's; this file holds
|
||||
what every host shares: the fetch and its allow-list, paging and
|
||||
completeness, the classifier, and the plugin itself.
|
||||
completeness, the per-role detail reads, robots.txt, the classifier, and the
|
||||
plugin itself.
|
||||
|
||||
The allow-list and the classifier are ported from `openings.go` and
|
||||
`cmd/seedgen/main.go` in
|
||||
@@ -32,9 +33,10 @@ import json
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from collections.abc import Callable, Iterator
|
||||
from datetime import UTC, datetime
|
||||
from collections.abc import Callable, Iterator, MutableMapping
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
from urllib.robotparser import RobotFileParser
|
||||
|
||||
import httpx
|
||||
|
||||
@@ -110,6 +112,11 @@ class _TooLarge(ValueError):
|
||||
retry at the host's `fallback_url`."""
|
||||
|
||||
|
||||
class _Disallowed(Exception):
|
||||
"""The site's robots.txt disallows the page a board is read from (see
|
||||
`_check_robots`)."""
|
||||
|
||||
|
||||
def _get(client: httpx.Client, url: str, host: str, slug: str, *,
|
||||
method: str = "GET", payload: dict[str, Any] | None = None,
|
||||
headers: dict[str, str] | None = None) -> bytes:
|
||||
@@ -329,7 +336,8 @@ def careers_url(target: str) -> str:
|
||||
#: Pages read from one paged board at most, and the pause between two of
|
||||
#: them. Workable answers ten roles a page, SmartRecruiters a hundred and
|
||||
#: Teamtailor fifty, so the cap is 1,000, 10,000 and 5,000 roles; the pause
|
||||
#: keeps Workable, which answers a burst with 429, from seeing one.
|
||||
#: keeps Workable, which answers a burst with 429, from seeing one. The same
|
||||
#: pause separates two detail reads.
|
||||
MAX_PAGES = 100
|
||||
PAGE_DELAY = 0.5
|
||||
|
||||
@@ -408,18 +416,142 @@ def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
|
||||
return module.parse(body, slug), True
|
||||
|
||||
|
||||
# -------------------------------------------------------------- robots.txt
|
||||
|
||||
#: How long a robots.txt answer holds for one board before it is read again.
|
||||
ROBOTS_TTL = timedelta(days=1)
|
||||
|
||||
|
||||
def _robots_allows(body: bytes, page: str) -> bool:
|
||||
"""Whether a robots.txt lets `page` be fetched: by its rules for every
|
||||
agent (`*`), and by any it has for this app's own User-Agent token, so a
|
||||
site that shuts out this app by name is honoured as well."""
|
||||
parser = RobotFileParser()
|
||||
parser.parse(body.decode("utf-8", "replace").splitlines())
|
||||
return parser.can_fetch("*", page) and parser.can_fetch(USER_AGENT, page)
|
||||
|
||||
|
||||
def _check_robots(client: httpx.Client, target: str,
|
||||
state: MutableMapping[str, str] | None = None) -> None:
|
||||
"""Raise `_Disallowed` when the site of a host with a `robots_page` has a
|
||||
robots.txt that disallows that page. Read through `_get`, so from the
|
||||
board's own allowed hosts only, and at most once per `ROBOTS_TTL` for a
|
||||
board when `state` is given: the verdict and when it was reached are kept
|
||||
under `ats:robots:<target>`. A robots.txt that is missing (404 or 410)
|
||||
allows, and that verdict is kept like any other; one that cannot be read
|
||||
for another reason allows too, but is not kept, so the next pull asks
|
||||
again."""
|
||||
host, _, slug = target.partition(":")
|
||||
module = atshosts.HOSTS.get(host)
|
||||
if module is None or not hasattr(module, "robots_page") or not is_known_target(target):
|
||||
return
|
||||
page = module.robots_page(slug)
|
||||
key = f"ats:robots:{target}"
|
||||
now = datetime.now(UTC)
|
||||
if state is not None:
|
||||
try:
|
||||
kept = json.loads(state.get(key) or "{}")
|
||||
fresh = now - datetime.fromisoformat(kept["checked"]) < ROBOTS_TTL
|
||||
except (ValueError, KeyError, TypeError):
|
||||
kept, fresh = {}, False
|
||||
if fresh:
|
||||
if not kept.get("allowed", True):
|
||||
raise _Disallowed(f"robots.txt disallows {page}")
|
||||
return
|
||||
url = f"https://{httpx.URL(page).host}/robots.txt"
|
||||
try:
|
||||
body = _getter(client, host, slug)(url, headers={"Accept": "text/plain"})
|
||||
except httpx.HTTPStatusError as exc:
|
||||
if exc.response.status_code not in (404, 410):
|
||||
log.warning("ats: %s: robots.txt answered %s; allowed until the next pull",
|
||||
target, exc.response.status_code)
|
||||
return
|
||||
allowed = True
|
||||
except Exception as exc:
|
||||
log.warning("ats: %s: robots.txt not read (%s); allowed until the next pull",
|
||||
target, exc)
|
||||
return
|
||||
else:
|
||||
allowed = _robots_allows(body, page)
|
||||
if state is not None:
|
||||
state[key] = json.dumps({"checked": now.isoformat(timespec="seconds"),
|
||||
"allowed": allowed})
|
||||
if not allowed:
|
||||
raise _Disallowed(f"robots.txt disallows {page}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- details
|
||||
|
||||
#: Roles read in detail from one board in one pull at most. The rest of a
|
||||
#: large new board is read over the next pulls, 200 at a time.
|
||||
DETAIL_CAP = 200
|
||||
|
||||
|
||||
def _read_details(client: httpx.Client, target: str, rows: list[Row], complete: bool,
|
||||
detailed: set[str]) -> tuple[list[Row], set[str]]:
|
||||
"""The rows of a board whose host has `detail`, each role not yet in
|
||||
`detailed` (compound ids) filled from its own page, at most `DETAIL_CAP`
|
||||
of them and `PAGE_DELAY` apart. Returns the rows and the ids read in
|
||||
detail so far, for `ats:detailed:<target>`.
|
||||
|
||||
Every row not read in detail in this call goes out with an empty
|
||||
description: a role read on an earlier pull keeps the one it has, since
|
||||
`upsert_role` never overwrites a filled column with an empty value, and a
|
||||
teaser from the listing must not replace it. A detail that raises is
|
||||
logged and the role left for the next pull; a 429 stops the reads for this
|
||||
board until then. None of this touches completeness, which is the
|
||||
listing's alone. The ids of roles no longer listed are dropped, but only
|
||||
after a complete listing, so one cut short does not make the next pull
|
||||
read its missing roles again."""
|
||||
host, _, slug = target.partition(":")
|
||||
module = atshosts.HOSTS[host]
|
||||
get = _getter(client, host, slug)
|
||||
done = set(detailed)
|
||||
out: list[Row] = []
|
||||
tried = 0
|
||||
stopped = False
|
||||
for row in rows:
|
||||
compound = f"{host}:{slug}:{row['source_id']}"
|
||||
if compound in done or stopped or tried >= DETAIL_CAP:
|
||||
out.append(row | {"description": ""})
|
||||
continue
|
||||
time.sleep(PAGE_DELAY)
|
||||
tried += 1
|
||||
try:
|
||||
filled = module.detail(get, slug, row)
|
||||
except Exception as exc:
|
||||
if isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code == 429:
|
||||
log.warning("ats: %s: 429 on a detail read; the rest wait for the next pull",
|
||||
target)
|
||||
stopped = True
|
||||
else:
|
||||
log.warning("ats: %s: detail of %s failed (%s)", target, row["source_id"], exc)
|
||||
out.append(row | {"description": ""})
|
||||
continue
|
||||
out.append(row | filled | {"source_id": row["source_id"]})
|
||||
done.add(compound)
|
||||
if complete:
|
||||
done &= {f"{host}:{slug}:{row['source_id']}" for row in rows}
|
||||
return out, done
|
||||
|
||||
|
||||
def fetch_board(target: str) -> tuple[list[Row], bool]:
|
||||
"""Fetch and parse one board directly: no `Source`, no `self.state`, just
|
||||
"does this board answer, what does it list, and is that all of it". For
|
||||
`app companies add|check`, which ask that question about one board at a
|
||||
time outside any pull. Raises the same way `_fetch_rows` does."""
|
||||
time outside any pull. The listing only, with no detail reads, and
|
||||
robots.txt read afresh. Raises the same way `_fetch_rows` does, and
|
||||
`_Disallowed` for a board robots.txt shuts out."""
|
||||
with httpx.Client() as client:
|
||||
_check_robots(client, target)
|
||||
return _fetch_rows(client, target)
|
||||
|
||||
|
||||
def failure_reason(exc: BaseException) -> str:
|
||||
"""One short line for why a board did not answer: the status of an HTTP
|
||||
error, else the exception's class and the first line of its message."""
|
||||
if isinstance(exc, _Disallowed):
|
||||
return "robots.txt disallows it"
|
||||
if isinstance(exc, httpx.HTTPStatusError):
|
||||
return f"{exc.response.status_code} {exc.response.reason_phrase}".strip()
|
||||
lines = str(exc).strip().splitlines()
|
||||
@@ -465,6 +597,7 @@ class ATSSource(Source):
|
||||
host, _, slug = target.partition(":")
|
||||
failing = f"ats:failing:{target}"
|
||||
try:
|
||||
_check_robots(client, target, self.state)
|
||||
rows, complete = _fetch_rows(client, target)
|
||||
except Exception as exc:
|
||||
log.warning("ats: %s failed (%s)", target, exc)
|
||||
@@ -481,6 +614,15 @@ class ATSSource(Source):
|
||||
continue
|
||||
self.state.pop(failing, None)
|
||||
|
||||
detailed_key = f"ats:detailed:{target}"
|
||||
detailed: set[str] | None = None
|
||||
if hasattr(atshosts.HOSTS.get(host), "detail"):
|
||||
try:
|
||||
detailed = set(json.loads(self.state.get(detailed_key) or "[]"))
|
||||
except (ValueError, TypeError):
|
||||
detailed = set()
|
||||
rows, detailed = _read_details(client, target, rows, complete, detailed)
|
||||
|
||||
seen_ids = []
|
||||
for row in rows:
|
||||
# The compound id, matching `roles.source_id` — what
|
||||
@@ -520,7 +662,12 @@ class ATSSource(Source):
|
||||
# board that blew up half way through is neither marked ok nor
|
||||
# has a partial id list treated as the whole of it. `complete`
|
||||
# says whether the ids are the whole board, which the pull
|
||||
# needs before it closes what they leave out.
|
||||
# needs before it closes what they leave out. The ids read in
|
||||
# detail are kept here too, after every row went out, so a
|
||||
# pull stopped half way reads the rest again rather than
|
||||
# losing their descriptions.
|
||||
self.state[f"ats:ok:{target}"] = run_marker
|
||||
self.state[f"ats:complete:{target}"] = "1" if complete else ""
|
||||
self.state[f"ats:ids:{target}"] = json.dumps(seen_ids)
|
||||
if detailed is not None:
|
||||
self.state[detailed_key] = json.dumps(sorted(detailed))
|
||||
Reference in new issue
Block a user