Runner: readers read from their declaration, ats and search thin over it
plugins/readers/_runner.py takes the guarded fetch, the listing (whole, paged, one series per filter value unioned by id), completeness from READS, the sweep of a board over the page cap across pulls, the 10% guard on a filtered or empty="suspect" read, facets kept a day, the detail reads, the raw an Item carries and the failure bookkeeping out of plugins/ats.py. Ask comes from a subscription's params (ask_for, open for the reach), with a declared key resolved into Ask.params["key"] and refused when missing. The runtime robots.txt check goes (decided at build time, READS.robots). plugins/search.py reads the providers of readers.PROVIDERS through the runner and applies the saved pages param. A filtered board's close keeps roles with a vote or an application. DETAIL_FIELDS gain places, teaser, country and employment. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
779b136757
commit
902089a5e2
14 files changed
+1608
-1492
No files matched your search
+131
-589
@@ -2,50 +2,44 @@
|
||||
`plugins/readers/`, one module per host.
|
||||
|
||||
Every one of these hosts serves a company's own list of roles as public,
|
||||
unauthenticated JSON, XML or RSS — first-party, no scraping, and a role that
|
||||
drops off the board is *closed*, which is the cheapest staleness check there
|
||||
is. What a host says and how its feed reads is its module's; this file holds
|
||||
what every host shares: the fetch and its allow-list, paging and
|
||||
completeness, the per-role detail reads, robots.txt, the classifier, and the
|
||||
plugin itself.
|
||||
unauthenticated JSON, XML or RSS, and a role that drops off the board is
|
||||
*closed*, which is the cheapest staleness check there is. What a host says
|
||||
and how its feed reads is its module's, what every reader shares is
|
||||
`plugins/readers/_runner.py`'s; this file is the `ats` Source over the runner,
|
||||
the classifier, and the names the app and the tests reach for on `ats`.
|
||||
|
||||
The allow-list and the classifier are ported from `openings.go` and
|
||||
`cmd/seedgen/main.go` in
|
||||
[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT), including
|
||||
`classify`'s nine functions and their keywords. The body cap is not a
|
||||
straight port — see `MAX_BYTES` below.
|
||||
`classify`'s nine functions and their keywords.
|
||||
|
||||
A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
|
||||
`lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what
|
||||
its slug is. Every board is fetched independently; one that fails or times
|
||||
out is logged, recorded as failing under `ats:failing:<target>` (see
|
||||
`ATSSource.fetch`), and skipped, and the rest continue.
|
||||
its slug is. Every board is read independently; one that fails is logged,
|
||||
recorded as failing under `ats:failing:<target>`, and skipped. A board's
|
||||
subscription `params` (`{"countries": ["DK"], "remote": true}`) are its
|
||||
filter, narrowed to what its reader declares (`_runner.ask_for`).
|
||||
|
||||
A host that pages states a total or does not. A board with a total counts as
|
||||
complete only when it read as many distinct roles as the total says, one
|
||||
without when it read up to an empty page (see `_paged`); a host that answers
|
||||
with the whole board at once is complete whenever it parses.
|
||||
The limits below are read when a board is read, so a test that replaces one
|
||||
on this module reaches the runner.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ipaddress
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
from collections.abc import Callable, Iterator, MutableMapping
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from typing import Any
|
||||
from collections.abc import Callable, Iterator
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from urllib.parse import unquote
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
from desk.sources import (
|
||||
API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source,
|
||||
SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT,
|
||||
SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT, # noqa: F401
|
||||
)
|
||||
|
||||
try:
|
||||
@@ -58,6 +52,7 @@ except ModuleNotFoundError:
|
||||
# file. So the root goes on the path then, and only then.
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from plugins import readers
|
||||
from plugins.readers import _runner
|
||||
from plugins.readers._shared import Get, Row
|
||||
|
||||
# The names the parsers had here before they moved to their host modules,
|
||||
@@ -70,156 +65,57 @@ from plugins.readers.teamtailor import parse as _parse_teamtailor # noqa: F401
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
#: openings.go's 8 MB body cap is not kept: it was written against arbitrary URLs, and every
|
||||
#: URL fetched here is a feed on a fixed allow-list of ATS hosts, Greenhouse's
|
||||
#: with `?content=true`. The real run of 2026-09-23 hit it on the biggest employers —
|
||||
#: Anduril at 41 MB, Anthropic, OpenAI, Databricks, Carvana, Axon and
|
||||
#: Airwallex between 9 and 19 MB — and skipped their boards outright. Raised
|
||||
#: to 96 MB so those boards fit; a board that still exceeds it gets one retry
|
||||
#: at its host's `fallback_url` where it has one (Greenhouse, without
|
||||
#: `?content=true`, see `_fetch_rows`), so at least the titles land.
|
||||
MAX_BYTES = 96 << 20
|
||||
#: The runner's limits, kept here as the `ats` Source's own: see `_runner`.
|
||||
MAX_BYTES = _runner.MAX_BYTES
|
||||
TIMEOUTS = _runner.TIMEOUTS
|
||||
DEADLINE = _runner.DEADLINE
|
||||
MAX_PAGES = _runner.MAX_PAGES
|
||||
PAGE_DELAY = _runner.PAGE_DELAY
|
||||
DETAIL_CAP = _runner.DETAIL_CAP
|
||||
DETAIL_FIELDS = _runner.DETAIL_FIELDS
|
||||
|
||||
#: openings.go's 10 s did not hold: `greenhouse:mongodb` and `greenhouse:feedzai`
|
||||
#: failed a pull of 2026-09-28 with a read timeout. Measured live on
|
||||
#: 2026-09-29, three runs each of mongodb, feedzai and the five Greenhouse
|
||||
#: boards with the most open roles (databricks, stripe, anthropic, feverup,
|
||||
#: datadog), with and without `pay_transparency=true`, cached and with a
|
||||
#: cache-busting parameter (84 requests; CloudFront answered every one
|
||||
#: "Miss", so all reached the origin): the body arrives in under 1 s once it
|
||||
#: starts (at most 0.09 s between two chunks, 10.4 MB the largest), but the
|
||||
#: first byte sometimes waits: 16.6 s (feverup, no flag), 14.4 s (anthropic,
|
||||
#: flag), 10.9 s (feverup, flag), 5.2 s and 3.6 s, and under 2.6 s otherwise.
|
||||
#: The flag made no difference that three runs could tell apart from that
|
||||
#: noise. So a read may wait 30 s, about twice the worst first byte seen,
|
||||
#: and a whole request, redirects and body included, 60 s (see `_get`),
|
||||
#: about four times the slowest seen, which still leaves a 96 MB body room
|
||||
#: on a link far slower than the one measured.
|
||||
TIMEOUTS = httpx.Timeout(10.0, read=30.0)
|
||||
DEADLINE = 60.0
|
||||
#: The runner's errors, under the names they had here.
|
||||
_Refused = _runner.Refused
|
||||
_TooLarge = _runner.TooLarge
|
||||
_local_host = _runner.local_host
|
||||
|
||||
#: host -> the name the host gives itself: the hosts this plugin reads, in
|
||||
#: the order a select offers them. What the Sources page's host select is
|
||||
#: rendered from, so it cannot offer a host `is_known_target` refuses.
|
||||
HOSTS: dict[str, str] = {name: module.LABEL for name, module in readers.HOSTS.items()}
|
||||
|
||||
#: host -> parser, for the hosts whose module parses a whole body.
|
||||
_PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = {
|
||||
name: module.parse for name, module in readers.HOSTS.items() if hasattr(module, "parse")}
|
||||
|
||||
|
||||
def _limits() -> _runner.Limits:
|
||||
return _runner.Limits(max_pages=MAX_PAGES, page_delay=PAGE_DELAY, detail_cap=DETAIL_CAP)
|
||||
|
||||
|
||||
def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
|
||||
"""The hosts a request or a redirect may land on for one board: what its
|
||||
host module's `allowed` says, which is one fixed API host or the
|
||||
company's own site. Anything else is refused, at the first hop and at
|
||||
every redirect: a feed is external content, and a board row must never
|
||||
be able to point this fetch at an internal address."""
|
||||
host module's `allowed` says. Anything else is refused, at the first hop
|
||||
and at every redirect."""
|
||||
module = readers.HOSTS.get(host)
|
||||
return module.allowed(slug) if module is not None else lambda h: False
|
||||
|
||||
|
||||
class _Refused(Exception):
|
||||
"""A board answered somewhere its allow-list does not cover."""
|
||||
|
||||
|
||||
class _TooLarge(ValueError):
|
||||
"""A response exceeded `MAX_BYTES`. A `ValueError` so a caller that only
|
||||
knows the old shape still catches it; `_fetch_rows` catches it by type to
|
||||
retry at the host's `fallback_url`."""
|
||||
|
||||
|
||||
class _Disallowed(Exception):
|
||||
"""The site's robots.txt disallows the page a board is read from (see
|
||||
`_check_robots`)."""
|
||||
|
||||
|
||||
class _RobotsUnreachable(Exception):
|
||||
"""The site's robots.txt could not be read: a 5xx, a 429, a timeout, a
|
||||
connection error or a redirect off the site. RFC 9309 says to assume
|
||||
the site disallows everything then, so the board is not read this pull."""
|
||||
|
||||
|
||||
#: Host name suffixes that name a machine on a private network, never a
|
||||
#: company's careers site.
|
||||
_LOCAL_SUFFIXES = (".localhost", ".local", ".internal", ".lan", ".home.arpa")
|
||||
|
||||
|
||||
def _local_host(name: str) -> bool:
|
||||
"""Whether a host name is an address or a local name rather than a public
|
||||
DNS name: an IP literal (v4 or v6), anything whose last label is a number
|
||||
(`2130706433`, `127.1` and `0x7f000001` all reach 127.0.0.1 through the
|
||||
resolver), `localhost`, or a private suffix. A Teamtailor slug can be any
|
||||
host name, so the allow-list alone would let one point the fetch here."""
|
||||
name = name.strip("[]").rstrip(".").lower()
|
||||
try:
|
||||
ipaddress.ip_address(name)
|
||||
return True
|
||||
except ValueError:
|
||||
pass
|
||||
last = name.rsplit(".", 1)[-1]
|
||||
return (last.isdigit() or bool(re.fullmatch(r"0x[0-9a-f]+", last))
|
||||
or name == "localhost" or name.endswith(_LOCAL_SUFFIXES))
|
||||
|
||||
|
||||
def _get(client: httpx.Client, url: str, host: str, slug: str, *,
|
||||
method: str = "GET", payload: dict[str, Any] | None = None,
|
||||
headers: dict[str, str] | None = None, www_twin: bool = False) -> bytes:
|
||||
"""Fetch one URL for a board: https only on the default port, never an
|
||||
address or a local name (`_local_host`), the host's own allow-list, at
|
||||
most three redirect hops each checked the same way, and the body cut off
|
||||
at `MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST
|
||||
sends `payload` as JSON, `{}` when there is none. `headers` are sent on
|
||||
top of the User-Agent and Accept every request carries, and replace
|
||||
either where they name it. With `www_twin`, a redirect may also land on
|
||||
the first URL's host with `www.` added or taken off, which only the
|
||||
robots.txt read asks for (see `_check_robots`).
|
||||
|
||||
The body is streamed and counted as it arrives, so an oversized one is
|
||||
refused at `MAX_BYTES` rather than after it has been read whole, and
|
||||
the whole request, redirects included, must end within `DEADLINE`: the
|
||||
per-read `TIMEOUTS` alone let a board that trickles bytes run on for
|
||||
good. The deadline is checked between chunks, so it can be overrun by
|
||||
at most one read timeout."""
|
||||
allow = _allowed_host(host, slug)
|
||||
if www_twin:
|
||||
bare = httpx.URL(url).host.removeprefix("www.")
|
||||
board_allow = allow
|
||||
allow = lambda h: board_allow(h) or h.removeprefix("www.") == bare # noqa: E731
|
||||
sent = {"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"} | (headers or {})
|
||||
deadline = time.monotonic() + DEADLINE
|
||||
current = url
|
||||
for _hop in range(4):
|
||||
parsed = httpx.URL(current)
|
||||
if (parsed.scheme != "https" or parsed.port not in (None, 443)
|
||||
or _local_host(parsed.host) or not allow(parsed.host)):
|
||||
raise _Refused(f"{current} is not an allowed host for {host}")
|
||||
with client.stream(
|
||||
method, current, timeout=TIMEOUTS, follow_redirects=False, headers=sent,
|
||||
json=(payload or {}) if method == "POST" else None,
|
||||
) as response:
|
||||
if response.is_redirect:
|
||||
location = response.headers.get("location") or ""
|
||||
if not location:
|
||||
raise ValueError("redirect with no location")
|
||||
current = str(httpx.URL(current).join(location))
|
||||
continue
|
||||
response.raise_for_status()
|
||||
chunks: list[bytes] = []
|
||||
size = 0
|
||||
for chunk in response.iter_bytes():
|
||||
size += len(chunk)
|
||||
if size > MAX_BYTES:
|
||||
raise _TooLarge(f"response too large (over {MAX_BYTES} bytes)")
|
||||
if time.monotonic() > deadline:
|
||||
raise httpx.ReadTimeout(f"no whole answer within {DEADLINE:.0f} s",
|
||||
request=response.request)
|
||||
chunks.append(chunk)
|
||||
return b"".join(chunks)
|
||||
raise ValueError("too many redirects")
|
||||
headers: dict[str, str] | None = None) -> bytes:
|
||||
"""`_runner.fetch` for one board, with this module's `MAX_BYTES` and
|
||||
`DEADLINE`."""
|
||||
return _runner.fetch(client, url, _allowed_host(host, slug), method=method,
|
||||
payload=payload, headers=headers, max_bytes=MAX_BYTES,
|
||||
deadline=DEADLINE, name=host)
|
||||
|
||||
|
||||
def _getter(client: httpx.Client, host: str, slug: str) -> Get:
|
||||
"""The `get` a host module is handed: `_get` bound to one board. `_get`
|
||||
is looked up when a request is made, not when this is built, so a test
|
||||
that replaces `ats._get` reaches every host; `payload` and `headers` are
|
||||
passed on only when given, which keeps a replacement that predates them
|
||||
working."""
|
||||
is looked up when a request is made, so a test that replaces `ats._get`
|
||||
reaches every host; `payload` and `headers` are passed on only when
|
||||
given, which keeps a replacement that predates them working."""
|
||||
def get(url: str, *, method: str = "GET", payload: dict[str, Any] | None = None,
|
||||
headers: dict[str, str] | None = None) -> bytes:
|
||||
extra: dict[str, Any] = {}
|
||||
@@ -231,12 +127,6 @@ def _getter(client: httpx.Client, host: str, slug: str) -> Get:
|
||||
return get
|
||||
|
||||
|
||||
#: host -> parser, for the hosts whose module parses a whole body. Every one
|
||||
#: takes the raw response body and the slug.
|
||||
_PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = {
|
||||
name: module.parse for name, module in readers.HOSTS.items() if hasattr(module, "parse")}
|
||||
|
||||
|
||||
# --------------------------------------------------------- the classifier
|
||||
|
||||
#: The fixed set of functions, and the keyword rules that map a title
|
||||
@@ -381,362 +271,40 @@ def careers_url(target: str) -> str:
|
||||
return module.careers_url(slug) if module is not None else ""
|
||||
|
||||
|
||||
#: Pages read from one paged board at most, and the pause between two of
|
||||
#: them. Workable answers ten roles a page, SmartRecruiters a hundred and
|
||||
#: Teamtailor fifty, so the cap is 1,000, 10,000 and 5,000 roles; the pause
|
||||
#: keeps Workable, which answers a burst with 429, from seeing one. The same
|
||||
#: pause separates two detail reads.
|
||||
MAX_PAGES = 100
|
||||
PAGE_DELAY = 0.5
|
||||
|
||||
|
||||
def _paged(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
|
||||
"""Every page of a board whose host pages, `PAGE_DELAY` apart, until the
|
||||
host has no next page, the stated total is reached, or `MAX_PAGES`.
|
||||
Returns the rows and whether the board is complete: as many distinct
|
||||
results read as the host's stated total, or, for a host that states none
|
||||
(Teamtailor), the read ended on an empty page (a page with nothing new in
|
||||
it ends the read short of that, incomplete, so an offset the host ignores
|
||||
cannot loop). Counted on the raw results, so a row the parser drops does
|
||||
not make a board incomplete, and on distinct ids, so a page that shifted
|
||||
under an offset and repeated a role does not make it complete.
|
||||
|
||||
A 429 after the first page stops the read with what it has, incomplete;
|
||||
one on the first page raises like any other failure."""
|
||||
host, _, slug = target.partition(":")
|
||||
module = readers.HOSTS[host]
|
||||
get = _getter(client, host, slug)
|
||||
rows: dict[str, Row] = {}
|
||||
seen: set[str] = set()
|
||||
total: Any = None
|
||||
cursor: Any = None
|
||||
for page in range(MAX_PAGES):
|
||||
if page:
|
||||
time.sleep(PAGE_DELAY)
|
||||
try:
|
||||
data, results, total, cursor = module.page(get, slug, cursor)
|
||||
except httpx.HTTPStatusError as exc:
|
||||
if not page or exc.response.status_code != 429:
|
||||
raise
|
||||
log.warning("ats: %s: 429 on page %d; kept %d of %s, not complete",
|
||||
target, page + 1, len(seen), total)
|
||||
return list(rows.values()), False
|
||||
before = len(seen)
|
||||
seen |= {str(r.get("id") or r.get("shortcode") or "") for r in results}
|
||||
for row in module.rows(data, slug):
|
||||
rows.setdefault(row["source_id"], row)
|
||||
if (cursor is None or (isinstance(total, int) and len(seen) >= total)
|
||||
or (total is None and len(seen) == before)):
|
||||
break
|
||||
complete = len(seen) == total if isinstance(total, int) else total is None and cursor is None
|
||||
if not complete:
|
||||
log.warning("ats: %s: read %d of %s stated; not complete", target, len(seen),
|
||||
"an unstated total" if total is None else total)
|
||||
return list(rows.values()), complete
|
||||
|
||||
|
||||
def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
|
||||
"""One board's listing, fetched and parsed: its rows and whether they are
|
||||
the whole board. Raises on any failure — refused host, timeout, a feed
|
||||
shape the host's parser does not understand. A host that does not page
|
||||
answers with the whole board, so it is complete whenever it parses; a
|
||||
paged one is complete when `_paged` says so.
|
||||
|
||||
A body that exceeds `MAX_BYTES` gets one retry at the host's
|
||||
`fallback_url` where it has one (Greenhouse, without `?content=true`):
|
||||
the description is lost, but the titles land rather than the whole board
|
||||
being skipped."""
|
||||
def _fetch_rows(client: httpx.Client, target: str,
|
||||
ask: readers.Ask | None = None) -> tuple[list[Row], bool]:
|
||||
"""One board's listing, read and parsed, unfiltered unless `ask` says:
|
||||
its rows and whether they are the whole board. Raises on any failure.
|
||||
No sweep and no state: what `fetch_board` and the host tests read."""
|
||||
if not is_known_target(target):
|
||||
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
||||
host, _, slug = target.partition(":")
|
||||
module = readers.HOSTS[host]
|
||||
if hasattr(module, "page"):
|
||||
return _paged(client, target)
|
||||
get = _getter(client, host, slug)
|
||||
try:
|
||||
body = get(module.feed_url(slug))
|
||||
except _TooLarge:
|
||||
if not hasattr(module, "fallback_url"):
|
||||
raise
|
||||
log.warning("ats: %s: body exceeded %d bytes; retrying at the smaller feed",
|
||||
target, MAX_BYTES)
|
||||
body = get(module.fallback_url(slug))
|
||||
return module.parse(body, slug), True
|
||||
|
||||
|
||||
# -------------------------------------------------------------- robots.txt
|
||||
|
||||
#: How long a robots.txt answer holds for one board before it is read again.
|
||||
ROBOTS_TTL = timedelta(days=1)
|
||||
|
||||
|
||||
#: This app's product token, the name a robots.txt group addresses it by:
|
||||
#: the `desk` of `desk/<version> (personal aggregator)`.
|
||||
ROBOTS_TOKEN = USER_AGENT.split("/", 1)[0].strip().lower()
|
||||
|
||||
#: One rule of a group: `(allow, path pattern)`.
|
||||
_Rule = tuple[bool, str]
|
||||
|
||||
|
||||
def _robots_groups(text: str) -> list[tuple[set[str], list[_Rule]]]:
|
||||
"""The groups of a robots.txt, per RFC 9309: one or more `user-agent`
|
||||
lines (their tokens lowercased) and the `allow` and `disallow` rules that
|
||||
follow them, up to the next `user-agent` line after a rule. Comments,
|
||||
other keys and rules before any `user-agent` line are left out, and so is
|
||||
an empty rule, which allows everything and so says nothing."""
|
||||
groups: list[tuple[set[str], list[_Rule]]] = []
|
||||
in_agents = False
|
||||
for line in text.splitlines():
|
||||
key, sep, value = line.split("#", 1)[0].partition(":")
|
||||
key, value = key.strip().lower(), value.strip()
|
||||
if not sep:
|
||||
continue
|
||||
if key == "user-agent":
|
||||
if not in_agents:
|
||||
groups.append((set(), []))
|
||||
in_agents = True
|
||||
groups[-1][0].add(value.lower())
|
||||
elif key in ("allow", "disallow") and groups:
|
||||
in_agents = False
|
||||
if value:
|
||||
groups[-1][1].append((key == "allow", value))
|
||||
return groups
|
||||
|
||||
|
||||
def _robots_rules(groups: list[tuple[set[str], list[_Rule]]], token: str) -> list[_Rule]:
|
||||
"""The rules for `token`: every group that names it (matched whole and
|
||||
ignoring case, as RFC 9309 says, not as a substring), merged; else every
|
||||
`*` group, merged; else none, which allows everything."""
|
||||
for wanted in (token, "*"):
|
||||
rules = [rule for agents, group in groups if wanted in agents for rule in group]
|
||||
if rules or any(wanted in agents for agents, _group in groups):
|
||||
return rules
|
||||
return []
|
||||
|
||||
|
||||
def _robots_pattern(path: str) -> re.Pattern[str]:
|
||||
"""A rule's path as a regex anchored at the start: `*` is any run of
|
||||
characters, and a `$` at the end anchors the end."""
|
||||
anchored = path.endswith("$")
|
||||
body = ".*".join(re.escape(part) for part in path.rstrip("$").split("*"))
|
||||
return re.compile(body + (r"\Z" if anchored else ""), re.DOTALL)
|
||||
|
||||
|
||||
def _robots_verdict(rules: list[_Rule], path: str) -> bool:
|
||||
"""Whether `path` is allowed by `rules`: the longest matching rule wins,
|
||||
an allow wins a tie, and no match allows. Both sides are compared
|
||||
percent-decoded, so `%7E` and `~` are one character."""
|
||||
path = unquote(path)
|
||||
best: tuple[int, bool] | None = None
|
||||
for allow, pattern in rules:
|
||||
if _robots_pattern(unquote(pattern)).match(path):
|
||||
weight = (len(pattern), allow)
|
||||
best = weight if best is None or weight > best else best
|
||||
return best is None or best[1]
|
||||
|
||||
|
||||
def _robots_allows(body: bytes, page: str) -> bool:
|
||||
"""Whether a robots.txt lets `page` be fetched, by RFC 9309: by its rules
|
||||
for every agent (`*`), and by those it has for this app's own token
|
||||
(`ROBOTS_TOKEN`, which fall back to `*`), so a site that shuts out this
|
||||
app by name is honoured as well. Paths are case-sensitive; a byte order
|
||||
mark is dropped."""
|
||||
groups = _robots_groups(body.decode("utf-8-sig", "replace"))
|
||||
url = httpx.URL(page)
|
||||
path = (url.raw_path or b"/").decode("ascii", "replace")
|
||||
return all(_robots_verdict(_robots_rules(groups, token), path)
|
||||
for token in ("*", ROBOTS_TOKEN))
|
||||
|
||||
|
||||
def _check_robots(client: httpx.Client, target: str,
|
||||
state: MutableMapping[str, str] | None = None) -> None:
|
||||
"""Raise `_Disallowed` when the site of a host with a `robots_page` has a
|
||||
robots.txt that disallows that page, and `_RobotsUnreachable` when it
|
||||
cannot be read. Read through `_get`, so from the board's own allowed
|
||||
hosts only, except that a redirect between the host and its `www.` twin
|
||||
is followed (at most three hops, https only); any other redirect off the
|
||||
site is unreachable. With `state`, the verdict and when it was reached
|
||||
are kept under `ats:robots:<target>` and held for `ROBOTS_TTL`.
|
||||
|
||||
By RFC 9309: a 4xx other than 429 means there is no robots.txt to obey,
|
||||
which allows, and is kept like any other verdict. A 5xx, a 429, a
|
||||
timeout or a connection error means the file is unreachable, which the
|
||||
RFC says to treat as disallowing everything: the board is not read on
|
||||
this pull, fails with "robots.txt could not be read", and nothing is
|
||||
kept, so the next pull asks again. A 429 counts as unreachable although
|
||||
it is a 4xx, since it says "not now" rather than "there is none"."""
|
||||
host, _, slug = target.partition(":")
|
||||
module = readers.HOSTS.get(host)
|
||||
if module is None or not hasattr(module, "robots_page") or not is_known_target(target):
|
||||
return
|
||||
page = module.robots_page(slug)
|
||||
key = f"ats:robots:{target}"
|
||||
now = datetime.now(UTC)
|
||||
if state is not None:
|
||||
try:
|
||||
kept = json.loads(state.get(key) or "{}")
|
||||
fresh = now - datetime.fromisoformat(kept["checked"]) < ROBOTS_TTL
|
||||
except (ValueError, KeyError, TypeError):
|
||||
kept, fresh = {}, False
|
||||
if fresh:
|
||||
if not kept.get("allowed", True):
|
||||
raise _Disallowed(f"robots.txt disallows {page}")
|
||||
return
|
||||
url = f"https://{httpx.URL(page).host}/robots.txt"
|
||||
try:
|
||||
body = _get(client, url, host, slug, headers={"Accept": "text/plain"}, www_twin=True)
|
||||
except httpx.HTTPStatusError as exc:
|
||||
code = exc.response.status_code
|
||||
if not 400 <= code < 500 or code == 429:
|
||||
raise _RobotsUnreachable(f"robots.txt answered {code}") from exc
|
||||
allowed = True
|
||||
except Exception as exc:
|
||||
raise _RobotsUnreachable(f"robots.txt not read: {exc}") from exc
|
||||
else:
|
||||
# A host that serves a path whatever its case is matched without
|
||||
# case, so a rule written `/Jobs/` forbids a site named `jobs` too.
|
||||
if getattr(module, "ROBOTS_ANY_CASE", False):
|
||||
allowed = _robots_allows(body.lower(), page.lower())
|
||||
else:
|
||||
allowed = _robots_allows(body, page)
|
||||
if state is not None:
|
||||
state[key] = json.dumps({"checked": now.isoformat(timespec="seconds"),
|
||||
"allowed": allowed})
|
||||
if not allowed:
|
||||
raise _Disallowed(f"robots.txt disallows {page}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- details
|
||||
|
||||
#: Roles read in detail from one board in one pull at most. The rest of a
|
||||
#: large new board is read over the next pulls, 200 at a time.
|
||||
DETAIL_CAP = 200
|
||||
|
||||
#: The row keys a host's `detail` may fill: those whose column survives a
|
||||
#: later pull that sends the listing's empty value for them, so what a role's
|
||||
#: own page said is not wiped the pull after it was read. `description`,
|
||||
#: `location` (`place`, `place_key`, `country`), `remote` and `currency` are
|
||||
#: `upsert_role`'s keep-filled columns, `salary_min` and `salary_max` are only
|
||||
#: overwritten by a figure, and `posted` feeds `posted_at`, which keeps the
|
||||
#: earliest date a role ever had. A filled column is still overwritten by
|
||||
#: another filled value, so `_read_details` withholds the listing's place and
|
||||
#: remote for a role already read. `title` and `url` are not here, since the
|
||||
#: listing always states them and would overwrite them; nor are `department`
|
||||
#: (no column of its own, only `raw` and the classifier's guess) and
|
||||
#: `salary_text`, both rewritten from the listing on every pull.
|
||||
DETAIL_FIELDS = ("description", "location", "remote", "currency", "salary_min",
|
||||
"salary_max", "posted")
|
||||
listing = _runner.read_listing(readers.HOSTS[host], slug, _getter(client, host, slug), ask,
|
||||
limits=_limits())
|
||||
return listing.rows, listing.complete
|
||||
|
||||
|
||||
def _read_details(client: httpx.Client, target: str, rows: list[Row], complete: bool,
|
||||
detailed: set[str], failed: set[str]
|
||||
) -> tuple[list[Row], set[str], set[str]]:
|
||||
"""The rows of a board whose host has `detail`, with the roles not yet
|
||||
read in detail filled from their own pages: at most `DETAIL_CAP` of them,
|
||||
`PAGE_DELAY` apart, first the roles never tried and then those whose
|
||||
detail failed before (`failed`), so a few roles that always fail cannot
|
||||
take the whole cap pull after pull. Returns the rows and the compound ids
|
||||
read (for `ats:detailed:<target>`) and failed (for
|
||||
`ats:detailfailed:<target>`).
|
||||
|
||||
Only `DETAIL_FIELDS` are taken from what `detail` returns; any other key
|
||||
is left out with a debug line. Every row not read in detail in this call
|
||||
goes out with an empty description: a role read on an earlier pull keeps
|
||||
the one it has, since `upsert_role` never overwrites a filled column with
|
||||
an empty value, and a teaser from the listing must not replace it. A role
|
||||
read on an earlier pull goes out without the listing's place and remote
|
||||
as well, for the same reason: the listing states them every pull
|
||||
("2 Locations"), and would replace what the role's own page said.
|
||||
|
||||
A detail that answers 404 or 410 counts as read: the role is gone or has
|
||||
no page of its own, and asking again would not change that. One that
|
||||
raises otherwise is logged and the role kept in `failed`; a 429 stops the
|
||||
reads for this board until the next pull, and does not mark the role as
|
||||
failing. None of this touches completeness, which is the listing's alone.
|
||||
The ids of roles no longer listed are dropped from both sets, but only
|
||||
after a complete listing, so one cut short does not make the next pull
|
||||
read its missing roles again."""
|
||||
"""`_runner.read_details` for one board, with `rows` as the whole listing
|
||||
when `complete`."""
|
||||
host, _, slug = target.partition(":")
|
||||
module = readers.HOSTS[host]
|
||||
get = _getter(client, host, slug)
|
||||
done, failing = set(detailed), set(failed) - set(detailed)
|
||||
|
||||
def compound(row: Row) -> str:
|
||||
return f"{host}:{slug}:{row['source_id']}"
|
||||
|
||||
unread = [row for row in rows if compound(row) not in done]
|
||||
queue = ([row for row in unread if compound(row) not in failing]
|
||||
+ [row for row in unread if compound(row) in failing])[:DETAIL_CAP]
|
||||
filled: dict[str, Row] = {}
|
||||
for row in queue:
|
||||
key = compound(row)
|
||||
time.sleep(PAGE_DELAY)
|
||||
try:
|
||||
answer = module.detail(get, slug, row)
|
||||
except httpx.HTTPStatusError as exc:
|
||||
code = exc.response.status_code
|
||||
if code == 429:
|
||||
log.warning("ats: %s: 429 on a detail read; the rest wait for the next pull",
|
||||
target)
|
||||
break
|
||||
if code in (404, 410):
|
||||
done.add(key)
|
||||
failing.discard(key)
|
||||
else:
|
||||
log.warning("ats: %s: detail of %s answered %s", target, row["source_id"], code)
|
||||
failing.add(key)
|
||||
continue
|
||||
except Exception as exc:
|
||||
log.warning("ats: %s: detail of %s failed (%s)", target, row["source_id"], exc)
|
||||
failing.add(key)
|
||||
continue
|
||||
if ignored := sorted(set(answer) - set(DETAIL_FIELDS)):
|
||||
log.debug("ats: %s: detail of %s: %s left out, not in DETAIL_FIELDS",
|
||||
target, row["source_id"], ", ".join(ignored))
|
||||
filled[key] = {k: v for k, v in answer.items() if k in DETAIL_FIELDS}
|
||||
done.add(key)
|
||||
failing.discard(key)
|
||||
out = []
|
||||
for row in rows:
|
||||
key = compound(row)
|
||||
if key in filled:
|
||||
out.append(row | {"description": ""} | filled[key])
|
||||
elif key in detailed:
|
||||
out.append(row | {"description": "", "location": "", "remote": None})
|
||||
else:
|
||||
out.append(row | {"description": ""})
|
||||
if complete:
|
||||
listed = {compound(row) for row in rows}
|
||||
done &= listed
|
||||
failing &= listed
|
||||
return out, done, failing
|
||||
listed = {f"{host}:{slug}:{row['source_id']}" for row in rows} if complete else None
|
||||
return _runner.read_details(readers.HOSTS[host], host, slug, _getter(client, host, slug),
|
||||
rows, detailed, failed, listed=listed, limits=_limits())
|
||||
|
||||
|
||||
def fetch_board(target: str) -> tuple[list[Row], bool]:
|
||||
"""Fetch and parse one board directly: no `Source`, no `self.state`, just
|
||||
"does this board answer, what does it list, and is that all of it". For
|
||||
`app companies add|check`, which ask that question about one board at a
|
||||
time outside any pull. The listing only, with no detail reads, and
|
||||
robots.txt read afresh. Raises the same way `_fetch_rows` does, and
|
||||
`_Disallowed` or `_RobotsUnreachable` for a board robots.txt shuts out or
|
||||
that cannot be checked."""
|
||||
"""Fetch and parse one board directly: no `Source`, no state, just "does
|
||||
this board answer, what does it list, and is that all of it". For `app
|
||||
companies add|check`. The listing only, unfiltered, no detail reads.
|
||||
Raises the same way `_fetch_rows` does."""
|
||||
with httpx.Client() as client:
|
||||
_check_robots(client, target)
|
||||
return _fetch_rows(client, target)
|
||||
|
||||
|
||||
def failure_reason(exc: BaseException) -> str:
|
||||
"""One short line for why a board did not answer: the status of an HTTP
|
||||
error, else the exception's class and the first line of its message."""
|
||||
if isinstance(exc, _Disallowed):
|
||||
return "robots.txt disallows it"
|
||||
if isinstance(exc, _RobotsUnreachable):
|
||||
return "robots.txt could not be read"
|
||||
if isinstance(exc, httpx.HTTPStatusError):
|
||||
return f"{exc.response.status_code} {exc.response.reason_phrase}".strip()
|
||||
lines = str(exc).strip().splitlines()
|
||||
text = f"{type(exc).__name__}: {lines[0]}" if lines else type(exc).__name__
|
||||
return text[:200]
|
||||
failure_reason = _runner.failure_reason
|
||||
|
||||
|
||||
# -------------------------------------------------------------- the plugin
|
||||
@@ -745,9 +313,8 @@ class ATSSource(Source):
|
||||
name = "ats"
|
||||
description = f"Company job boards: {', '.join(HOSTS.values())}."
|
||||
#: The raw entries under `sources.ats.boards` are `{"url": target, "label":
|
||||
#: company}` — `url` carries the `host:slug` target and `label` the
|
||||
#: company name, which is how `Source.targets()`'s generic dict reading
|
||||
#: hands both back without a plugin-specific override.
|
||||
#: company, "params": {...}}`: `url` carries the `host:slug` target,
|
||||
#: `label` the company name and `params` the board's filter.
|
||||
subscription_field = "boards"
|
||||
|
||||
spec = PluginSpec(
|
||||
@@ -763,99 +330,74 @@ class ATSSource(Source):
|
||||
)},
|
||||
)
|
||||
|
||||
def _id_set(self, key: str) -> set[str]:
|
||||
"""A JSON list of ids kept in state, as a set; empty when there is
|
||||
none or it does not parse."""
|
||||
try:
|
||||
return {str(i) for i in json.loads(self.state.get(key) or "[]")}
|
||||
except (ValueError, TypeError):
|
||||
return set()
|
||||
|
||||
def fetch(self) -> Iterator[Item]:
|
||||
with httpx.Client() as client:
|
||||
# Handed in by the batch that runs this plugin, as the marker of
|
||||
# the pull run that queued it (see `app/jobs/pull.py`), and never
|
||||
# read from state, where a later run may have replaced it. A
|
||||
# plugin run on its own — a test, a dry run — has no run to stamp
|
||||
# itself against, so it makes one up.
|
||||
# the pull run that queued it (see `app/jobs/pull.py`). A plugin
|
||||
# run on its own (a test, a dry run) makes one up.
|
||||
run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat(
|
||||
timespec="seconds")
|
||||
|
||||
for target, label in self.targets():
|
||||
host, _, slug = target.partition(":")
|
||||
failing = f"ats:failing:{target}"
|
||||
try:
|
||||
_check_robots(client, target, self.state)
|
||||
rows, complete = _fetch_rows(client, target)
|
||||
except Exception as exc:
|
||||
log.warning("ats: %s failed (%s)", target, exc)
|
||||
# The first failure in a row sets `since`, later ones only
|
||||
# the reason; `app.jobs.pull` closes the board's roles
|
||||
# once `since` is `STALE_DAYS` old.
|
||||
try:
|
||||
since = json.loads(self.state.get(failing, "") or "{}").get("since")
|
||||
except ValueError:
|
||||
since = None
|
||||
self.state[failing] = json.dumps({
|
||||
"since": since or datetime.now(UTC).isoformat(timespec="seconds"),
|
||||
"reason": failure_reason(exc)})
|
||||
continue
|
||||
self.state.pop(failing, None)
|
||||
yield from self._board(client, target, label, run_marker)
|
||||
|
||||
detail_keys = (f"ats:detailed:{target}", f"ats:detailfailed:{target}")
|
||||
detail_sets: tuple[set[str], set[str]] | None = None
|
||||
if hasattr(readers.HOSTS.get(host), "detail"):
|
||||
detailed, failed = (self._id_set(key) for key in detail_keys)
|
||||
rows, detailed, failed = _read_details(client, target, rows, complete,
|
||||
detailed, failed)
|
||||
detail_sets = (detailed, failed)
|
||||
def _board(self, client: httpx.Client, target: str, label: str,
|
||||
run_marker: str) -> Iterator[Item]:
|
||||
host, _, slug = target.partition(":")
|
||||
failing = f"ats:failing:{target}"
|
||||
try:
|
||||
if not is_known_target(target):
|
||||
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
||||
module = readers.HOSTS[host]
|
||||
get = _getter(client, host, slug)
|
||||
ask = _runner.ask_for(module, self.params_for(target),
|
||||
keys=self.options.get("keys"))
|
||||
listing = _runner.read(self.state, "ats", target, module, slug, get, ask,
|
||||
limits=_limits())
|
||||
except Exception as exc:
|
||||
log.warning("ats: %s failed (%s)", target, exc)
|
||||
_runner.record_failure(self.state, failing, exc)
|
||||
return
|
||||
self.state.pop(failing, None)
|
||||
|
||||
seen_ids = []
|
||||
for row in rows:
|
||||
# The compound id, matching `roles.source_id` — what
|
||||
# `Store.mark_closed` compares against, not the board's own
|
||||
# bare job id.
|
||||
compound_id = f"{host}:{slug}:{row['source_id']}"
|
||||
seen_ids.append(compound_id)
|
||||
yield Item(
|
||||
source_id=compound_id,
|
||||
location=row["url"],
|
||||
title=row["title"],
|
||||
excerpt=(row["description"] or "")[:300],
|
||||
body=row["description"],
|
||||
published_at=row["posted"],
|
||||
subscription=label,
|
||||
kind="role",
|
||||
raw={
|
||||
"company": label,
|
||||
"place": row["location"],
|
||||
"department": row["department"],
|
||||
"function": classify(row["department"], row["title"]),
|
||||
"remote": row["remote"],
|
||||
# Every parser sets salary_text, empty or not (see
|
||||
# "salary_text" in `readers/_shared.py`); only
|
||||
# the hosts that state structured pay also set the
|
||||
# normalised figures, so the rest leave them at
|
||||
# row.get's default of no figure.
|
||||
"salary_text": row.get("salary_text", ""),
|
||||
"salary_min": row.get("salary_min"),
|
||||
"salary_max": row.get("salary_max"),
|
||||
"currency": row.get("currency", ""),
|
||||
"board": host,
|
||||
"slug": slug,
|
||||
},
|
||||
)
|
||||
# Stamped only once the board has parsed without raising, so a
|
||||
# board that blew up half way through is neither marked ok nor
|
||||
# has a partial id list treated as the whole of it. `complete`
|
||||
# says whether the ids are the whole board, which the pull
|
||||
# needs before it closes what they leave out. The ids read in
|
||||
# detail are kept here too, after every row went out, so a
|
||||
# pull stopped half way reads the rest again rather than
|
||||
# losing their descriptions.
|
||||
self.state[f"ats:ok:{target}"] = run_marker
|
||||
self.state[f"ats:complete:{target}"] = "1" if complete else ""
|
||||
self.state[f"ats:ids:{target}"] = json.dumps(seen_ids)
|
||||
if detail_sets is not None:
|
||||
for key, ids in zip(detail_keys, detail_sets, strict=True):
|
||||
self.state[key] = json.dumps(sorted(ids))
|
||||
def compound(source_id: str) -> str:
|
||||
# Matches `roles.source_id`, which `Store.mark_closed` compares.
|
||||
return f"{host}:{slug}:{source_id}"
|
||||
|
||||
rows = listing.rows
|
||||
detail_keys = (f"ats:detailed:{target}", f"ats:detailfailed:{target}")
|
||||
detail_sets: tuple[set[str], set[str]] | None = None
|
||||
if hasattr(module, "detail"):
|
||||
detailed, failed = (_runner.id_set(self.state, key) for key in detail_keys)
|
||||
listed = {compound(i) for i in listing.ids} if listing.complete else None
|
||||
rows, detailed, failed = _runner.read_details(
|
||||
module, host, slug, get, rows, detailed, failed, listed=listed,
|
||||
limits=_limits())
|
||||
detail_sets = (detailed, failed)
|
||||
|
||||
for row in rows:
|
||||
description = row.get("description") or ""
|
||||
yield Item(
|
||||
source_id=compound(row["source_id"]),
|
||||
location=row["url"],
|
||||
title=row["title"],
|
||||
excerpt=description[:300],
|
||||
body=description,
|
||||
published_at=row.get("posted"),
|
||||
subscription=label,
|
||||
kind="role",
|
||||
raw=_runner.raw_of(row) | {
|
||||
"company": label,
|
||||
"function": classify(row.get("department") or "", row["title"]),
|
||||
"board": host,
|
||||
"slug": slug,
|
||||
},
|
||||
)
|
||||
# Stamped only once every row went out, so a board that blew up half
|
||||
# way through is neither marked ok nor has a partial id list treated
|
||||
# as the whole of it, and a pull stopped half way reads the rest of
|
||||
# its details again.
|
||||
_runner.stamp(self.state, "ats", target, run_marker, listing,
|
||||
[compound(i) for i in listing.ids])
|
||||
if detail_sets is not None:
|
||||
for key, ids in zip(detail_keys, detail_sets, strict=True):
|
||||
self.state[key] = json.dumps(sorted(ids))
|
||||
Reference in new issue
Block a user