Files
jobboard/plugins/ats.py
T

849 lines
41 KiB
Python

"""ATS boards: a company's own careers page on one of the hosts in
`plugins/atshosts/`, one module per host.
Every one of these hosts serves a company's own list of roles as public,
unauthenticated JSON, XML or RSS — first-party, no scraping, and a role that
drops off the board is *closed*, which is the cheapest staleness check there
is. What a host says and how its feed reads is its module's; this file holds
what every host shares: the fetch and its allow-list, paging and
completeness, the per-role detail reads, robots.txt, the classifier, and the
plugin itself.
The allow-list and the classifier are ported from `openings.go` and
`cmd/seedgen/main.go` in
[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT), including
`classify`'s nine functions and their keywords. The body cap is not a
straight port — see `MAX_BYTES` below.
A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
`lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what
its slug is. Every board is fetched independently; one that fails or times
out is logged, recorded as failing under `ats:failing:<target>` (see
`ATSSource.fetch`), and skipped, and the rest continue.
A host that pages states a total or does not. A board with a total counts as
complete only when it read as many distinct roles as the total says, one
without when it read up to an empty page (see `_paged`); a host that answers
with the whole board at once is complete whenever it parses.
"""
from __future__ import annotations
import ipaddress
import json
import logging
import re
import sys
import time
from collections.abc import Callable, Iterator, MutableMapping
from datetime import UTC, datetime, timedelta
from typing import Any
from pathlib import Path
from urllib.parse import unquote
import httpx
from desk.sources import (
API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source,
SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT,
)
try:
from plugins import atshosts
except ModuleNotFoundError:
# `desk.sources.registry` loads this file by its path, under a name of its
# own, and the app puts the repo root on `sys.path` before it does; a
# registry run without the app (a script, another working directory)
# does not, and the host modules are found as the package beside this
# file. So the root goes on the path then, and only then.
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from plugins import atshosts
from plugins.atshosts._shared import Get, Row
# The names the parsers had here before they moved to their host modules,
# kept for the callers and the tests that still reach for them on `ats`.
from plugins.atshosts._shared import ( # noqa: F401
Page, is_web_url as _is_web_url, parse_dt as _parse_dt, yearly_amount as _yearly_amount,
)
from plugins.atshosts.greenhouse import _salary as _greenhouse_salary # noqa: F401
from plugins.atshosts.teamtailor import parse as _parse_teamtailor # noqa: F401
log = logging.getLogger(__name__)
#: openings.go's 8 MB body cap is not kept: it was written against arbitrary URLs, and every
#: URL fetched here is a feed on a fixed allow-list of ATS hosts, Greenhouse's
#: with `?content=true`. The real run of 2026-09-23 hit it on the biggest employers —
#: Anduril at 41 MB, Anthropic, OpenAI, Databricks, Carvana, Axon and
#: Airwallex between 9 and 19 MB — and skipped their boards outright. Raised
#: to 96 MB so those boards fit; a board that still exceeds it gets one retry
#: at its host's `fallback_url` where it has one (Greenhouse, without
#: `?content=true`, see `_fetch_rows`), so at least the titles land.
MAX_BYTES = 96 << 20
#: openings.go's 10 s did not hold: `greenhouse:mongodb` and `greenhouse:feedzai`
#: failed a pull of 2026-09-28 with a read timeout. Measured live on
#: 2026-09-29, three runs each of mongodb, feedzai and the five Greenhouse
#: boards with the most open roles (databricks, stripe, anthropic, feverup,
#: datadog), with and without `pay_transparency=true`, cached and with a
#: cache-busting parameter (84 requests; CloudFront answered every one
#: "Miss", so all reached the origin): the body arrives in under 1 s once it
#: starts (at most 0.09 s between two chunks, 10.4 MB the largest), but the
#: first byte sometimes waits: 16.6 s (feverup, no flag), 14.4 s (anthropic,
#: flag), 10.9 s (feverup, flag), 5.2 s and 3.6 s, and under 2.6 s otherwise.
#: The flag made no difference that three runs could tell apart from that
#: noise. So a read may wait 30 s, about twice the worst first byte seen,
#: and a whole request, redirects and body included, 60 s (see `_get`),
#: about four times the slowest seen, which still leaves a 96 MB body room
#: on a link far slower than the one measured.
TIMEOUTS = httpx.Timeout(10.0, read=30.0)
DEADLINE = 60.0
#: host -> the name the host gives itself: the hosts this plugin reads, in
#: the order a select offers them. What the Sources page's host select is
#: rendered from, so it cannot offer a host `is_known_target` refuses.
HOSTS: dict[str, str] = {name: module.LABEL for name, module in atshosts.HOSTS.items()}
def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
"""The hosts a request or a redirect may land on for one board: what its
host module's `allowed` says, which is one fixed API host or the
company's own site. Anything else is refused, at the first hop and at
every redirect: a feed is external content, and a board row must never
be able to point this fetch at an internal address."""
module = atshosts.HOSTS.get(host)
return module.allowed(slug) if module is not None else lambda h: False
class _Refused(Exception):
"""A board answered somewhere its allow-list does not cover."""
class _TooLarge(ValueError):
"""A response exceeded `MAX_BYTES`. A `ValueError` so a caller that only
knows the old shape still catches it; `_fetch_rows` catches it by type to
retry at the host's `fallback_url`."""
class _Disallowed(Exception):
"""The site's robots.txt disallows the page a board is read from (see
`_check_robots`)."""
class _RobotsUnreachable(Exception):
"""The site's robots.txt could not be read: a 5xx, a 429, a timeout, a
connection error or a redirect off the site. RFC 9309 says to assume
the site disallows everything then, so the board is not read this pull."""
#: Host name suffixes that name a machine on a private network, never a
#: company's careers site.
_LOCAL_SUFFIXES = (".localhost", ".local", ".internal", ".lan", ".home.arpa")
def _local_host(name: str) -> bool:
"""Whether a host name is an address or a local name rather than a public
DNS name: an IP literal (v4 or v6), anything whose last label is a number
(`2130706433`, `127.1` and `0x7f000001` all reach 127.0.0.1 through the
resolver), `localhost`, or a private suffix. A Teamtailor slug can be any
host name, so the allow-list alone would let one point the fetch here."""
name = name.strip("[]").rstrip(".").lower()
try:
ipaddress.ip_address(name)
return True
except ValueError:
pass
last = name.rsplit(".", 1)[-1]
return (last.isdigit() or bool(re.fullmatch(r"0x[0-9a-f]+", last))
or name == "localhost" or name.endswith(_LOCAL_SUFFIXES))
def _get(client: httpx.Client, url: str, host: str, slug: str, *,
method: str = "GET", payload: dict[str, Any] | None = None,
headers: dict[str, str] | None = None, www_twin: bool = False) -> bytes:
"""Fetch one URL for a board: https only on the default port, never an
address or a local name (`_local_host`), the host's own allow-list, at
most three redirect hops each checked the same way, and the body cut off
at `MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST
sends `payload` as JSON, `{}` when there is none. `headers` are sent on
top of the User-Agent and Accept every request carries, and replace
either where they name it. With `www_twin`, a redirect may also land on
the first URL's host with `www.` added or taken off, which only the
robots.txt read asks for (see `_check_robots`).
The body is streamed and counted as it arrives, so an oversized one is
refused at `MAX_BYTES` rather than after it has been read whole, and
the whole request, redirects included, must end within `DEADLINE`: the
per-read `TIMEOUTS` alone let a board that trickles bytes run on for
good. The deadline is checked between chunks, so it can be overrun by
at most one read timeout."""
allow = _allowed_host(host, slug)
if www_twin:
bare = httpx.URL(url).host.removeprefix("www.")
board_allow = allow
allow = lambda h: board_allow(h) or h.removeprefix("www.") == bare # noqa: E731
sent = {"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"} | (headers or {})
deadline = time.monotonic() + DEADLINE
current = url
for _hop in range(4):
parsed = httpx.URL(current)
if (parsed.scheme != "https" or parsed.port not in (None, 443)
or _local_host(parsed.host) or not allow(parsed.host)):
raise _Refused(f"{current} is not an allowed host for {host}")
with client.stream(
method, current, timeout=TIMEOUTS, follow_redirects=False, headers=sent,
json=(payload or {}) if method == "POST" else None,
) as response:
if response.is_redirect:
location = response.headers.get("location") or ""
if not location:
raise ValueError("redirect with no location")
current = str(httpx.URL(current).join(location))
continue
response.raise_for_status()
chunks: list[bytes] = []
size = 0
for chunk in response.iter_bytes():
size += len(chunk)
if size > MAX_BYTES:
raise _TooLarge(f"response too large (over {MAX_BYTES} bytes)")
if time.monotonic() > deadline:
raise httpx.ReadTimeout(f"no whole answer within {DEADLINE:.0f} s",
request=response.request)
chunks.append(chunk)
return b"".join(chunks)
raise ValueError("too many redirects")
def _getter(client: httpx.Client, host: str, slug: str) -> Get:
"""The `get` a host module is handed: `_get` bound to one board. `_get`
is looked up when a request is made, not when this is built, so a test
that replaces `ats._get` reaches every host; `payload` and `headers` are
passed on only when given, which keeps a replacement that predates them
working."""
def get(url: str, *, method: str = "GET", payload: dict[str, Any] | None = None,
headers: dict[str, str] | None = None) -> bytes:
extra: dict[str, Any] = {}
if payload is not None:
extra["payload"] = payload
if headers:
extra["headers"] = headers
return _get(client, url, host, slug, method=method, **extra)
return get
#: host -> parser, for the hosts whose module parses a whole body. Every one
#: takes the raw response body and the slug.
_PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = {
name: module.parse for name, module in atshosts.HOSTS.items() if hasattr(module, "parse")}
# --------------------------------------------------------- the classifier
#: The fixed set of functions, and the keyword rules that map a title
#: (preferred) or a department to one of them. Ported from `cmd/seedgen/
#: main.go`'s `rules` — first match wins, and "product" is genuinely listed
#: twice: the first, narrower entry is checked before
#: recruiting/design/marketing/support/sales/data, and the second, which also
#: matches the bare word "product", is checked only after all of those, so a
#: title like "Data Product Manager" keeps classifying as data/product by the
#: same rule order the Go tool used.
#:
#: Not a straight port any more: the Go tool matched substrings, which let
#: " it" (meant to catch "IT" as a department) match inside "Italy" and
#: "Italia" — a European department like "Commercial Italy" filed as
#: engineering. `_classify_text` now matches on word boundaries. A handful of
#: short tokens (`_WHOLE_WORD`) need a boundary on *both* sides, because they
#: are themselves the start of an unrelated word ("it" of "Italy", "seo" of
#: "Seoul", "qa" of "Qatar"); every other keyword keeps a leading boundary
#: only; most of them are deliberate stems ("manag", "engineer", "recruit")
#: meant to also catch "management", "engineering", "recruiting", and a
#: trailing boundary on those would stop them matching the plural and
#: gerund forms real postings use ("Software Developers", "Account
#: Managers").
FUNCTIONS = ("engineering", "data", "product", "design", "sales", "marketing",
"customer success/support", "recruiting/people", "finance/legal/ops")
_RULES: tuple[tuple[str, tuple[str, ...]], ...] = (
("product", ("product manag", "product management", "product operations",
"program manag", "technical program")),
("recruiting/people", ("recruit", "talent", "people ops", "people &",
"people team", "people operations", "human resources",
"hr", "hrbp")),
("design", ("design", "ux", "user experience", "creative")),
("marketing", ("marketing", "growth", "content", "communications", "brand",
"seo", "social media", "demand gen", "events")),
("customer success/support", ("customer success", "customer support",
"customer experience", "customer service",
"support", "success", "implementation",
"onboarding", "professional services",
"solutions consult", "technical account",
"customer engineer", "deployment")),
("sales", ("sales", "account executive", "account manager", "account director",
"business development", "bdr", "sdr", "revenue", "partnership",
"go-to-market", "gtm", "solutions engineer", "solution engineer",
"pre-sales", "presales", "channel",
# "Commercial" is what French, Italian and Spanish org charts
# call the sales function ("Commercial Italy" is a sales
# department; see the docstring above), and the rest are the
# sales words of the seed's other languages that "sales"
# itself does not already cover (German, Dutch, Danish,
# Norwegian, Swedish, French) — a modest set, not a full port;
# `extract` is the rest.
"commercial", "vertrieb", "verkauf", "verkoop", "salg",
"försäljning", "vente")),
("data", ("data", "analytics", "machine learning", "ml",
"applied science", "research scientist", "statistic")),
("product", ("product manag", "product management", "program manag",
"product operations", "technical program", "product")),
("finance/legal/ops", ("finance", "financ", "accounting", "legal", "counsel",
"compliance", "operations", "ops", "administrat",
"office", "procurement", "strategy", "corporate",
"facilities", "risk", "audit", "tax", "treasury",
"payroll", "executive assistant", "chief of staff",
"workplace", "supply chain", "logistics", "fraud",
"trust & safety", "trust and safety", "policy")),
("engineering", ("engineer", "software", "developer", "infrastructure",
"platform", "devops", "sre", "site reliability", "security",
"information technology", "it", "technology",
"technical staff", "hardware", "qa", "quality assurance",
"architect", "research", "science", "scientist", "systems",
"network", "cloud", "mobile", "frontend", "backend",
"full stack", "fullstack", "web",
# The developer/engineer words of the seed's other
# languages, same reasoning as sales' above.
"entwickler", "ingenieur", "ontwikkelaar", "udvikler",
"utvikler", "utvecklare", "ingenjör", "ingeniør",
"développeur", "developpeur", "ingénieur")),
)
#: Keywords that must match a whole word, not just its start: each one is
#: itself the first letters of an unrelated word that would otherwise false-
#: match ("it" of "Italy", "seo" of "Seoul", "qa" of "Qatar"). Every keyword
#: not listed here keeps the old prefix behaviour on purpose — see the
#: `_RULES` docstring above.
_WHOLE_WORD = frozenset({
"it", "hr", "hrbp", "ux", "ml", "qa", "seo", "web", "tax", "ops",
"sre", "bdr", "sdr", "gtm",
})
def _keyword_pattern(keyword: str) -> re.Pattern[str]:
trailing = r"\b" if keyword in _WHOLE_WORD else ""
return re.compile(r"\b" + re.escape(keyword) + trailing, re.IGNORECASE)
#: `_RULES`, compiled once: `(function, patterns)` in the same order.
_COMPILED_RULES: tuple[tuple[str, tuple[re.Pattern[str], ...]], ...] = tuple(
(function, tuple(_keyword_pattern(keyword) for keyword in keywords))
for function, keywords in _RULES)
def _classify_text(text: str) -> str:
stripped = (text or "").strip()
if not stripped:
return ""
for function, patterns in _COMPILED_RULES:
if any(pattern.search(stripped) for pattern in patterns):
return function
return ""
def classify(department: str, title: str) -> str:
"""A title (preferred) or a department, to one of `FUNCTIONS`, or `''`
when neither names one — `''` and not `"other"`, which would claim a
knowledge the classifier does not have (`extract` may still fill it from
reading the posting; see `Store.set_extracted`).
Title before department: a department is often a generic umbrella
("Operations", "Engineering & Product") that says less about the role
than its title does. A hand-checked sample of 60 roles where the two
orders disagreed (docs/decisions.md) found title-first right about twice
as often as department-first, mostly because a specific title beats a
broad department name, so title now wins; department is the fallback
when the title matches nothing."""
return _classify_text(title or "") or _classify_text(department or "") or ""
def is_known_target(target: str) -> bool:
"""`host:slug`, with `host` one of `atshosts.HOSTS` and a slug its module's
`slug_ok` accepts. What the target validator refuses: `app companies add`
and the pull job both call this before touching the network."""
host, sep, slug = target.partition(":")
module = atshosts.HOSTS.get(host)
return bool(sep) and module is not None and bool(slug.strip()) and module.slug_ok(slug)
def careers_url(target: str) -> str:
"""The company's own careers page for `host:slug`, or "" for an unknown
host. What `app companies add` stores alongside the board."""
host, _, slug = target.partition(":")
module = atshosts.HOSTS.get(host)
return module.careers_url(slug) if module is not None else ""
#: Pages read from one paged board at most, and the pause between two of
#: them. Workable answers ten roles a page, SmartRecruiters a hundred and
#: Teamtailor fifty, so the cap is 1,000, 10,000 and 5,000 roles; the pause
#: keeps Workable, which answers a burst with 429, from seeing one. The same
#: pause separates two detail reads.
MAX_PAGES = 100
PAGE_DELAY = 0.5
def _paged(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
"""Every page of a board whose host pages, `PAGE_DELAY` apart, until the
host has no next page, the stated total is reached, or `MAX_PAGES`.
Returns the rows and whether the board is complete: as many distinct
results read as the host's stated total, or, for a host that states none
(Teamtailor), the read ended on an empty page (a page with nothing new in
it ends the read short of that, incomplete, so an offset the host ignores
cannot loop). Counted on the raw results, so a row the parser drops does
not make a board incomplete, and on distinct ids, so a page that shifted
under an offset and repeated a role does not make it complete.
A 429 after the first page stops the read with what it has, incomplete;
one on the first page raises like any other failure."""
host, _, slug = target.partition(":")
module = atshosts.HOSTS[host]
get = _getter(client, host, slug)
rows: dict[str, Row] = {}
seen: set[str] = set()
total: Any = None
cursor: Any = None
for page in range(MAX_PAGES):
if page:
time.sleep(PAGE_DELAY)
try:
data, results, total, cursor = module.page(get, slug, cursor)
except httpx.HTTPStatusError as exc:
if not page or exc.response.status_code != 429:
raise
log.warning("ats: %s: 429 on page %d; kept %d of %s, not complete",
target, page + 1, len(seen), total)
return list(rows.values()), False
before = len(seen)
seen |= {str(r.get("id") or r.get("shortcode") or "") for r in results}
for row in module.rows(data, slug):
rows.setdefault(row["source_id"], row)
if (cursor is None or (isinstance(total, int) and len(seen) >= total)
or (total is None and len(seen) == before)):
break
complete = len(seen) == total if isinstance(total, int) else total is None and cursor is None
if not complete:
log.warning("ats: %s: read %d of %s stated; not complete", target, len(seen),
"an unstated total" if total is None else total)
return list(rows.values()), complete
def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
"""One board's listing, fetched and parsed: its rows and whether they are
the whole board. Raises on any failure — refused host, timeout, a feed
shape the host's parser does not understand. A host that does not page
answers with the whole board, so it is complete whenever it parses; a
paged one is complete when `_paged` says so.
A body that exceeds `MAX_BYTES` gets one retry at the host's
`fallback_url` where it has one (Greenhouse, without `?content=true`):
the description is lost, but the titles land rather than the whole board
being skipped."""
if not is_known_target(target):
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
host, _, slug = target.partition(":")
module = atshosts.HOSTS[host]
if hasattr(module, "page"):
return _paged(client, target)
get = _getter(client, host, slug)
try:
body = get(module.feed_url(slug))
except _TooLarge:
if not hasattr(module, "fallback_url"):
raise
log.warning("ats: %s: body exceeded %d bytes; retrying at the smaller feed",
target, MAX_BYTES)
body = get(module.fallback_url(slug))
return module.parse(body, slug), True
# -------------------------------------------------------------- robots.txt
#: How long a robots.txt answer holds for one board before it is read again.
ROBOTS_TTL = timedelta(days=1)
#: This app's product token, the name a robots.txt group addresses it by:
#: the `desk` of `desk/<version> (personal aggregator)`.
ROBOTS_TOKEN = USER_AGENT.split("/", 1)[0].strip().lower()
#: One rule of a group: `(allow, path pattern)`.
_Rule = tuple[bool, str]
def _robots_groups(text: str) -> list[tuple[set[str], list[_Rule]]]:
"""The groups of a robots.txt, per RFC 9309: one or more `user-agent`
lines (their tokens lowercased) and the `allow` and `disallow` rules that
follow them, up to the next `user-agent` line after a rule. Comments,
other keys and rules before any `user-agent` line are left out, and so is
an empty rule, which allows everything and so says nothing."""
groups: list[tuple[set[str], list[_Rule]]] = []
in_agents = False
for line in text.splitlines():
key, sep, value = line.split("#", 1)[0].partition(":")
key, value = key.strip().lower(), value.strip()
if not sep:
continue
if key == "user-agent":
if not in_agents:
groups.append((set(), []))
in_agents = True
groups[-1][0].add(value.lower())
elif key in ("allow", "disallow") and groups:
in_agents = False
if value:
groups[-1][1].append((key == "allow", value))
return groups
def _robots_rules(groups: list[tuple[set[str], list[_Rule]]], token: str) -> list[_Rule]:
"""The rules for `token`: every group that names it (matched whole and
ignoring case, as RFC 9309 says, not as a substring), merged; else every
`*` group, merged; else none, which allows everything."""
for wanted in (token, "*"):
rules = [rule for agents, group in groups if wanted in agents for rule in group]
if rules or any(wanted in agents for agents, _group in groups):
return rules
return []
def _robots_pattern(path: str) -> re.Pattern[str]:
"""A rule's path as a regex anchored at the start: `*` is any run of
characters, and a `$` at the end anchors the end."""
anchored = path.endswith("$")
body = ".*".join(re.escape(part) for part in path.rstrip("$").split("*"))
return re.compile(body + (r"\Z" if anchored else ""), re.DOTALL)
def _robots_verdict(rules: list[_Rule], path: str) -> bool:
"""Whether `path` is allowed by `rules`: the longest matching rule wins,
an allow wins a tie, and no match allows. Both sides are compared
percent-decoded, so `%7E` and `~` are one character."""
path = unquote(path)
best: tuple[int, bool] | None = None
for allow, pattern in rules:
if _robots_pattern(unquote(pattern)).match(path):
weight = (len(pattern), allow)
best = weight if best is None or weight > best else best
return best is None or best[1]
def _robots_allows(body: bytes, page: str) -> bool:
"""Whether a robots.txt lets `page` be fetched, by RFC 9309: by its rules
for every agent (`*`), and by those it has for this app's own token
(`ROBOTS_TOKEN`, which fall back to `*`), so a site that shuts out this
app by name is honoured as well. Paths are case-sensitive; a byte order
mark is dropped."""
groups = _robots_groups(body.decode("utf-8-sig", "replace"))
url = httpx.URL(page)
path = (url.raw_path or b"/").decode("ascii", "replace")
return all(_robots_verdict(_robots_rules(groups, token), path)
for token in ("*", ROBOTS_TOKEN))
def _check_robots(client: httpx.Client, target: str,
state: MutableMapping[str, str] | None = None) -> None:
"""Raise `_Disallowed` when the site of a host with a `robots_page` has a
robots.txt that disallows that page, and `_RobotsUnreachable` when it
cannot be read. Read through `_get`, so from the board's own allowed
hosts only, except that a redirect between the host and its `www.` twin
is followed (at most three hops, https only); any other redirect off the
site is unreachable. With `state`, the verdict and when it was reached
are kept under `ats:robots:<target>` and held for `ROBOTS_TTL`.
By RFC 9309: a 4xx other than 429 means there is no robots.txt to obey,
which allows, and is kept like any other verdict. A 5xx, a 429, a
timeout or a connection error means the file is unreachable, which the
RFC says to treat as disallowing everything: the board is not read on
this pull, fails with "robots.txt could not be read", and nothing is
kept, so the next pull asks again. A 429 counts as unreachable although
it is a 4xx, since it says "not now" rather than "there is none"."""
host, _, slug = target.partition(":")
module = atshosts.HOSTS.get(host)
if module is None or not hasattr(module, "robots_page") or not is_known_target(target):
return
page = module.robots_page(slug)
key = f"ats:robots:{target}"
now = datetime.now(UTC)
if state is not None:
try:
kept = json.loads(state.get(key) or "{}")
fresh = now - datetime.fromisoformat(kept["checked"]) < ROBOTS_TTL
except (ValueError, KeyError, TypeError):
kept, fresh = {}, False
if fresh:
if not kept.get("allowed", True):
raise _Disallowed(f"robots.txt disallows {page}")
return
url = f"https://{httpx.URL(page).host}/robots.txt"
try:
body = _get(client, url, host, slug, headers={"Accept": "text/plain"}, www_twin=True)
except httpx.HTTPStatusError as exc:
code = exc.response.status_code
if not 400 <= code < 500 or code == 429:
raise _RobotsUnreachable(f"robots.txt answered {code}") from exc
allowed = True
except Exception as exc:
raise _RobotsUnreachable(f"robots.txt not read: {exc}") from exc
else:
# A host that serves a path whatever its case is matched without
# case, so a rule written `/Jobs/` forbids a site named `jobs` too.
if getattr(module, "ROBOTS_ANY_CASE", False):
allowed = _robots_allows(body.lower(), page.lower())
else:
allowed = _robots_allows(body, page)
if state is not None:
state[key] = json.dumps({"checked": now.isoformat(timespec="seconds"),
"allowed": allowed})
if not allowed:
raise _Disallowed(f"robots.txt disallows {page}")
# ---------------------------------------------------------------- details
#: Roles read in detail from one board in one pull at most. The rest of a
#: large new board is read over the next pulls, 200 at a time.
DETAIL_CAP = 200
#: The row keys a host's `detail` may fill: those whose column survives a
#: later pull that sends the listing's empty value for them, so what a role's
#: own page said is not wiped the pull after it was read. `description`,
#: `location` (`place`, `place_key`, `country`), `remote` and `currency` are
#: `upsert_role`'s keep-filled columns, `salary_min` and `salary_max` are only
#: overwritten by a figure, and `posted` feeds `posted_at`, which keeps the
#: earliest date a role ever had. `title` and `url` are not here, since the
#: listing always states them and would overwrite them; nor are `department`
#: (no column of its own, only `raw` and the classifier's guess) and
#: `salary_text`, both rewritten from the listing on every pull.
DETAIL_FIELDS = ("description", "location", "remote", "currency", "salary_min",
"salary_max", "posted")
def _read_details(client: httpx.Client, target: str, rows: list[Row], complete: bool,
detailed: set[str], failed: set[str]
) -> tuple[list[Row], set[str], set[str]]:
"""The rows of a board whose host has `detail`, with the roles not yet
read in detail filled from their own pages: at most `DETAIL_CAP` of them,
`PAGE_DELAY` apart, first the roles never tried and then those whose
detail failed before (`failed`), so a few roles that always fail cannot
take the whole cap pull after pull. Returns the rows and the compound ids
read (for `ats:detailed:<target>`) and failed (for
`ats:detailfailed:<target>`).
Only `DETAIL_FIELDS` are taken from what `detail` returns; any other key
is left out with a debug line. Every row not read in detail in this call
goes out with an empty description: a role read on an earlier pull keeps
the one it has, since `upsert_role` never overwrites a filled column with
an empty value, and a teaser from the listing must not replace it.
A detail that answers 404 or 410 counts as read: the role is gone or has
no page of its own, and asking again would not change that. One that
raises otherwise is logged and the role kept in `failed`; a 429 stops the
reads for this board until the next pull, and does not mark the role as
failing. None of this touches completeness, which is the listing's alone.
The ids of roles no longer listed are dropped from both sets, but only
after a complete listing, so one cut short does not make the next pull
read its missing roles again."""
host, _, slug = target.partition(":")
module = atshosts.HOSTS[host]
get = _getter(client, host, slug)
done, failing = set(detailed), set(failed) - set(detailed)
def compound(row: Row) -> str:
return f"{host}:{slug}:{row['source_id']}"
unread = [row for row in rows if compound(row) not in done]
queue = ([row for row in unread if compound(row) not in failing]
+ [row for row in unread if compound(row) in failing])[:DETAIL_CAP]
filled: dict[str, Row] = {}
for row in queue:
key = compound(row)
time.sleep(PAGE_DELAY)
try:
answer = module.detail(get, slug, row)
except httpx.HTTPStatusError as exc:
code = exc.response.status_code
if code == 429:
log.warning("ats: %s: 429 on a detail read; the rest wait for the next pull",
target)
break
if code in (404, 410):
done.add(key)
failing.discard(key)
else:
log.warning("ats: %s: detail of %s answered %s", target, row["source_id"], code)
failing.add(key)
continue
except Exception as exc:
log.warning("ats: %s: detail of %s failed (%s)", target, row["source_id"], exc)
failing.add(key)
continue
if ignored := sorted(set(answer) - set(DETAIL_FIELDS)):
log.debug("ats: %s: detail of %s: %s left out, not in DETAIL_FIELDS",
target, row["source_id"], ", ".join(ignored))
filled[key] = {k: v for k, v in answer.items() if k in DETAIL_FIELDS}
done.add(key)
failing.discard(key)
out = [row | {"description": ""} | filled.get(compound(row), {}) for row in rows]
if complete:
listed = {compound(row) for row in rows}
done &= listed
failing &= listed
return out, done, failing
def fetch_board(target: str) -> tuple[list[Row], bool]:
"""Fetch and parse one board directly: no `Source`, no `self.state`, just
"does this board answer, what does it list, and is that all of it". For
`app companies add|check`, which ask that question about one board at a
time outside any pull. The listing only, with no detail reads, and
robots.txt read afresh. Raises the same way `_fetch_rows` does, and
`_Disallowed` or `_RobotsUnreachable` for a board robots.txt shuts out or
that cannot be checked."""
with httpx.Client() as client:
_check_robots(client, target)
return _fetch_rows(client, target)
def failure_reason(exc: BaseException) -> str:
"""One short line for why a board did not answer: the status of an HTTP
error, else the exception's class and the first line of its message."""
if isinstance(exc, _Disallowed):
return "robots.txt disallows it"
if isinstance(exc, _RobotsUnreachable):
return "robots.txt could not be read"
if isinstance(exc, httpx.HTTPStatusError):
return f"{exc.response.status_code} {exc.response.reason_phrase}".strip()
lines = str(exc).strip().splitlines()
text = f"{type(exc).__name__}: {lines[0]}" if lines else type(exc).__name__
return text[:200]
# -------------------------------------------------------------- the plugin
class ATSSource(Source):
name = "ats"
description = f"Company job boards: {', '.join(HOSTS.values())}."
#: The raw entries under `sources.ats.boards` are `{"url": target, "label":
#: company}` — `url` carries the `host:slug` target and `label` the
#: company name, which is how `Source.targets()`'s generic dict reading
#: hands both back without a plugin-specific override.
subscription_field = "boards"
spec = PluginSpec(
api_version=API_VERSION, id=name, version="1", label="ATS boards",
targets=TargetSpec(mode="editable", kind="board", defaults=[]),
profiles={"default": Profile(
kind="role",
tasks=Tasks(
summary=SummaryPolicy(enabled=False),
tagging=TaggingPolicy(enabled=False),
),
presentation=ARTICLE_READER,
)},
)
def _id_set(self, key: str) -> set[str]:
"""A JSON list of ids kept in state, as a set; empty when there is
none or it does not parse."""
try:
return {str(i) for i in json.loads(self.state.get(key) or "[]")}
except (ValueError, TypeError):
return set()
def fetch(self) -> Iterator[Item]:
with httpx.Client() as client:
# Handed in by the batch that runs this plugin, as the marker of
# the pull run that queued it (see `app/jobs/pull.py`), and never
# read from state, where a later run may have replaced it. A
# plugin run on its own — a test, a dry run — has no run to stamp
# itself against, so it makes one up.
run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat(
timespec="seconds")
for target, label in self.targets():
host, _, slug = target.partition(":")
failing = f"ats:failing:{target}"
try:
_check_robots(client, target, self.state)
rows, complete = _fetch_rows(client, target)
except Exception as exc:
log.warning("ats: %s failed (%s)", target, exc)
# The first failure in a row sets `since`, later ones only
# the reason; `app.jobs.pull` closes the board's roles
# once `since` is `STALE_DAYS` old.
try:
since = json.loads(self.state.get(failing, "") or "{}").get("since")
except ValueError:
since = None
self.state[failing] = json.dumps({
"since": since or datetime.now(UTC).isoformat(timespec="seconds"),
"reason": failure_reason(exc)})
continue
self.state.pop(failing, None)
detail_keys = (f"ats:detailed:{target}", f"ats:detailfailed:{target}")
detail_sets: tuple[set[str], set[str]] | None = None
if hasattr(atshosts.HOSTS.get(host), "detail"):
detailed, failed = (self._id_set(key) for key in detail_keys)
rows, detailed, failed = _read_details(client, target, rows, complete,
detailed, failed)
detail_sets = (detailed, failed)
seen_ids = []
for row in rows:
# The compound id, matching `roles.source_id` — what
# `Store.mark_closed` compares against, not the board's own
# bare job id.
compound_id = f"{host}:{slug}:{row['source_id']}"
seen_ids.append(compound_id)
yield Item(
source_id=compound_id,
location=row["url"],
title=row["title"],
excerpt=(row["description"] or "")[:300],
body=row["description"],
published_at=row["posted"],
subscription=label,
kind="role",
raw={
"company": label,
"place": row["location"],
"department": row["department"],
"function": classify(row["department"], row["title"]),
"remote": row["remote"],
# Every parser sets salary_text, empty or not (see
# "salary_text" in `atshosts/_shared.py`); only
# the hosts that state structured pay also set the
# normalised figures, so the rest leave them at
# row.get's default of no figure.
"salary_text": row.get("salary_text", ""),
"salary_min": row.get("salary_min"),
"salary_max": row.get("salary_max"),
"currency": row.get("currency", ""),
"board": host,
"slug": slug,
},
)
# Stamped only once the board has parsed without raising, so a
# board that blew up half way through is neither marked ok nor
# has a partial id list treated as the whole of it. `complete`
# says whether the ids are the whole board, which the pull
# needs before it closes what they leave out. The ids read in
# detail are kept here too, after every row went out, so a
# pull stopped half way reads the rest again rather than
# losing their descriptions.
self.state[f"ats:ok:{target}"] = run_marker
self.state[f"ats:complete:{target}"] = "1" if complete else ""
self.state[f"ats:ids:{target}"] = json.dumps(seen_ids)
if detail_sets is not None:
for key, ids in zip(detail_keys, detail_sets, strict=True):
self.state[key] = json.dumps(sorted(ids))