674 lines
33 KiB
Python
674 lines
33 KiB
Python
"""ATS boards: a company's own careers page on one of the hosts in
|
|
`plugins/atshosts/`, one module per host.
|
|
|
|
Every one of these hosts serves a company's own list of roles as public,
|
|
unauthenticated JSON, XML or RSS — first-party, no scraping, and a role that
|
|
drops off the board is *closed*, which is the cheapest staleness check there
|
|
is. What a host says and how its feed reads is its module's; this file holds
|
|
what every host shares: the fetch and its allow-list, paging and
|
|
completeness, the per-role detail reads, robots.txt, the classifier, and the
|
|
plugin itself.
|
|
|
|
The allow-list and the classifier are ported from `openings.go` and
|
|
`cmd/seedgen/main.go` in
|
|
[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT), including
|
|
`classify`'s nine functions and their keywords. The body cap is not a
|
|
straight port — see `MAX_BYTES` below.
|
|
|
|
A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
|
|
`lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what
|
|
its slug is. Every board is fetched independently; one that fails or times
|
|
out is logged, recorded as failing under `ats:failing:<target>` (see
|
|
`ATSSource.fetch`), and skipped, and the rest continue.
|
|
|
|
A host that pages states a total or does not. A board with a total counts as
|
|
complete only when it read as many distinct roles as the total says, one
|
|
without when it read up to an empty page (see `_paged`); a host that answers
|
|
with the whole board at once is complete whenever it parses.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import re
|
|
import time
|
|
from collections.abc import Callable, Iterator, MutableMapping
|
|
from datetime import UTC, datetime, timedelta
|
|
from typing import Any
|
|
from urllib.robotparser import RobotFileParser
|
|
|
|
import httpx
|
|
|
|
from desk.sources import (
|
|
API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source,
|
|
SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT,
|
|
)
|
|
|
|
from plugins import atshosts
|
|
from plugins.atshosts._shared import Get, Row
|
|
|
|
# The names the parsers had here before they moved to their host modules,
|
|
# kept for the callers and the tests that still reach for them on `ats`.
|
|
from plugins.atshosts._shared import ( # noqa: F401
|
|
Page, is_web_url as _is_web_url, parse_dt as _parse_dt, yearly_amount as _yearly_amount,
|
|
)
|
|
from plugins.atshosts.greenhouse import _salary as _greenhouse_salary # noqa: F401
|
|
from plugins.atshosts.teamtailor import parse as _parse_teamtailor # noqa: F401
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
#: openings.go's 8 MB body cap is not kept: it was written against arbitrary URLs, and every
|
|
#: URL fetched here is a feed on a fixed allow-list of ATS hosts, Greenhouse's
|
|
#: with `?content=true`. The real run of 2026-09-23 hit it on the biggest employers —
|
|
#: Anduril at 41 MB, Anthropic, OpenAI, Databricks, Carvana, Axon and
|
|
#: Airwallex between 9 and 19 MB — and skipped their boards outright. Raised
|
|
#: to 96 MB so those boards fit; a board that still exceeds it gets one retry
|
|
#: at its host's `fallback_url` where it has one (Greenhouse, without
|
|
#: `?content=true`, see `_fetch_rows`), so at least the titles land.
|
|
MAX_BYTES = 96 << 20
|
|
|
|
#: openings.go's 10 s did not hold: `greenhouse:mongodb` and `greenhouse:feedzai`
|
|
#: failed a pull of 2026-09-28 with a read timeout. Measured live on
|
|
#: 2026-09-29, three runs each of mongodb, feedzai and the five Greenhouse
|
|
#: boards with the most open roles (databricks, stripe, anthropic, feverup,
|
|
#: datadog), with and without `pay_transparency=true`, cached and with a
|
|
#: cache-busting parameter (84 requests; CloudFront answered every one
|
|
#: "Miss", so all reached the origin): the body arrives in under 1 s once it
|
|
#: starts (at most 0.09 s between two chunks, 10.4 MB the largest), but the
|
|
#: first byte sometimes waits: 16.6 s (feverup, no flag), 14.4 s (anthropic,
|
|
#: flag), 10.9 s (feverup, flag), 5.2 s and 3.6 s, and under 2.6 s otherwise.
|
|
#: The flag made no difference that three runs could tell apart from that
|
|
#: noise. So a read may wait 30 s, about twice the worst first byte seen,
|
|
#: and a whole request, redirects and body included, 60 s (see `_get`),
|
|
#: about four times the slowest seen, which still leaves a 96 MB body room
|
|
#: on a link far slower than the one measured.
|
|
TIMEOUTS = httpx.Timeout(10.0, read=30.0)
|
|
DEADLINE = 60.0
|
|
|
|
#: host -> the name the host gives itself: the hosts this plugin reads, in
|
|
#: the order a select offers them. What the Sources page's host select is
|
|
#: rendered from, so it cannot offer a host `is_known_target` refuses.
|
|
HOSTS: dict[str, str] = {name: module.LABEL for name, module in atshosts.HOSTS.items()}
|
|
|
|
|
|
def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
|
|
"""The hosts a request or a redirect may land on for one board: what its
|
|
host module's `allowed` says, which is one fixed API host or the
|
|
company's own site. Anything else is refused, at the first hop and at
|
|
every redirect: a feed is external content, and a board row must never
|
|
be able to point this fetch at an internal address."""
|
|
module = atshosts.HOSTS.get(host)
|
|
return module.allowed(slug) if module is not None else lambda h: False
|
|
|
|
|
|
class _Refused(Exception):
|
|
"""A board answered somewhere its allow-list does not cover."""
|
|
|
|
|
|
class _TooLarge(ValueError):
|
|
"""A response exceeded `MAX_BYTES`. A `ValueError` so a caller that only
|
|
knows the old shape still catches it; `_fetch_rows` catches it by type to
|
|
retry at the host's `fallback_url`."""
|
|
|
|
|
|
class _Disallowed(Exception):
|
|
"""The site's robots.txt disallows the page a board is read from (see
|
|
`_check_robots`)."""
|
|
|
|
|
|
def _get(client: httpx.Client, url: str, host: str, slug: str, *,
|
|
method: str = "GET", payload: dict[str, Any] | None = None,
|
|
headers: dict[str, str] | None = None) -> bytes:
|
|
"""Fetch one URL for a board: https only, the host's own allow-list, at
|
|
most three redirect hops each checked the same way, and the body cut off
|
|
at `MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST
|
|
sends `payload` as JSON, `{}` when there is none. `headers` are sent on
|
|
top of the User-Agent and Accept every request carries, and replace
|
|
either where they name it.
|
|
|
|
The body is streamed and counted as it arrives, so an oversized one is
|
|
refused at `MAX_BYTES` rather than after it has been read whole, and
|
|
the whole request, redirects included, must end within `DEADLINE`: the
|
|
per-read `TIMEOUTS` alone let a board that trickles bytes run on for
|
|
good. The deadline is checked between chunks, so it can be overrun by
|
|
at most one read timeout."""
|
|
allow = _allowed_host(host, slug)
|
|
sent = {"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"} | (headers or {})
|
|
deadline = time.monotonic() + DEADLINE
|
|
current = url
|
|
for _hop in range(4):
|
|
parsed = httpx.URL(current)
|
|
if parsed.scheme != "https" or not allow(parsed.host):
|
|
raise _Refused(f"{current} is not an allowed host for {host}")
|
|
with client.stream(
|
|
method, current, timeout=TIMEOUTS, follow_redirects=False, headers=sent,
|
|
json=(payload or {}) if method == "POST" else None,
|
|
) as response:
|
|
if response.is_redirect:
|
|
location = response.headers.get("location") or ""
|
|
if not location:
|
|
raise ValueError("redirect with no location")
|
|
current = str(httpx.URL(current).join(location))
|
|
continue
|
|
response.raise_for_status()
|
|
chunks: list[bytes] = []
|
|
size = 0
|
|
for chunk in response.iter_bytes():
|
|
size += len(chunk)
|
|
if size > MAX_BYTES:
|
|
raise _TooLarge(f"response too large (over {MAX_BYTES} bytes)")
|
|
if time.monotonic() > deadline:
|
|
raise httpx.ReadTimeout(f"no whole answer within {DEADLINE:.0f} s",
|
|
request=response.request)
|
|
chunks.append(chunk)
|
|
return b"".join(chunks)
|
|
raise ValueError("too many redirects")
|
|
|
|
|
|
def _getter(client: httpx.Client, host: str, slug: str) -> Get:
|
|
"""The `get` a host module is handed: `_get` bound to one board. `_get`
|
|
is looked up when a request is made, not when this is built, so a test
|
|
that replaces `ats._get` reaches every host; `payload` and `headers` are
|
|
passed on only when given, which keeps a replacement that predates them
|
|
working."""
|
|
def get(url: str, *, method: str = "GET", payload: dict[str, Any] | None = None,
|
|
headers: dict[str, str] | None = None) -> bytes:
|
|
extra: dict[str, Any] = {}
|
|
if payload is not None:
|
|
extra["payload"] = payload
|
|
if headers:
|
|
extra["headers"] = headers
|
|
return _get(client, url, host, slug, method=method, **extra)
|
|
return get
|
|
|
|
|
|
#: host -> parser, for the hosts whose module parses a whole body. Every one
|
|
#: takes the raw response body and the slug.
|
|
_PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = {
|
|
name: module.parse for name, module in atshosts.HOSTS.items() if hasattr(module, "parse")}
|
|
|
|
|
|
# --------------------------------------------------------- the classifier
|
|
|
|
#: The fixed set of functions, and the keyword rules that map a title
|
|
#: (preferred) or a department to one of them. Ported from `cmd/seedgen/
|
|
#: main.go`'s `rules` — first match wins, and "product" is genuinely listed
|
|
#: twice: the first, narrower entry is checked before
|
|
#: recruiting/design/marketing/support/sales/data, and the second, which also
|
|
#: matches the bare word "product", is checked only after all of those, so a
|
|
#: title like "Data Product Manager" keeps classifying as data/product by the
|
|
#: same rule order the Go tool used.
|
|
#:
|
|
#: Not a straight port any more: the Go tool matched substrings, which let
|
|
#: " it" (meant to catch "IT" as a department) match inside "Italy" and
|
|
#: "Italia" — a European department like "Commercial Italy" filed as
|
|
#: engineering. `_classify_text` now matches on word boundaries. A handful of
|
|
#: short tokens (`_WHOLE_WORD`) need a boundary on *both* sides, because they
|
|
#: are themselves the start of an unrelated word ("it" of "Italy", "seo" of
|
|
#: "Seoul", "qa" of "Qatar"); every other keyword keeps a leading boundary
|
|
#: only; most of them are deliberate stems ("manag", "engineer", "recruit")
|
|
#: meant to also catch "management", "engineering", "recruiting", and a
|
|
#: trailing boundary on those would stop them matching the plural and
|
|
#: gerund forms real postings use ("Software Developers", "Account
|
|
#: Managers").
|
|
FUNCTIONS = ("engineering", "data", "product", "design", "sales", "marketing",
|
|
"customer success/support", "recruiting/people", "finance/legal/ops")
|
|
|
|
_RULES: tuple[tuple[str, tuple[str, ...]], ...] = (
|
|
("product", ("product manag", "product management", "product operations",
|
|
"program manag", "technical program")),
|
|
("recruiting/people", ("recruit", "talent", "people ops", "people &",
|
|
"people team", "people operations", "human resources",
|
|
"hr", "hrbp")),
|
|
("design", ("design", "ux", "user experience", "creative")),
|
|
("marketing", ("marketing", "growth", "content", "communications", "brand",
|
|
"seo", "social media", "demand gen", "events")),
|
|
("customer success/support", ("customer success", "customer support",
|
|
"customer experience", "customer service",
|
|
"support", "success", "implementation",
|
|
"onboarding", "professional services",
|
|
"solutions consult", "technical account",
|
|
"customer engineer", "deployment")),
|
|
("sales", ("sales", "account executive", "account manager", "account director",
|
|
"business development", "bdr", "sdr", "revenue", "partnership",
|
|
"go-to-market", "gtm", "solutions engineer", "solution engineer",
|
|
"pre-sales", "presales", "channel",
|
|
# "Commercial" is what French, Italian and Spanish org charts
|
|
# call the sales function ("Commercial Italy" is a sales
|
|
# department; see the docstring above), and the rest are the
|
|
# sales words of the seed's other languages that "sales"
|
|
# itself does not already cover (German, Dutch, Danish,
|
|
# Norwegian, Swedish, French) — a modest set, not a full port;
|
|
# `extract` is the rest.
|
|
"commercial", "vertrieb", "verkauf", "verkoop", "salg",
|
|
"försäljning", "vente")),
|
|
("data", ("data", "analytics", "machine learning", "ml",
|
|
"applied science", "research scientist", "statistic")),
|
|
("product", ("product manag", "product management", "program manag",
|
|
"product operations", "technical program", "product")),
|
|
("finance/legal/ops", ("finance", "financ", "accounting", "legal", "counsel",
|
|
"compliance", "operations", "ops", "administrat",
|
|
"office", "procurement", "strategy", "corporate",
|
|
"facilities", "risk", "audit", "tax", "treasury",
|
|
"payroll", "executive assistant", "chief of staff",
|
|
"workplace", "supply chain", "logistics", "fraud",
|
|
"trust & safety", "trust and safety", "policy")),
|
|
("engineering", ("engineer", "software", "developer", "infrastructure",
|
|
"platform", "devops", "sre", "site reliability", "security",
|
|
"information technology", "it", "technology",
|
|
"technical staff", "hardware", "qa", "quality assurance",
|
|
"architect", "research", "science", "scientist", "systems",
|
|
"network", "cloud", "mobile", "frontend", "backend",
|
|
"full stack", "fullstack", "web",
|
|
# The developer/engineer words of the seed's other
|
|
# languages, same reasoning as sales' above.
|
|
"entwickler", "ingenieur", "ontwikkelaar", "udvikler",
|
|
"utvikler", "utvecklare", "ingenjör", "ingeniør",
|
|
"développeur", "developpeur", "ingénieur")),
|
|
)
|
|
|
|
#: Keywords that must match a whole word, not just its start: each one is
|
|
#: itself the first letters of an unrelated word that would otherwise false-
|
|
#: match ("it" of "Italy", "seo" of "Seoul", "qa" of "Qatar"). Every keyword
|
|
#: not listed here keeps the old prefix behaviour on purpose — see the
|
|
#: `_RULES` docstring above.
|
|
_WHOLE_WORD = frozenset({
|
|
"it", "hr", "hrbp", "ux", "ml", "qa", "seo", "web", "tax", "ops",
|
|
"sre", "bdr", "sdr", "gtm",
|
|
})
|
|
|
|
|
|
def _keyword_pattern(keyword: str) -> re.Pattern[str]:
|
|
trailing = r"\b" if keyword in _WHOLE_WORD else ""
|
|
return re.compile(r"\b" + re.escape(keyword) + trailing, re.IGNORECASE)
|
|
|
|
|
|
#: `_RULES`, compiled once: `(function, patterns)` in the same order.
|
|
_COMPILED_RULES: tuple[tuple[str, tuple[re.Pattern[str], ...]], ...] = tuple(
|
|
(function, tuple(_keyword_pattern(keyword) for keyword in keywords))
|
|
for function, keywords in _RULES)
|
|
|
|
|
|
def _classify_text(text: str) -> str:
|
|
stripped = (text or "").strip()
|
|
if not stripped:
|
|
return ""
|
|
for function, patterns in _COMPILED_RULES:
|
|
if any(pattern.search(stripped) for pattern in patterns):
|
|
return function
|
|
return ""
|
|
|
|
|
|
def classify(department: str, title: str) -> str:
|
|
"""A title (preferred) or a department, to one of `FUNCTIONS`, or `''`
|
|
when neither names one — `''` and not `"other"`, which would claim a
|
|
knowledge the classifier does not have (`extract` may still fill it from
|
|
reading the posting; see `Store.set_extracted`).
|
|
|
|
Title before department: a department is often a generic umbrella
|
|
("Operations", "Engineering & Product") that says less about the role
|
|
than its title does. A hand-checked sample of 60 roles where the two
|
|
orders disagreed (docs/decisions.md) found title-first right about twice
|
|
as often as department-first, mostly because a specific title beats a
|
|
broad department name, so title now wins; department is the fallback
|
|
when the title matches nothing."""
|
|
return _classify_text(title or "") or _classify_text(department or "") or ""
|
|
|
|
|
|
def is_known_target(target: str) -> bool:
|
|
"""`host:slug`, with `host` one of `atshosts.HOSTS` and a slug its module's
|
|
`slug_ok` accepts. What the target validator refuses: `app companies add`
|
|
and the pull job both call this before touching the network."""
|
|
host, sep, slug = target.partition(":")
|
|
module = atshosts.HOSTS.get(host)
|
|
return bool(sep) and module is not None and bool(slug.strip()) and module.slug_ok(slug)
|
|
|
|
|
|
def careers_url(target: str) -> str:
|
|
"""The company's own careers page for `host:slug`, or "" for an unknown
|
|
host. What `app companies add` stores alongside the board."""
|
|
host, _, slug = target.partition(":")
|
|
module = atshosts.HOSTS.get(host)
|
|
return module.careers_url(slug) if module is not None else ""
|
|
|
|
|
|
#: Pages read from one paged board at most, and the pause between two of
|
|
#: them. Workable answers ten roles a page, SmartRecruiters a hundred and
|
|
#: Teamtailor fifty, so the cap is 1,000, 10,000 and 5,000 roles; the pause
|
|
#: keeps Workable, which answers a burst with 429, from seeing one. The same
|
|
#: pause separates two detail reads.
|
|
MAX_PAGES = 100
|
|
PAGE_DELAY = 0.5
|
|
|
|
|
|
def _paged(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
|
|
"""Every page of a board whose host pages, `PAGE_DELAY` apart, until the
|
|
host has no next page, the stated total is reached, or `MAX_PAGES`.
|
|
Returns the rows and whether the board is complete: as many distinct
|
|
results read as the host's stated total, or, for a host that states none
|
|
(Teamtailor), the read ended on an empty page (a page with nothing new in
|
|
it ends the read short of that, incomplete, so an offset the host ignores
|
|
cannot loop). Counted on the raw results, so a row the parser drops does
|
|
not make a board incomplete, and on distinct ids, so a page that shifted
|
|
under an offset and repeated a role does not make it complete.
|
|
|
|
A 429 after the first page stops the read with what it has, incomplete;
|
|
one on the first page raises like any other failure."""
|
|
host, _, slug = target.partition(":")
|
|
module = atshosts.HOSTS[host]
|
|
get = _getter(client, host, slug)
|
|
rows: dict[str, Row] = {}
|
|
seen: set[str] = set()
|
|
total: Any = None
|
|
cursor: Any = None
|
|
for page in range(MAX_PAGES):
|
|
if page:
|
|
time.sleep(PAGE_DELAY)
|
|
try:
|
|
data, results, total, cursor = module.page(get, slug, cursor)
|
|
except httpx.HTTPStatusError as exc:
|
|
if not page or exc.response.status_code != 429:
|
|
raise
|
|
log.warning("ats: %s: 429 on page %d; kept %d of %s, not complete",
|
|
target, page + 1, len(seen), total)
|
|
return list(rows.values()), False
|
|
before = len(seen)
|
|
seen |= {str(r.get("id") or r.get("shortcode") or "") for r in results}
|
|
for row in module.rows(data, slug):
|
|
rows.setdefault(row["source_id"], row)
|
|
if (cursor is None or (isinstance(total, int) and len(seen) >= total)
|
|
or (total is None and len(seen) == before)):
|
|
break
|
|
complete = len(seen) == total if isinstance(total, int) else total is None and cursor is None
|
|
if not complete:
|
|
log.warning("ats: %s: read %d of %s stated; not complete", target, len(seen),
|
|
"an unstated total" if total is None else total)
|
|
return list(rows.values()), complete
|
|
|
|
|
|
def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
|
|
"""One board's listing, fetched and parsed: its rows and whether they are
|
|
the whole board. Raises on any failure — refused host, timeout, a feed
|
|
shape the host's parser does not understand. A host that does not page
|
|
answers with the whole board, so it is complete whenever it parses; a
|
|
paged one is complete when `_paged` says so.
|
|
|
|
A body that exceeds `MAX_BYTES` gets one retry at the host's
|
|
`fallback_url` where it has one (Greenhouse, without `?content=true`):
|
|
the description is lost, but the titles land rather than the whole board
|
|
being skipped."""
|
|
if not is_known_target(target):
|
|
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
|
host, _, slug = target.partition(":")
|
|
module = atshosts.HOSTS[host]
|
|
if hasattr(module, "page"):
|
|
return _paged(client, target)
|
|
get = _getter(client, host, slug)
|
|
try:
|
|
body = get(module.feed_url(slug))
|
|
except _TooLarge:
|
|
if not hasattr(module, "fallback_url"):
|
|
raise
|
|
log.warning("ats: %s: body exceeded %d bytes; retrying at the smaller feed",
|
|
target, MAX_BYTES)
|
|
body = get(module.fallback_url(slug))
|
|
return module.parse(body, slug), True
|
|
|
|
|
|
# -------------------------------------------------------------- robots.txt
|
|
|
|
#: How long a robots.txt answer holds for one board before it is read again.
|
|
ROBOTS_TTL = timedelta(days=1)
|
|
|
|
|
|
def _robots_allows(body: bytes, page: str) -> bool:
|
|
"""Whether a robots.txt lets `page` be fetched: by its rules for every
|
|
agent (`*`), and by any it has for this app's own User-Agent token, so a
|
|
site that shuts out this app by name is honoured as well."""
|
|
parser = RobotFileParser()
|
|
parser.parse(body.decode("utf-8", "replace").splitlines())
|
|
return parser.can_fetch("*", page) and parser.can_fetch(USER_AGENT, page)
|
|
|
|
|
|
def _check_robots(client: httpx.Client, target: str,
|
|
state: MutableMapping[str, str] | None = None) -> None:
|
|
"""Raise `_Disallowed` when the site of a host with a `robots_page` has a
|
|
robots.txt that disallows that page. Read through `_get`, so from the
|
|
board's own allowed hosts only, and at most once per `ROBOTS_TTL` for a
|
|
board when `state` is given: the verdict and when it was reached are kept
|
|
under `ats:robots:<target>`. A robots.txt that is missing (404 or 410)
|
|
allows, and that verdict is kept like any other; one that cannot be read
|
|
for another reason allows too, but is not kept, so the next pull asks
|
|
again."""
|
|
host, _, slug = target.partition(":")
|
|
module = atshosts.HOSTS.get(host)
|
|
if module is None or not hasattr(module, "robots_page") or not is_known_target(target):
|
|
return
|
|
page = module.robots_page(slug)
|
|
key = f"ats:robots:{target}"
|
|
now = datetime.now(UTC)
|
|
if state is not None:
|
|
try:
|
|
kept = json.loads(state.get(key) or "{}")
|
|
fresh = now - datetime.fromisoformat(kept["checked"]) < ROBOTS_TTL
|
|
except (ValueError, KeyError, TypeError):
|
|
kept, fresh = {}, False
|
|
if fresh:
|
|
if not kept.get("allowed", True):
|
|
raise _Disallowed(f"robots.txt disallows {page}")
|
|
return
|
|
url = f"https://{httpx.URL(page).host}/robots.txt"
|
|
try:
|
|
body = _getter(client, host, slug)(url, headers={"Accept": "text/plain"})
|
|
except httpx.HTTPStatusError as exc:
|
|
if exc.response.status_code not in (404, 410):
|
|
log.warning("ats: %s: robots.txt answered %s; allowed until the next pull",
|
|
target, exc.response.status_code)
|
|
return
|
|
allowed = True
|
|
except Exception as exc:
|
|
log.warning("ats: %s: robots.txt not read (%s); allowed until the next pull",
|
|
target, exc)
|
|
return
|
|
else:
|
|
allowed = _robots_allows(body, page)
|
|
if state is not None:
|
|
state[key] = json.dumps({"checked": now.isoformat(timespec="seconds"),
|
|
"allowed": allowed})
|
|
if not allowed:
|
|
raise _Disallowed(f"robots.txt disallows {page}")
|
|
|
|
|
|
# ---------------------------------------------------------------- details
|
|
|
|
#: Roles read in detail from one board in one pull at most. The rest of a
|
|
#: large new board is read over the next pulls, 200 at a time.
|
|
DETAIL_CAP = 200
|
|
|
|
|
|
def _read_details(client: httpx.Client, target: str, rows: list[Row], complete: bool,
|
|
detailed: set[str]) -> tuple[list[Row], set[str]]:
|
|
"""The rows of a board whose host has `detail`, each role not yet in
|
|
`detailed` (compound ids) filled from its own page, at most `DETAIL_CAP`
|
|
of them and `PAGE_DELAY` apart. Returns the rows and the ids read in
|
|
detail so far, for `ats:detailed:<target>`.
|
|
|
|
Every row not read in detail in this call goes out with an empty
|
|
description: a role read on an earlier pull keeps the one it has, since
|
|
`upsert_role` never overwrites a filled column with an empty value, and a
|
|
teaser from the listing must not replace it. A detail that raises is
|
|
logged and the role left for the next pull; a 429 stops the reads for this
|
|
board until then. None of this touches completeness, which is the
|
|
listing's alone. The ids of roles no longer listed are dropped, but only
|
|
after a complete listing, so one cut short does not make the next pull
|
|
read its missing roles again."""
|
|
host, _, slug = target.partition(":")
|
|
module = atshosts.HOSTS[host]
|
|
get = _getter(client, host, slug)
|
|
done = set(detailed)
|
|
out: list[Row] = []
|
|
tried = 0
|
|
stopped = False
|
|
for row in rows:
|
|
compound = f"{host}:{slug}:{row['source_id']}"
|
|
if compound in done or stopped or tried >= DETAIL_CAP:
|
|
out.append(row | {"description": ""})
|
|
continue
|
|
time.sleep(PAGE_DELAY)
|
|
tried += 1
|
|
try:
|
|
filled = module.detail(get, slug, row)
|
|
except Exception as exc:
|
|
if isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code == 429:
|
|
log.warning("ats: %s: 429 on a detail read; the rest wait for the next pull",
|
|
target)
|
|
stopped = True
|
|
else:
|
|
log.warning("ats: %s: detail of %s failed (%s)", target, row["source_id"], exc)
|
|
out.append(row | {"description": ""})
|
|
continue
|
|
out.append(row | filled | {"source_id": row["source_id"]})
|
|
done.add(compound)
|
|
if complete:
|
|
done &= {f"{host}:{slug}:{row['source_id']}" for row in rows}
|
|
return out, done
|
|
|
|
|
|
def fetch_board(target: str) -> tuple[list[Row], bool]:
|
|
"""Fetch and parse one board directly: no `Source`, no `self.state`, just
|
|
"does this board answer, what does it list, and is that all of it". For
|
|
`app companies add|check`, which ask that question about one board at a
|
|
time outside any pull. The listing only, with no detail reads, and
|
|
robots.txt read afresh. Raises the same way `_fetch_rows` does, and
|
|
`_Disallowed` for a board robots.txt shuts out."""
|
|
with httpx.Client() as client:
|
|
_check_robots(client, target)
|
|
return _fetch_rows(client, target)
|
|
|
|
|
|
def failure_reason(exc: BaseException) -> str:
|
|
"""One short line for why a board did not answer: the status of an HTTP
|
|
error, else the exception's class and the first line of its message."""
|
|
if isinstance(exc, _Disallowed):
|
|
return "robots.txt disallows it"
|
|
if isinstance(exc, httpx.HTTPStatusError):
|
|
return f"{exc.response.status_code} {exc.response.reason_phrase}".strip()
|
|
lines = str(exc).strip().splitlines()
|
|
text = f"{type(exc).__name__}: {lines[0]}" if lines else type(exc).__name__
|
|
return text[:200]
|
|
|
|
|
|
# -------------------------------------------------------------- the plugin
|
|
|
|
class ATSSource(Source):
|
|
name = "ats"
|
|
description = f"Company job boards: {', '.join(HOSTS.values())}."
|
|
#: The raw entries under `sources.ats.boards` are `{"url": target, "label":
|
|
#: company}` — `url` carries the `host:slug` target and `label` the
|
|
#: company name, which is how `Source.targets()`'s generic dict reading
|
|
#: hands both back without a plugin-specific override.
|
|
subscription_field = "boards"
|
|
|
|
spec = PluginSpec(
|
|
api_version=API_VERSION, id=name, version="1", label="ATS boards",
|
|
targets=TargetSpec(mode="editable", kind="board", defaults=[]),
|
|
profiles={"default": Profile(
|
|
kind="role",
|
|
tasks=Tasks(
|
|
summary=SummaryPolicy(enabled=False),
|
|
tagging=TaggingPolicy(enabled=False),
|
|
),
|
|
presentation=ARTICLE_READER,
|
|
)},
|
|
)
|
|
|
|
def fetch(self) -> Iterator[Item]:
|
|
with httpx.Client() as client:
|
|
# Handed in by the batch that runs this plugin, as the marker of
|
|
# the pull run that queued it (see `app/jobs/pull.py`), and never
|
|
# read from state, where a later run may have replaced it. A
|
|
# plugin run on its own — a test, a dry run — has no run to stamp
|
|
# itself against, so it makes one up.
|
|
run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat(
|
|
timespec="seconds")
|
|
|
|
for target, label in self.targets():
|
|
host, _, slug = target.partition(":")
|
|
failing = f"ats:failing:{target}"
|
|
try:
|
|
_check_robots(client, target, self.state)
|
|
rows, complete = _fetch_rows(client, target)
|
|
except Exception as exc:
|
|
log.warning("ats: %s failed (%s)", target, exc)
|
|
# The first failure in a row sets `since`, later ones only
|
|
# the reason; `app.jobs.pull` closes the board's roles
|
|
# once `since` is `STALE_DAYS` old.
|
|
try:
|
|
since = json.loads(self.state.get(failing, "") or "{}").get("since")
|
|
except ValueError:
|
|
since = None
|
|
self.state[failing] = json.dumps({
|
|
"since": since or datetime.now(UTC).isoformat(timespec="seconds"),
|
|
"reason": failure_reason(exc)})
|
|
continue
|
|
self.state.pop(failing, None)
|
|
|
|
detailed_key = f"ats:detailed:{target}"
|
|
detailed: set[str] | None = None
|
|
if hasattr(atshosts.HOSTS.get(host), "detail"):
|
|
try:
|
|
detailed = set(json.loads(self.state.get(detailed_key) or "[]"))
|
|
except (ValueError, TypeError):
|
|
detailed = set()
|
|
rows, detailed = _read_details(client, target, rows, complete, detailed)
|
|
|
|
seen_ids = []
|
|
for row in rows:
|
|
# The compound id, matching `roles.source_id` — what
|
|
# `Store.mark_closed` compares against, not the board's own
|
|
# bare job id.
|
|
compound_id = f"{host}:{slug}:{row['source_id']}"
|
|
seen_ids.append(compound_id)
|
|
yield Item(
|
|
source_id=compound_id,
|
|
location=row["url"],
|
|
title=row["title"],
|
|
excerpt=(row["description"] or "")[:300],
|
|
body=row["description"],
|
|
published_at=row["posted"],
|
|
subscription=label,
|
|
kind="role",
|
|
raw={
|
|
"company": label,
|
|
"place": row["location"],
|
|
"department": row["department"],
|
|
"function": classify(row["department"], row["title"]),
|
|
"remote": row["remote"],
|
|
# Every parser sets salary_text, empty or not (see
|
|
# "salary_text" in `atshosts/_shared.py`); only
|
|
# the hosts that state structured pay also set the
|
|
# normalised figures, so the rest leave them at
|
|
# row.get's default of no figure.
|
|
"salary_text": row.get("salary_text", ""),
|
|
"salary_min": row.get("salary_min"),
|
|
"salary_max": row.get("salary_max"),
|
|
"currency": row.get("currency", ""),
|
|
"board": host,
|
|
"slug": slug,
|
|
},
|
|
)
|
|
# Stamped only once the board has parsed without raising, so a
|
|
# board that blew up half way through is neither marked ok nor
|
|
# has a partial id list treated as the whole of it. `complete`
|
|
# says whether the ids are the whole board, which the pull
|
|
# needs before it closes what they leave out. The ids read in
|
|
# detail are kept here too, after every row went out, so a
|
|
# pull stopped half way reads the rest again rather than
|
|
# losing their descriptions.
|
|
self.state[f"ats:ok:{target}"] = run_marker
|
|
self.state[f"ats:complete:{target}"] = "1" if complete else ""
|
|
self.state[f"ats:ids:{target}"] = json.dumps(seen_ids)
|
|
if detailed is not None:
|
|
self.state[detailed_key] = json.dumps(sorted(detailed))
|