Files
jobboard/plugins/ats.py
T

674 lines
33 KiB
Python

"""ATS boards: a company's own careers page on one of the hosts in
`plugins/atshosts/`, one module per host.
Every one of these hosts serves a company's own list of roles as public,
unauthenticated JSON, XML or RSS — first-party, no scraping, and a role that
drops off the board is *closed*, which is the cheapest staleness check there
is. What a host says and how its feed reads is its module's; this file holds
what every host shares: the fetch and its allow-list, paging and
completeness, the per-role detail reads, robots.txt, the classifier, and the
plugin itself.
The allow-list and the classifier are ported from `openings.go` and
`cmd/seedgen/main.go` in
[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT), including
`classify`'s nine functions and their keywords. The body cap is not a
straight port — see `MAX_BYTES` below.
A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
`lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what
its slug is. Every board is fetched independently; one that fails or times
out is logged, recorded as failing under `ats:failing:<target>` (see
`ATSSource.fetch`), and skipped, and the rest continue.
A host that pages states a total or does not. A board with a total counts as
complete only when it read as many distinct roles as the total says, one
without when it read up to an empty page (see `_paged`); a host that answers
with the whole board at once is complete whenever it parses.
"""
from __future__ import annotations
import json
import logging
import re
import time
from collections.abc import Callable, Iterator, MutableMapping
from datetime import UTC, datetime, timedelta
from typing import Any
from urllib.robotparser import RobotFileParser
import httpx
from desk.sources import (
API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source,
SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT,
)
from plugins import atshosts
from plugins.atshosts._shared import Get, Row
# The names the parsers had here before they moved to their host modules,
# kept for the callers and the tests that still reach for them on `ats`.
from plugins.atshosts._shared import ( # noqa: F401
Page, is_web_url as _is_web_url, parse_dt as _parse_dt, yearly_amount as _yearly_amount,
)
from plugins.atshosts.greenhouse import _salary as _greenhouse_salary # noqa: F401
from plugins.atshosts.teamtailor import parse as _parse_teamtailor # noqa: F401
log = logging.getLogger(__name__)
#: openings.go's 8 MB body cap is not kept: it was written against arbitrary URLs, and every
#: URL fetched here is a feed on a fixed allow-list of ATS hosts, Greenhouse's
#: with `?content=true`. The real run of 2026-09-23 hit it on the biggest employers —
#: Anduril at 41 MB, Anthropic, OpenAI, Databricks, Carvana, Axon and
#: Airwallex between 9 and 19 MB — and skipped their boards outright. Raised
#: to 96 MB so those boards fit; a board that still exceeds it gets one retry
#: at its host's `fallback_url` where it has one (Greenhouse, without
#: `?content=true`, see `_fetch_rows`), so at least the titles land.
MAX_BYTES = 96 << 20
#: openings.go's 10 s did not hold: `greenhouse:mongodb` and `greenhouse:feedzai`
#: failed a pull of 2026-09-28 with a read timeout. Measured live on
#: 2026-09-29, three runs each of mongodb, feedzai and the five Greenhouse
#: boards with the most open roles (databricks, stripe, anthropic, feverup,
#: datadog), with and without `pay_transparency=true`, cached and with a
#: cache-busting parameter (84 requests; CloudFront answered every one
#: "Miss", so all reached the origin): the body arrives in under 1 s once it
#: starts (at most 0.09 s between two chunks, 10.4 MB the largest), but the
#: first byte sometimes waits: 16.6 s (feverup, no flag), 14.4 s (anthropic,
#: flag), 10.9 s (feverup, flag), 5.2 s and 3.6 s, and under 2.6 s otherwise.
#: The flag made no difference that three runs could tell apart from that
#: noise. So a read may wait 30 s, about twice the worst first byte seen,
#: and a whole request, redirects and body included, 60 s (see `_get`),
#: about four times the slowest seen, which still leaves a 96 MB body room
#: on a link far slower than the one measured.
TIMEOUTS = httpx.Timeout(10.0, read=30.0)
DEADLINE = 60.0
#: host -> the name the host gives itself: the hosts this plugin reads, in
#: the order a select offers them. What the Sources page's host select is
#: rendered from, so it cannot offer a host `is_known_target` refuses.
HOSTS: dict[str, str] = {name: module.LABEL for name, module in atshosts.HOSTS.items()}
def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
"""The hosts a request or a redirect may land on for one board: what its
host module's `allowed` says, which is one fixed API host or the
company's own site. Anything else is refused, at the first hop and at
every redirect: a feed is external content, and a board row must never
be able to point this fetch at an internal address."""
module = atshosts.HOSTS.get(host)
return module.allowed(slug) if module is not None else lambda h: False
class _Refused(Exception):
"""A board answered somewhere its allow-list does not cover."""
class _TooLarge(ValueError):
"""A response exceeded `MAX_BYTES`. A `ValueError` so a caller that only
knows the old shape still catches it; `_fetch_rows` catches it by type to
retry at the host's `fallback_url`."""
class _Disallowed(Exception):
"""The site's robots.txt disallows the page a board is read from (see
`_check_robots`)."""
def _get(client: httpx.Client, url: str, host: str, slug: str, *,
method: str = "GET", payload: dict[str, Any] | None = None,
headers: dict[str, str] | None = None) -> bytes:
"""Fetch one URL for a board: https only, the host's own allow-list, at
most three redirect hops each checked the same way, and the body cut off
at `MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST
sends `payload` as JSON, `{}` when there is none. `headers` are sent on
top of the User-Agent and Accept every request carries, and replace
either where they name it.
The body is streamed and counted as it arrives, so an oversized one is
refused at `MAX_BYTES` rather than after it has been read whole, and
the whole request, redirects included, must end within `DEADLINE`: the
per-read `TIMEOUTS` alone let a board that trickles bytes run on for
good. The deadline is checked between chunks, so it can be overrun by
at most one read timeout."""
allow = _allowed_host(host, slug)
sent = {"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"} | (headers or {})
deadline = time.monotonic() + DEADLINE
current = url
for _hop in range(4):
parsed = httpx.URL(current)
if parsed.scheme != "https" or not allow(parsed.host):
raise _Refused(f"{current} is not an allowed host for {host}")
with client.stream(
method, current, timeout=TIMEOUTS, follow_redirects=False, headers=sent,
json=(payload or {}) if method == "POST" else None,
) as response:
if response.is_redirect:
location = response.headers.get("location") or ""
if not location:
raise ValueError("redirect with no location")
current = str(httpx.URL(current).join(location))
continue
response.raise_for_status()
chunks: list[bytes] = []
size = 0
for chunk in response.iter_bytes():
size += len(chunk)
if size > MAX_BYTES:
raise _TooLarge(f"response too large (over {MAX_BYTES} bytes)")
if time.monotonic() > deadline:
raise httpx.ReadTimeout(f"no whole answer within {DEADLINE:.0f} s",
request=response.request)
chunks.append(chunk)
return b"".join(chunks)
raise ValueError("too many redirects")
def _getter(client: httpx.Client, host: str, slug: str) -> Get:
"""The `get` a host module is handed: `_get` bound to one board. `_get`
is looked up when a request is made, not when this is built, so a test
that replaces `ats._get` reaches every host; `payload` and `headers` are
passed on only when given, which keeps a replacement that predates them
working."""
def get(url: str, *, method: str = "GET", payload: dict[str, Any] | None = None,
headers: dict[str, str] | None = None) -> bytes:
extra: dict[str, Any] = {}
if payload is not None:
extra["payload"] = payload
if headers:
extra["headers"] = headers
return _get(client, url, host, slug, method=method, **extra)
return get
#: host -> parser, for the hosts whose module parses a whole body. Every one
#: takes the raw response body and the slug.
_PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = {
name: module.parse for name, module in atshosts.HOSTS.items() if hasattr(module, "parse")}
# --------------------------------------------------------- the classifier
#: The fixed set of functions, and the keyword rules that map a title
#: (preferred) or a department to one of them. Ported from `cmd/seedgen/
#: main.go`'s `rules` — first match wins, and "product" is genuinely listed
#: twice: the first, narrower entry is checked before
#: recruiting/design/marketing/support/sales/data, and the second, which also
#: matches the bare word "product", is checked only after all of those, so a
#: title like "Data Product Manager" keeps classifying as data/product by the
#: same rule order the Go tool used.
#:
#: Not a straight port any more: the Go tool matched substrings, which let
#: " it" (meant to catch "IT" as a department) match inside "Italy" and
#: "Italia" — a European department like "Commercial Italy" filed as
#: engineering. `_classify_text` now matches on word boundaries. A handful of
#: short tokens (`_WHOLE_WORD`) need a boundary on *both* sides, because they
#: are themselves the start of an unrelated word ("it" of "Italy", "seo" of
#: "Seoul", "qa" of "Qatar"); every other keyword keeps a leading boundary
#: only; most of them are deliberate stems ("manag", "engineer", "recruit")
#: meant to also catch "management", "engineering", "recruiting", and a
#: trailing boundary on those would stop them matching the plural and
#: gerund forms real postings use ("Software Developers", "Account
#: Managers").
FUNCTIONS = ("engineering", "data", "product", "design", "sales", "marketing",
"customer success/support", "recruiting/people", "finance/legal/ops")
_RULES: tuple[tuple[str, tuple[str, ...]], ...] = (
("product", ("product manag", "product management", "product operations",
"program manag", "technical program")),
("recruiting/people", ("recruit", "talent", "people ops", "people &",
"people team", "people operations", "human resources",
"hr", "hrbp")),
("design", ("design", "ux", "user experience", "creative")),
("marketing", ("marketing", "growth", "content", "communications", "brand",
"seo", "social media", "demand gen", "events")),
("customer success/support", ("customer success", "customer support",
"customer experience", "customer service",
"support", "success", "implementation",
"onboarding", "professional services",
"solutions consult", "technical account",
"customer engineer", "deployment")),
("sales", ("sales", "account executive", "account manager", "account director",
"business development", "bdr", "sdr", "revenue", "partnership",
"go-to-market", "gtm", "solutions engineer", "solution engineer",
"pre-sales", "presales", "channel",
# "Commercial" is what French, Italian and Spanish org charts
# call the sales function ("Commercial Italy" is a sales
# department; see the docstring above), and the rest are the
# sales words of the seed's other languages that "sales"
# itself does not already cover (German, Dutch, Danish,
# Norwegian, Swedish, French) — a modest set, not a full port;
# `extract` is the rest.
"commercial", "vertrieb", "verkauf", "verkoop", "salg",
"försäljning", "vente")),
("data", ("data", "analytics", "machine learning", "ml",
"applied science", "research scientist", "statistic")),
("product", ("product manag", "product management", "program manag",
"product operations", "technical program", "product")),
("finance/legal/ops", ("finance", "financ", "accounting", "legal", "counsel",
"compliance", "operations", "ops", "administrat",
"office", "procurement", "strategy", "corporate",
"facilities", "risk", "audit", "tax", "treasury",
"payroll", "executive assistant", "chief of staff",
"workplace", "supply chain", "logistics", "fraud",
"trust & safety", "trust and safety", "policy")),
("engineering", ("engineer", "software", "developer", "infrastructure",
"platform", "devops", "sre", "site reliability", "security",
"information technology", "it", "technology",
"technical staff", "hardware", "qa", "quality assurance",
"architect", "research", "science", "scientist", "systems",
"network", "cloud", "mobile", "frontend", "backend",
"full stack", "fullstack", "web",
# The developer/engineer words of the seed's other
# languages, same reasoning as sales' above.
"entwickler", "ingenieur", "ontwikkelaar", "udvikler",
"utvikler", "utvecklare", "ingenjör", "ingeniør",
"développeur", "developpeur", "ingénieur")),
)
#: Keywords that must match a whole word, not just its start: each one is
#: itself the first letters of an unrelated word that would otherwise false-
#: match ("it" of "Italy", "seo" of "Seoul", "qa" of "Qatar"). Every keyword
#: not listed here keeps the old prefix behaviour on purpose — see the
#: `_RULES` docstring above.
_WHOLE_WORD = frozenset({
"it", "hr", "hrbp", "ux", "ml", "qa", "seo", "web", "tax", "ops",
"sre", "bdr", "sdr", "gtm",
})
def _keyword_pattern(keyword: str) -> re.Pattern[str]:
trailing = r"\b" if keyword in _WHOLE_WORD else ""
return re.compile(r"\b" + re.escape(keyword) + trailing, re.IGNORECASE)
#: `_RULES`, compiled once: `(function, patterns)` in the same order.
_COMPILED_RULES: tuple[tuple[str, tuple[re.Pattern[str], ...]], ...] = tuple(
(function, tuple(_keyword_pattern(keyword) for keyword in keywords))
for function, keywords in _RULES)
def _classify_text(text: str) -> str:
stripped = (text or "").strip()
if not stripped:
return ""
for function, patterns in _COMPILED_RULES:
if any(pattern.search(stripped) for pattern in patterns):
return function
return ""
def classify(department: str, title: str) -> str:
"""A title (preferred) or a department, to one of `FUNCTIONS`, or `''`
when neither names one — `''` and not `"other"`, which would claim a
knowledge the classifier does not have (`extract` may still fill it from
reading the posting; see `Store.set_extracted`).
Title before department: a department is often a generic umbrella
("Operations", "Engineering & Product") that says less about the role
than its title does. A hand-checked sample of 60 roles where the two
orders disagreed (docs/decisions.md) found title-first right about twice
as often as department-first, mostly because a specific title beats a
broad department name, so title now wins; department is the fallback
when the title matches nothing."""
return _classify_text(title or "") or _classify_text(department or "") or ""
def is_known_target(target: str) -> bool:
"""`host:slug`, with `host` one of `atshosts.HOSTS` and a slug its module's
`slug_ok` accepts. What the target validator refuses: `app companies add`
and the pull job both call this before touching the network."""
host, sep, slug = target.partition(":")
module = atshosts.HOSTS.get(host)
return bool(sep) and module is not None and bool(slug.strip()) and module.slug_ok(slug)
def careers_url(target: str) -> str:
"""The company's own careers page for `host:slug`, or "" for an unknown
host. What `app companies add` stores alongside the board."""
host, _, slug = target.partition(":")
module = atshosts.HOSTS.get(host)
return module.careers_url(slug) if module is not None else ""
#: Pages read from one paged board at most, and the pause between two of
#: them. Workable answers ten roles a page, SmartRecruiters a hundred and
#: Teamtailor fifty, so the cap is 1,000, 10,000 and 5,000 roles; the pause
#: keeps Workable, which answers a burst with 429, from seeing one. The same
#: pause separates two detail reads.
MAX_PAGES = 100
PAGE_DELAY = 0.5
def _paged(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
"""Every page of a board whose host pages, `PAGE_DELAY` apart, until the
host has no next page, the stated total is reached, or `MAX_PAGES`.
Returns the rows and whether the board is complete: as many distinct
results read as the host's stated total, or, for a host that states none
(Teamtailor), the read ended on an empty page (a page with nothing new in
it ends the read short of that, incomplete, so an offset the host ignores
cannot loop). Counted on the raw results, so a row the parser drops does
not make a board incomplete, and on distinct ids, so a page that shifted
under an offset and repeated a role does not make it complete.
A 429 after the first page stops the read with what it has, incomplete;
one on the first page raises like any other failure."""
host, _, slug = target.partition(":")
module = atshosts.HOSTS[host]
get = _getter(client, host, slug)
rows: dict[str, Row] = {}
seen: set[str] = set()
total: Any = None
cursor: Any = None
for page in range(MAX_PAGES):
if page:
time.sleep(PAGE_DELAY)
try:
data, results, total, cursor = module.page(get, slug, cursor)
except httpx.HTTPStatusError as exc:
if not page or exc.response.status_code != 429:
raise
log.warning("ats: %s: 429 on page %d; kept %d of %s, not complete",
target, page + 1, len(seen), total)
return list(rows.values()), False
before = len(seen)
seen |= {str(r.get("id") or r.get("shortcode") or "") for r in results}
for row in module.rows(data, slug):
rows.setdefault(row["source_id"], row)
if (cursor is None or (isinstance(total, int) and len(seen) >= total)
or (total is None and len(seen) == before)):
break
complete = len(seen) == total if isinstance(total, int) else total is None and cursor is None
if not complete:
log.warning("ats: %s: read %d of %s stated; not complete", target, len(seen),
"an unstated total" if total is None else total)
return list(rows.values()), complete
def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]:
"""One board's listing, fetched and parsed: its rows and whether they are
the whole board. Raises on any failure — refused host, timeout, a feed
shape the host's parser does not understand. A host that does not page
answers with the whole board, so it is complete whenever it parses; a
paged one is complete when `_paged` says so.
A body that exceeds `MAX_BYTES` gets one retry at the host's
`fallback_url` where it has one (Greenhouse, without `?content=true`):
the description is lost, but the titles land rather than the whole board
being skipped."""
if not is_known_target(target):
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
host, _, slug = target.partition(":")
module = atshosts.HOSTS[host]
if hasattr(module, "page"):
return _paged(client, target)
get = _getter(client, host, slug)
try:
body = get(module.feed_url(slug))
except _TooLarge:
if not hasattr(module, "fallback_url"):
raise
log.warning("ats: %s: body exceeded %d bytes; retrying at the smaller feed",
target, MAX_BYTES)
body = get(module.fallback_url(slug))
return module.parse(body, slug), True
# -------------------------------------------------------------- robots.txt
#: How long a robots.txt answer holds for one board before it is read again.
ROBOTS_TTL = timedelta(days=1)
def _robots_allows(body: bytes, page: str) -> bool:
"""Whether a robots.txt lets `page` be fetched: by its rules for every
agent (`*`), and by any it has for this app's own User-Agent token, so a
site that shuts out this app by name is honoured as well."""
parser = RobotFileParser()
parser.parse(body.decode("utf-8", "replace").splitlines())
return parser.can_fetch("*", page) and parser.can_fetch(USER_AGENT, page)
def _check_robots(client: httpx.Client, target: str,
state: MutableMapping[str, str] | None = None) -> None:
"""Raise `_Disallowed` when the site of a host with a `robots_page` has a
robots.txt that disallows that page. Read through `_get`, so from the
board's own allowed hosts only, and at most once per `ROBOTS_TTL` for a
board when `state` is given: the verdict and when it was reached are kept
under `ats:robots:<target>`. A robots.txt that is missing (404 or 410)
allows, and that verdict is kept like any other; one that cannot be read
for another reason allows too, but is not kept, so the next pull asks
again."""
host, _, slug = target.partition(":")
module = atshosts.HOSTS.get(host)
if module is None or not hasattr(module, "robots_page") or not is_known_target(target):
return
page = module.robots_page(slug)
key = f"ats:robots:{target}"
now = datetime.now(UTC)
if state is not None:
try:
kept = json.loads(state.get(key) or "{}")
fresh = now - datetime.fromisoformat(kept["checked"]) < ROBOTS_TTL
except (ValueError, KeyError, TypeError):
kept, fresh = {}, False
if fresh:
if not kept.get("allowed", True):
raise _Disallowed(f"robots.txt disallows {page}")
return
url = f"https://{httpx.URL(page).host}/robots.txt"
try:
body = _getter(client, host, slug)(url, headers={"Accept": "text/plain"})
except httpx.HTTPStatusError as exc:
if exc.response.status_code not in (404, 410):
log.warning("ats: %s: robots.txt answered %s; allowed until the next pull",
target, exc.response.status_code)
return
allowed = True
except Exception as exc:
log.warning("ats: %s: robots.txt not read (%s); allowed until the next pull",
target, exc)
return
else:
allowed = _robots_allows(body, page)
if state is not None:
state[key] = json.dumps({"checked": now.isoformat(timespec="seconds"),
"allowed": allowed})
if not allowed:
raise _Disallowed(f"robots.txt disallows {page}")
# ---------------------------------------------------------------- details
#: Roles read in detail from one board in one pull at most. The rest of a
#: large new board is read over the next pulls, 200 at a time.
DETAIL_CAP = 200
def _read_details(client: httpx.Client, target: str, rows: list[Row], complete: bool,
detailed: set[str]) -> tuple[list[Row], set[str]]:
"""The rows of a board whose host has `detail`, each role not yet in
`detailed` (compound ids) filled from its own page, at most `DETAIL_CAP`
of them and `PAGE_DELAY` apart. Returns the rows and the ids read in
detail so far, for `ats:detailed:<target>`.
Every row not read in detail in this call goes out with an empty
description: a role read on an earlier pull keeps the one it has, since
`upsert_role` never overwrites a filled column with an empty value, and a
teaser from the listing must not replace it. A detail that raises is
logged and the role left for the next pull; a 429 stops the reads for this
board until then. None of this touches completeness, which is the
listing's alone. The ids of roles no longer listed are dropped, but only
after a complete listing, so one cut short does not make the next pull
read its missing roles again."""
host, _, slug = target.partition(":")
module = atshosts.HOSTS[host]
get = _getter(client, host, slug)
done = set(detailed)
out: list[Row] = []
tried = 0
stopped = False
for row in rows:
compound = f"{host}:{slug}:{row['source_id']}"
if compound in done or stopped or tried >= DETAIL_CAP:
out.append(row | {"description": ""})
continue
time.sleep(PAGE_DELAY)
tried += 1
try:
filled = module.detail(get, slug, row)
except Exception as exc:
if isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code == 429:
log.warning("ats: %s: 429 on a detail read; the rest wait for the next pull",
target)
stopped = True
else:
log.warning("ats: %s: detail of %s failed (%s)", target, row["source_id"], exc)
out.append(row | {"description": ""})
continue
out.append(row | filled | {"source_id": row["source_id"]})
done.add(compound)
if complete:
done &= {f"{host}:{slug}:{row['source_id']}" for row in rows}
return out, done
def fetch_board(target: str) -> tuple[list[Row], bool]:
"""Fetch and parse one board directly: no `Source`, no `self.state`, just
"does this board answer, what does it list, and is that all of it". For
`app companies add|check`, which ask that question about one board at a
time outside any pull. The listing only, with no detail reads, and
robots.txt read afresh. Raises the same way `_fetch_rows` does, and
`_Disallowed` for a board robots.txt shuts out."""
with httpx.Client() as client:
_check_robots(client, target)
return _fetch_rows(client, target)
def failure_reason(exc: BaseException) -> str:
"""One short line for why a board did not answer: the status of an HTTP
error, else the exception's class and the first line of its message."""
if isinstance(exc, _Disallowed):
return "robots.txt disallows it"
if isinstance(exc, httpx.HTTPStatusError):
return f"{exc.response.status_code} {exc.response.reason_phrase}".strip()
lines = str(exc).strip().splitlines()
text = f"{type(exc).__name__}: {lines[0]}" if lines else type(exc).__name__
return text[:200]
# -------------------------------------------------------------- the plugin
class ATSSource(Source):
name = "ats"
description = f"Company job boards: {', '.join(HOSTS.values())}."
#: The raw entries under `sources.ats.boards` are `{"url": target, "label":
#: company}` — `url` carries the `host:slug` target and `label` the
#: company name, which is how `Source.targets()`'s generic dict reading
#: hands both back without a plugin-specific override.
subscription_field = "boards"
spec = PluginSpec(
api_version=API_VERSION, id=name, version="1", label="ATS boards",
targets=TargetSpec(mode="editable", kind="board", defaults=[]),
profiles={"default": Profile(
kind="role",
tasks=Tasks(
summary=SummaryPolicy(enabled=False),
tagging=TaggingPolicy(enabled=False),
),
presentation=ARTICLE_READER,
)},
)
def fetch(self) -> Iterator[Item]:
with httpx.Client() as client:
# Handed in by the batch that runs this plugin, as the marker of
# the pull run that queued it (see `app/jobs/pull.py`), and never
# read from state, where a later run may have replaced it. A
# plugin run on its own — a test, a dry run — has no run to stamp
# itself against, so it makes one up.
run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat(
timespec="seconds")
for target, label in self.targets():
host, _, slug = target.partition(":")
failing = f"ats:failing:{target}"
try:
_check_robots(client, target, self.state)
rows, complete = _fetch_rows(client, target)
except Exception as exc:
log.warning("ats: %s failed (%s)", target, exc)
# The first failure in a row sets `since`, later ones only
# the reason; `app.jobs.pull` closes the board's roles
# once `since` is `STALE_DAYS` old.
try:
since = json.loads(self.state.get(failing, "") or "{}").get("since")
except ValueError:
since = None
self.state[failing] = json.dumps({
"since": since or datetime.now(UTC).isoformat(timespec="seconds"),
"reason": failure_reason(exc)})
continue
self.state.pop(failing, None)
detailed_key = f"ats:detailed:{target}"
detailed: set[str] | None = None
if hasattr(atshosts.HOSTS.get(host), "detail"):
try:
detailed = set(json.loads(self.state.get(detailed_key) or "[]"))
except (ValueError, TypeError):
detailed = set()
rows, detailed = _read_details(client, target, rows, complete, detailed)
seen_ids = []
for row in rows:
# The compound id, matching `roles.source_id` — what
# `Store.mark_closed` compares against, not the board's own
# bare job id.
compound_id = f"{host}:{slug}:{row['source_id']}"
seen_ids.append(compound_id)
yield Item(
source_id=compound_id,
location=row["url"],
title=row["title"],
excerpt=(row["description"] or "")[:300],
body=row["description"],
published_at=row["posted"],
subscription=label,
kind="role",
raw={
"company": label,
"place": row["location"],
"department": row["department"],
"function": classify(row["department"], row["title"]),
"remote": row["remote"],
# Every parser sets salary_text, empty or not (see
# "salary_text" in `atshosts/_shared.py`); only
# the hosts that state structured pay also set the
# normalised figures, so the rest leave them at
# row.get's default of no figure.
"salary_text": row.get("salary_text", ""),
"salary_min": row.get("salary_min"),
"salary_max": row.get("salary_max"),
"currency": row.get("currency", ""),
"board": host,
"slug": slug,
},
)
# Stamped only once the board has parsed without raising, so a
# board that blew up half way through is neither marked ok nor
# has a partial id list treated as the whole of it. `complete`
# says whether the ids are the whole board, which the pull
# needs before it closes what they leave out. The ids read in
# detail are kept here too, after every row went out, so a
# pull stopped half way reads the rest again rather than
# losing their descriptions.
self.state[f"ats:ok:{target}"] = run_marker
self.state[f"ats:complete:{target}"] = "1" if complete else ""
self.state[f"ats:ids:{target}"] = json.dumps(seen_ids)
if detailed is not None:
self.state[detailed_key] = json.dumps(sorted(detailed))