"""ATS boards: a company's own careers page on one of the hosts in `plugins/atshosts/`, one module per host. Every one of these hosts serves a company's own list of roles as public, unauthenticated JSON, XML or RSS — first-party, no scraping, and a role that drops off the board is *closed*, which is the cheapest staleness check there is. What a host says and how its feed reads is its module's; this file holds what every host shares: the fetch and its allow-list, paging and completeness, the classifier, and the plugin itself. The allow-list and the classifier are ported from `openings.go` and `cmd/seedgen/main.go` in [jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT), including `classify`'s nine functions and their keywords. The body cap is not a straight port — see `MAX_BYTES` below. A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`, `lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what its slug is. Every board is fetched independently; one that fails or times out is logged, recorded as failing under `ats:failing:` (see `ATSSource.fetch`), and skipped, and the rest continue. A host that pages states a total or does not. A board with a total counts as complete only when it read as many distinct roles as the total says, one without when it read up to an empty page (see `_paged`); a host that answers with the whole board at once is complete whenever it parses. """ from __future__ import annotations import json import logging import re import time from collections.abc import Callable, Iterator from datetime import UTC, datetime from typing import Any import httpx from desk.sources import ( API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source, SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT, ) from plugins import atshosts from plugins.atshosts._shared import Get, Row # The names the parsers had here before they moved to their host modules, # kept for the callers and the tests that still reach for them on `ats`. from plugins.atshosts._shared import ( # noqa: F401 Page, is_web_url as _is_web_url, parse_dt as _parse_dt, yearly_amount as _yearly_amount, ) from plugins.atshosts.greenhouse import _salary as _greenhouse_salary # noqa: F401 from plugins.atshosts.teamtailor import parse as _parse_teamtailor # noqa: F401 log = logging.getLogger(__name__) #: openings.go's 8 MB body cap is not kept: it was written against arbitrary URLs, and every #: URL fetched here is a feed on a fixed allow-list of ATS hosts, Greenhouse's #: with `?content=true`. The real run of 2026-09-23 hit it on the biggest employers — #: Anduril at 41 MB, Anthropic, OpenAI, Databricks, Carvana, Axon and #: Airwallex between 9 and 19 MB — and skipped their boards outright. Raised #: to 96 MB so those boards fit; a board that still exceeds it gets one retry #: at its host's `fallback_url` where it has one (Greenhouse, without #: `?content=true`, see `_fetch_rows`), so at least the titles land. MAX_BYTES = 96 << 20 #: openings.go's 10 s did not hold: `greenhouse:mongodb` and `greenhouse:feedzai` #: failed a pull of 2026-09-28 with a read timeout. Measured live on #: 2026-09-29, three runs each of mongodb, feedzai and the five Greenhouse #: boards with the most open roles (databricks, stripe, anthropic, feverup, #: datadog), with and without `pay_transparency=true`, cached and with a #: cache-busting parameter (84 requests; CloudFront answered every one #: "Miss", so all reached the origin): the body arrives in under 1 s once it #: starts (at most 0.09 s between two chunks, 10.4 MB the largest), but the #: first byte sometimes waits: 16.6 s (feverup, no flag), 14.4 s (anthropic, #: flag), 10.9 s (feverup, flag), 5.2 s and 3.6 s, and under 2.6 s otherwise. #: The flag made no difference that three runs could tell apart from that #: noise. So a read may wait 30 s, about twice the worst first byte seen, #: and a whole request, redirects and body included, 60 s (see `_get`), #: about four times the slowest seen, which still leaves a 96 MB body room #: on a link far slower than the one measured. TIMEOUTS = httpx.Timeout(10.0, read=30.0) DEADLINE = 60.0 #: host -> the name the host gives itself: the hosts this plugin reads, in #: the order a select offers them. What the Sources page's host select is #: rendered from, so it cannot offer a host `is_known_target` refuses. HOSTS: dict[str, str] = {name: module.LABEL for name, module in atshosts.HOSTS.items()} def _allowed_host(host: str, slug: str) -> Callable[[str], bool]: """The hosts a request or a redirect may land on for one board: what its host module's `allowed` says, which is one fixed API host or the company's own site. Anything else is refused, at the first hop and at every redirect: a feed is external content, and a board row must never be able to point this fetch at an internal address.""" module = atshosts.HOSTS.get(host) return module.allowed(slug) if module is not None else lambda h: False class _Refused(Exception): """A board answered somewhere its allow-list does not cover.""" class _TooLarge(ValueError): """A response exceeded `MAX_BYTES`. A `ValueError` so a caller that only knows the old shape still catches it; `_fetch_rows` catches it by type to retry at the host's `fallback_url`.""" def _get(client: httpx.Client, url: str, host: str, slug: str, *, method: str = "GET", payload: dict[str, Any] | None = None, headers: dict[str, str] | None = None) -> bytes: """Fetch one URL for a board: https only, the host's own allow-list, at most three redirect hops each checked the same way, and the body cut off at `MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST sends `payload` as JSON, `{}` when there is none. `headers` are sent on top of the User-Agent and Accept every request carries, and replace either where they name it. The body is streamed and counted as it arrives, so an oversized one is refused at `MAX_BYTES` rather than after it has been read whole, and the whole request, redirects included, must end within `DEADLINE`: the per-read `TIMEOUTS` alone let a board that trickles bytes run on for good. The deadline is checked between chunks, so it can be overrun by at most one read timeout.""" allow = _allowed_host(host, slug) sent = {"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"} | (headers or {}) deadline = time.monotonic() + DEADLINE current = url for _hop in range(4): parsed = httpx.URL(current) if parsed.scheme != "https" or not allow(parsed.host): raise _Refused(f"{current} is not an allowed host for {host}") with client.stream( method, current, timeout=TIMEOUTS, follow_redirects=False, headers=sent, json=(payload or {}) if method == "POST" else None, ) as response: if response.is_redirect: location = response.headers.get("location") or "" if not location: raise ValueError("redirect with no location") current = str(httpx.URL(current).join(location)) continue response.raise_for_status() chunks: list[bytes] = [] size = 0 for chunk in response.iter_bytes(): size += len(chunk) if size > MAX_BYTES: raise _TooLarge(f"response too large (over {MAX_BYTES} bytes)") if time.monotonic() > deadline: raise httpx.ReadTimeout(f"no whole answer within {DEADLINE:.0f} s", request=response.request) chunks.append(chunk) return b"".join(chunks) raise ValueError("too many redirects") def _getter(client: httpx.Client, host: str, slug: str) -> Get: """The `get` a host module is handed: `_get` bound to one board. `_get` is looked up when a request is made, not when this is built, so a test that replaces `ats._get` reaches every host; `payload` and `headers` are passed on only when given, which keeps a replacement that predates them working.""" def get(url: str, *, method: str = "GET", payload: dict[str, Any] | None = None, headers: dict[str, str] | None = None) -> bytes: extra: dict[str, Any] = {} if payload is not None: extra["payload"] = payload if headers: extra["headers"] = headers return _get(client, url, host, slug, method=method, **extra) return get #: host -> parser, for the hosts whose module parses a whole body. Every one #: takes the raw response body and the slug. _PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = { name: module.parse for name, module in atshosts.HOSTS.items() if hasattr(module, "parse")} # --------------------------------------------------------- the classifier #: The fixed set of functions, and the keyword rules that map a title #: (preferred) or a department to one of them. Ported from `cmd/seedgen/ #: main.go`'s `rules` — first match wins, and "product" is genuinely listed #: twice: the first, narrower entry is checked before #: recruiting/design/marketing/support/sales/data, and the second, which also #: matches the bare word "product", is checked only after all of those, so a #: title like "Data Product Manager" keeps classifying as data/product by the #: same rule order the Go tool used. #: #: Not a straight port any more: the Go tool matched substrings, which let #: " it" (meant to catch "IT" as a department) match inside "Italy" and #: "Italia" — a European department like "Commercial Italy" filed as #: engineering. `_classify_text` now matches on word boundaries. A handful of #: short tokens (`_WHOLE_WORD`) need a boundary on *both* sides, because they #: are themselves the start of an unrelated word ("it" of "Italy", "seo" of #: "Seoul", "qa" of "Qatar"); every other keyword keeps a leading boundary #: only; most of them are deliberate stems ("manag", "engineer", "recruit") #: meant to also catch "management", "engineering", "recruiting", and a #: trailing boundary on those would stop them matching the plural and #: gerund forms real postings use ("Software Developers", "Account #: Managers"). FUNCTIONS = ("engineering", "data", "product", "design", "sales", "marketing", "customer success/support", "recruiting/people", "finance/legal/ops") _RULES: tuple[tuple[str, tuple[str, ...]], ...] = ( ("product", ("product manag", "product management", "product operations", "program manag", "technical program")), ("recruiting/people", ("recruit", "talent", "people ops", "people &", "people team", "people operations", "human resources", "hr", "hrbp")), ("design", ("design", "ux", "user experience", "creative")), ("marketing", ("marketing", "growth", "content", "communications", "brand", "seo", "social media", "demand gen", "events")), ("customer success/support", ("customer success", "customer support", "customer experience", "customer service", "support", "success", "implementation", "onboarding", "professional services", "solutions consult", "technical account", "customer engineer", "deployment")), ("sales", ("sales", "account executive", "account manager", "account director", "business development", "bdr", "sdr", "revenue", "partnership", "go-to-market", "gtm", "solutions engineer", "solution engineer", "pre-sales", "presales", "channel", # "Commercial" is what French, Italian and Spanish org charts # call the sales function ("Commercial Italy" is a sales # department; see the docstring above), and the rest are the # sales words of the seed's other languages that "sales" # itself does not already cover (German, Dutch, Danish, # Norwegian, Swedish, French) — a modest set, not a full port; # `extract` is the rest. "commercial", "vertrieb", "verkauf", "verkoop", "salg", "försäljning", "vente")), ("data", ("data", "analytics", "machine learning", "ml", "applied science", "research scientist", "statistic")), ("product", ("product manag", "product management", "program manag", "product operations", "technical program", "product")), ("finance/legal/ops", ("finance", "financ", "accounting", "legal", "counsel", "compliance", "operations", "ops", "administrat", "office", "procurement", "strategy", "corporate", "facilities", "risk", "audit", "tax", "treasury", "payroll", "executive assistant", "chief of staff", "workplace", "supply chain", "logistics", "fraud", "trust & safety", "trust and safety", "policy")), ("engineering", ("engineer", "software", "developer", "infrastructure", "platform", "devops", "sre", "site reliability", "security", "information technology", "it", "technology", "technical staff", "hardware", "qa", "quality assurance", "architect", "research", "science", "scientist", "systems", "network", "cloud", "mobile", "frontend", "backend", "full stack", "fullstack", "web", # The developer/engineer words of the seed's other # languages, same reasoning as sales' above. "entwickler", "ingenieur", "ontwikkelaar", "udvikler", "utvikler", "utvecklare", "ingenjör", "ingeniør", "développeur", "developpeur", "ingénieur")), ) #: Keywords that must match a whole word, not just its start: each one is #: itself the first letters of an unrelated word that would otherwise false- #: match ("it" of "Italy", "seo" of "Seoul", "qa" of "Qatar"). Every keyword #: not listed here keeps the old prefix behaviour on purpose — see the #: `_RULES` docstring above. _WHOLE_WORD = frozenset({ "it", "hr", "hrbp", "ux", "ml", "qa", "seo", "web", "tax", "ops", "sre", "bdr", "sdr", "gtm", }) def _keyword_pattern(keyword: str) -> re.Pattern[str]: trailing = r"\b" if keyword in _WHOLE_WORD else "" return re.compile(r"\b" + re.escape(keyword) + trailing, re.IGNORECASE) #: `_RULES`, compiled once: `(function, patterns)` in the same order. _COMPILED_RULES: tuple[tuple[str, tuple[re.Pattern[str], ...]], ...] = tuple( (function, tuple(_keyword_pattern(keyword) for keyword in keywords)) for function, keywords in _RULES) def _classify_text(text: str) -> str: stripped = (text or "").strip() if not stripped: return "" for function, patterns in _COMPILED_RULES: if any(pattern.search(stripped) for pattern in patterns): return function return "" def classify(department: str, title: str) -> str: """A title (preferred) or a department, to one of `FUNCTIONS`, or `''` when neither names one — `''` and not `"other"`, which would claim a knowledge the classifier does not have (`extract` may still fill it from reading the posting; see `Store.set_extracted`). Title before department: a department is often a generic umbrella ("Operations", "Engineering & Product") that says less about the role than its title does. A hand-checked sample of 60 roles where the two orders disagreed (docs/decisions.md) found title-first right about twice as often as department-first, mostly because a specific title beats a broad department name, so title now wins; department is the fallback when the title matches nothing.""" return _classify_text(title or "") or _classify_text(department or "") or "" def is_known_target(target: str) -> bool: """`host:slug`, with `host` one of `atshosts.HOSTS` and a slug its module's `slug_ok` accepts. What the target validator refuses: `app companies add` and the pull job both call this before touching the network.""" host, sep, slug = target.partition(":") module = atshosts.HOSTS.get(host) return bool(sep) and module is not None and bool(slug.strip()) and module.slug_ok(slug) def careers_url(target: str) -> str: """The company's own careers page for `host:slug`, or "" for an unknown host. What `app companies add` stores alongside the board.""" host, _, slug = target.partition(":") module = atshosts.HOSTS.get(host) return module.careers_url(slug) if module is not None else "" #: Pages read from one paged board at most, and the pause between two of #: them. Workable answers ten roles a page, SmartRecruiters a hundred and #: Teamtailor fifty, so the cap is 1,000, 10,000 and 5,000 roles; the pause #: keeps Workable, which answers a burst with 429, from seeing one. MAX_PAGES = 100 PAGE_DELAY = 0.5 def _paged(client: httpx.Client, target: str) -> tuple[list[Row], bool]: """Every page of a board whose host pages, `PAGE_DELAY` apart, until the host has no next page, the stated total is reached, or `MAX_PAGES`. Returns the rows and whether the board is complete: as many distinct results read as the host's stated total, or, for a host that states none (Teamtailor), the read ended on an empty page (a page with nothing new in it ends the read short of that, incomplete, so an offset the host ignores cannot loop). Counted on the raw results, so a row the parser drops does not make a board incomplete, and on distinct ids, so a page that shifted under an offset and repeated a role does not make it complete. A 429 after the first page stops the read with what it has, incomplete; one on the first page raises like any other failure.""" host, _, slug = target.partition(":") module = atshosts.HOSTS[host] get = _getter(client, host, slug) rows: dict[str, Row] = {} seen: set[str] = set() total: Any = None cursor: Any = None for page in range(MAX_PAGES): if page: time.sleep(PAGE_DELAY) try: data, results, total, cursor = module.page(get, slug, cursor) except httpx.HTTPStatusError as exc: if not page or exc.response.status_code != 429: raise log.warning("ats: %s: 429 on page %d; kept %d of %s, not complete", target, page + 1, len(seen), total) return list(rows.values()), False before = len(seen) seen |= {str(r.get("id") or r.get("shortcode") or "") for r in results} for row in module.rows(data, slug): rows.setdefault(row["source_id"], row) if (cursor is None or (isinstance(total, int) and len(seen) >= total) or (total is None and len(seen) == before)): break complete = len(seen) == total if isinstance(total, int) else total is None and cursor is None if not complete: log.warning("ats: %s: read %d of %s stated; not complete", target, len(seen), "an unstated total" if total is None else total) return list(rows.values()), complete def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]: """One board's listing, fetched and parsed: its rows and whether they are the whole board. Raises on any failure — refused host, timeout, a feed shape the host's parser does not understand. A host that does not page answers with the whole board, so it is complete whenever it parses; a paged one is complete when `_paged` says so. A body that exceeds `MAX_BYTES` gets one retry at the host's `fallback_url` where it has one (Greenhouse, without `?content=true`): the description is lost, but the titles land rather than the whole board being skipped.""" if not is_known_target(target): raise ValueError(f"{target!r} is not host:slug for a known ATS host") host, _, slug = target.partition(":") module = atshosts.HOSTS[host] if hasattr(module, "page"): return _paged(client, target) get = _getter(client, host, slug) try: body = get(module.feed_url(slug)) except _TooLarge: if not hasattr(module, "fallback_url"): raise log.warning("ats: %s: body exceeded %d bytes; retrying at the smaller feed", target, MAX_BYTES) body = get(module.fallback_url(slug)) return module.parse(body, slug), True def fetch_board(target: str) -> tuple[list[Row], bool]: """Fetch and parse one board directly: no `Source`, no `self.state`, just "does this board answer, what does it list, and is that all of it". For `app companies add|check`, which ask that question about one board at a time outside any pull. Raises the same way `_fetch_rows` does.""" with httpx.Client() as client: return _fetch_rows(client, target) def failure_reason(exc: BaseException) -> str: """One short line for why a board did not answer: the status of an HTTP error, else the exception's class and the first line of its message.""" if isinstance(exc, httpx.HTTPStatusError): return f"{exc.response.status_code} {exc.response.reason_phrase}".strip() lines = str(exc).strip().splitlines() text = f"{type(exc).__name__}: {lines[0]}" if lines else type(exc).__name__ return text[:200] # -------------------------------------------------------------- the plugin class ATSSource(Source): name = "ats" description = f"Company job boards: {', '.join(HOSTS.values())}." #: The raw entries under `sources.ats.boards` are `{"url": target, "label": #: company}` — `url` carries the `host:slug` target and `label` the #: company name, which is how `Source.targets()`'s generic dict reading #: hands both back without a plugin-specific override. subscription_field = "boards" spec = PluginSpec( api_version=API_VERSION, id=name, version="1", label="ATS boards", targets=TargetSpec(mode="editable", kind="board", defaults=[]), profiles={"default": Profile( kind="role", tasks=Tasks( summary=SummaryPolicy(enabled=False), tagging=TaggingPolicy(enabled=False), ), presentation=ARTICLE_READER, )}, ) def fetch(self) -> Iterator[Item]: with httpx.Client() as client: # Handed in by the batch that runs this plugin, as the marker of # the pull run that queued it (see `app/jobs/pull.py`), and never # read from state, where a later run may have replaced it. A # plugin run on its own — a test, a dry run — has no run to stamp # itself against, so it makes one up. run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat( timespec="seconds") for target, label in self.targets(): host, _, slug = target.partition(":") failing = f"ats:failing:{target}" try: rows, complete = _fetch_rows(client, target) except Exception as exc: log.warning("ats: %s failed (%s)", target, exc) # The first failure in a row sets `since`, later ones only # the reason; `app.jobs.pull` closes the board's roles # once `since` is `STALE_DAYS` old. try: since = json.loads(self.state.get(failing, "") or "{}").get("since") except ValueError: since = None self.state[failing] = json.dumps({ "since": since or datetime.now(UTC).isoformat(timespec="seconds"), "reason": failure_reason(exc)}) continue self.state.pop(failing, None) seen_ids = [] for row in rows: # The compound id, matching `roles.source_id` — what # `Store.mark_closed` compares against, not the board's own # bare job id. compound_id = f"{host}:{slug}:{row['source_id']}" seen_ids.append(compound_id) yield Item( source_id=compound_id, location=row["url"], title=row["title"], excerpt=(row["description"] or "")[:300], body=row["description"], published_at=row["posted"], subscription=label, kind="role", raw={ "company": label, "place": row["location"], "department": row["department"], "function": classify(row["department"], row["title"]), "remote": row["remote"], # Every parser sets salary_text, empty or not (see # "salary_text" in `atshosts/_shared.py`); only # the hosts that state structured pay also set the # normalised figures, so the rest leave them at # row.get's default of no figure. "salary_text": row.get("salary_text", ""), "salary_min": row.get("salary_min"), "salary_max": row.get("salary_max"), "currency": row.get("currency", ""), "board": host, "slug": slug, }, ) # Stamped only once the board has parsed without raising, so a # board that blew up half way through is neither marked ok nor # has a partial id list treated as the whole of it. `complete` # says whether the ids are the whole board, which the pull # needs before it closes what they leave out. self.state[f"ats:ok:{target}"] = run_marker self.state[f"ats:complete:{target}"] = "1" if complete else "" self.state[f"ats:ids:{target}"] = json.dumps(seen_ids)