"""ATS boards: a company's own careers page on one of the hosts in `plugins/atshosts/`, one module per host. Every one of these hosts serves a company's own list of roles as public, unauthenticated JSON, XML or RSS — first-party, no scraping, and a role that drops off the board is *closed*, which is the cheapest staleness check there is. What a host says and how its feed reads is its module's; this file holds what every host shares: the fetch and its allow-list, paging and completeness, the per-role detail reads, robots.txt, the classifier, and the plugin itself. The allow-list and the classifier are ported from `openings.go` and `cmd/seedgen/main.go` in [jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT), including `classify`'s nine functions and their keywords. The body cap is not a straight port — see `MAX_BYTES` below. A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`, `lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what its slug is. Every board is fetched independently; one that fails or times out is logged, recorded as failing under `ats:failing:` (see `ATSSource.fetch`), and skipped, and the rest continue. A host that pages states a total or does not. A board with a total counts as complete only when it read as many distinct roles as the total says, one without when it read up to an empty page (see `_paged`); a host that answers with the whole board at once is complete whenever it parses. """ from __future__ import annotations import json import logging import re import time from collections.abc import Callable, Iterator, MutableMapping from datetime import UTC, datetime, timedelta from typing import Any from urllib.parse import unquote import httpx from desk.sources import ( API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source, SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT, ) from plugins import atshosts from plugins.atshosts._shared import Get, Row # The names the parsers had here before they moved to their host modules, # kept for the callers and the tests that still reach for them on `ats`. from plugins.atshosts._shared import ( # noqa: F401 Page, is_web_url as _is_web_url, parse_dt as _parse_dt, yearly_amount as _yearly_amount, ) from plugins.atshosts.greenhouse import _salary as _greenhouse_salary # noqa: F401 from plugins.atshosts.teamtailor import parse as _parse_teamtailor # noqa: F401 log = logging.getLogger(__name__) #: openings.go's 8 MB body cap is not kept: it was written against arbitrary URLs, and every #: URL fetched here is a feed on a fixed allow-list of ATS hosts, Greenhouse's #: with `?content=true`. The real run of 2026-09-23 hit it on the biggest employers — #: Anduril at 41 MB, Anthropic, OpenAI, Databricks, Carvana, Axon and #: Airwallex between 9 and 19 MB — and skipped their boards outright. Raised #: to 96 MB so those boards fit; a board that still exceeds it gets one retry #: at its host's `fallback_url` where it has one (Greenhouse, without #: `?content=true`, see `_fetch_rows`), so at least the titles land. MAX_BYTES = 96 << 20 #: openings.go's 10 s did not hold: `greenhouse:mongodb` and `greenhouse:feedzai` #: failed a pull of 2026-09-28 with a read timeout. Measured live on #: 2026-09-29, three runs each of mongodb, feedzai and the five Greenhouse #: boards with the most open roles (databricks, stripe, anthropic, feverup, #: datadog), with and without `pay_transparency=true`, cached and with a #: cache-busting parameter (84 requests; CloudFront answered every one #: "Miss", so all reached the origin): the body arrives in under 1 s once it #: starts (at most 0.09 s between two chunks, 10.4 MB the largest), but the #: first byte sometimes waits: 16.6 s (feverup, no flag), 14.4 s (anthropic, #: flag), 10.9 s (feverup, flag), 5.2 s and 3.6 s, and under 2.6 s otherwise. #: The flag made no difference that three runs could tell apart from that #: noise. So a read may wait 30 s, about twice the worst first byte seen, #: and a whole request, redirects and body included, 60 s (see `_get`), #: about four times the slowest seen, which still leaves a 96 MB body room #: on a link far slower than the one measured. TIMEOUTS = httpx.Timeout(10.0, read=30.0) DEADLINE = 60.0 #: host -> the name the host gives itself: the hosts this plugin reads, in #: the order a select offers them. What the Sources page's host select is #: rendered from, so it cannot offer a host `is_known_target` refuses. HOSTS: dict[str, str] = {name: module.LABEL for name, module in atshosts.HOSTS.items()} def _allowed_host(host: str, slug: str) -> Callable[[str], bool]: """The hosts a request or a redirect may land on for one board: what its host module's `allowed` says, which is one fixed API host or the company's own site. Anything else is refused, at the first hop and at every redirect: a feed is external content, and a board row must never be able to point this fetch at an internal address.""" module = atshosts.HOSTS.get(host) return module.allowed(slug) if module is not None else lambda h: False class _Refused(Exception): """A board answered somewhere its allow-list does not cover.""" class _TooLarge(ValueError): """A response exceeded `MAX_BYTES`. A `ValueError` so a caller that only knows the old shape still catches it; `_fetch_rows` catches it by type to retry at the host's `fallback_url`.""" class _Disallowed(Exception): """The site's robots.txt disallows the page a board is read from (see `_check_robots`).""" class _RobotsUnreachable(Exception): """The site's robots.txt could not be read: a 5xx, a 429, a timeout, a connection error or a redirect off the site. RFC 9309 says to assume the site disallows everything then, so the board is not read this pull.""" def _get(client: httpx.Client, url: str, host: str, slug: str, *, method: str = "GET", payload: dict[str, Any] | None = None, headers: dict[str, str] | None = None, www_twin: bool = False) -> bytes: """Fetch one URL for a board: https only, the host's own allow-list, at most three redirect hops each checked the same way, and the body cut off at `MAX_BYTES`. Port of openings.go's `fetchOne`/`feedCache.allow`. A POST sends `payload` as JSON, `{}` when there is none. `headers` are sent on top of the User-Agent and Accept every request carries, and replace either where they name it. With `www_twin`, a redirect may also land on the first URL's host with `www.` added or taken off, which only the robots.txt read asks for (see `_check_robots`). The body is streamed and counted as it arrives, so an oversized one is refused at `MAX_BYTES` rather than after it has been read whole, and the whole request, redirects included, must end within `DEADLINE`: the per-read `TIMEOUTS` alone let a board that trickles bytes run on for good. The deadline is checked between chunks, so it can be overrun by at most one read timeout.""" allow = _allowed_host(host, slug) if www_twin: bare = httpx.URL(url).host.removeprefix("www.") board_allow = allow allow = lambda h: board_allow(h) or h.removeprefix("www.") == bare # noqa: E731 sent = {"User-Agent": USER_AGENT, "Accept": "application/json, text/xml"} | (headers or {}) deadline = time.monotonic() + DEADLINE current = url for _hop in range(4): parsed = httpx.URL(current) if parsed.scheme != "https" or not allow(parsed.host): raise _Refused(f"{current} is not an allowed host for {host}") with client.stream( method, current, timeout=TIMEOUTS, follow_redirects=False, headers=sent, json=(payload or {}) if method == "POST" else None, ) as response: if response.is_redirect: location = response.headers.get("location") or "" if not location: raise ValueError("redirect with no location") current = str(httpx.URL(current).join(location)) continue response.raise_for_status() chunks: list[bytes] = [] size = 0 for chunk in response.iter_bytes(): size += len(chunk) if size > MAX_BYTES: raise _TooLarge(f"response too large (over {MAX_BYTES} bytes)") if time.monotonic() > deadline: raise httpx.ReadTimeout(f"no whole answer within {DEADLINE:.0f} s", request=response.request) chunks.append(chunk) return b"".join(chunks) raise ValueError("too many redirects") def _getter(client: httpx.Client, host: str, slug: str) -> Get: """The `get` a host module is handed: `_get` bound to one board. `_get` is looked up when a request is made, not when this is built, so a test that replaces `ats._get` reaches every host; `payload` and `headers` are passed on only when given, which keeps a replacement that predates them working.""" def get(url: str, *, method: str = "GET", payload: dict[str, Any] | None = None, headers: dict[str, str] | None = None) -> bytes: extra: dict[str, Any] = {} if payload is not None: extra["payload"] = payload if headers: extra["headers"] = headers return _get(client, url, host, slug, method=method, **extra) return get #: host -> parser, for the hosts whose module parses a whole body. Every one #: takes the raw response body and the slug. _PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = { name: module.parse for name, module in atshosts.HOSTS.items() if hasattr(module, "parse")} # --------------------------------------------------------- the classifier #: The fixed set of functions, and the keyword rules that map a title #: (preferred) or a department to one of them. Ported from `cmd/seedgen/ #: main.go`'s `rules` — first match wins, and "product" is genuinely listed #: twice: the first, narrower entry is checked before #: recruiting/design/marketing/support/sales/data, and the second, which also #: matches the bare word "product", is checked only after all of those, so a #: title like "Data Product Manager" keeps classifying as data/product by the #: same rule order the Go tool used. #: #: Not a straight port any more: the Go tool matched substrings, which let #: " it" (meant to catch "IT" as a department) match inside "Italy" and #: "Italia" — a European department like "Commercial Italy" filed as #: engineering. `_classify_text` now matches on word boundaries. A handful of #: short tokens (`_WHOLE_WORD`) need a boundary on *both* sides, because they #: are themselves the start of an unrelated word ("it" of "Italy", "seo" of #: "Seoul", "qa" of "Qatar"); every other keyword keeps a leading boundary #: only; most of them are deliberate stems ("manag", "engineer", "recruit") #: meant to also catch "management", "engineering", "recruiting", and a #: trailing boundary on those would stop them matching the plural and #: gerund forms real postings use ("Software Developers", "Account #: Managers"). FUNCTIONS = ("engineering", "data", "product", "design", "sales", "marketing", "customer success/support", "recruiting/people", "finance/legal/ops") _RULES: tuple[tuple[str, tuple[str, ...]], ...] = ( ("product", ("product manag", "product management", "product operations", "program manag", "technical program")), ("recruiting/people", ("recruit", "talent", "people ops", "people &", "people team", "people operations", "human resources", "hr", "hrbp")), ("design", ("design", "ux", "user experience", "creative")), ("marketing", ("marketing", "growth", "content", "communications", "brand", "seo", "social media", "demand gen", "events")), ("customer success/support", ("customer success", "customer support", "customer experience", "customer service", "support", "success", "implementation", "onboarding", "professional services", "solutions consult", "technical account", "customer engineer", "deployment")), ("sales", ("sales", "account executive", "account manager", "account director", "business development", "bdr", "sdr", "revenue", "partnership", "go-to-market", "gtm", "solutions engineer", "solution engineer", "pre-sales", "presales", "channel", # "Commercial" is what French, Italian and Spanish org charts # call the sales function ("Commercial Italy" is a sales # department; see the docstring above), and the rest are the # sales words of the seed's other languages that "sales" # itself does not already cover (German, Dutch, Danish, # Norwegian, Swedish, French) — a modest set, not a full port; # `extract` is the rest. "commercial", "vertrieb", "verkauf", "verkoop", "salg", "försäljning", "vente")), ("data", ("data", "analytics", "machine learning", "ml", "applied science", "research scientist", "statistic")), ("product", ("product manag", "product management", "program manag", "product operations", "technical program", "product")), ("finance/legal/ops", ("finance", "financ", "accounting", "legal", "counsel", "compliance", "operations", "ops", "administrat", "office", "procurement", "strategy", "corporate", "facilities", "risk", "audit", "tax", "treasury", "payroll", "executive assistant", "chief of staff", "workplace", "supply chain", "logistics", "fraud", "trust & safety", "trust and safety", "policy")), ("engineering", ("engineer", "software", "developer", "infrastructure", "platform", "devops", "sre", "site reliability", "security", "information technology", "it", "technology", "technical staff", "hardware", "qa", "quality assurance", "architect", "research", "science", "scientist", "systems", "network", "cloud", "mobile", "frontend", "backend", "full stack", "fullstack", "web", # The developer/engineer words of the seed's other # languages, same reasoning as sales' above. "entwickler", "ingenieur", "ontwikkelaar", "udvikler", "utvikler", "utvecklare", "ingenjör", "ingeniør", "développeur", "developpeur", "ingénieur")), ) #: Keywords that must match a whole word, not just its start: each one is #: itself the first letters of an unrelated word that would otherwise false- #: match ("it" of "Italy", "seo" of "Seoul", "qa" of "Qatar"). Every keyword #: not listed here keeps the old prefix behaviour on purpose — see the #: `_RULES` docstring above. _WHOLE_WORD = frozenset({ "it", "hr", "hrbp", "ux", "ml", "qa", "seo", "web", "tax", "ops", "sre", "bdr", "sdr", "gtm", }) def _keyword_pattern(keyword: str) -> re.Pattern[str]: trailing = r"\b" if keyword in _WHOLE_WORD else "" return re.compile(r"\b" + re.escape(keyword) + trailing, re.IGNORECASE) #: `_RULES`, compiled once: `(function, patterns)` in the same order. _COMPILED_RULES: tuple[tuple[str, tuple[re.Pattern[str], ...]], ...] = tuple( (function, tuple(_keyword_pattern(keyword) for keyword in keywords)) for function, keywords in _RULES) def _classify_text(text: str) -> str: stripped = (text or "").strip() if not stripped: return "" for function, patterns in _COMPILED_RULES: if any(pattern.search(stripped) for pattern in patterns): return function return "" def classify(department: str, title: str) -> str: """A title (preferred) or a department, to one of `FUNCTIONS`, or `''` when neither names one — `''` and not `"other"`, which would claim a knowledge the classifier does not have (`extract` may still fill it from reading the posting; see `Store.set_extracted`). Title before department: a department is often a generic umbrella ("Operations", "Engineering & Product") that says less about the role than its title does. A hand-checked sample of 60 roles where the two orders disagreed (docs/decisions.md) found title-first right about twice as often as department-first, mostly because a specific title beats a broad department name, so title now wins; department is the fallback when the title matches nothing.""" return _classify_text(title or "") or _classify_text(department or "") or "" def is_known_target(target: str) -> bool: """`host:slug`, with `host` one of `atshosts.HOSTS` and a slug its module's `slug_ok` accepts. What the target validator refuses: `app companies add` and the pull job both call this before touching the network.""" host, sep, slug = target.partition(":") module = atshosts.HOSTS.get(host) return bool(sep) and module is not None and bool(slug.strip()) and module.slug_ok(slug) def careers_url(target: str) -> str: """The company's own careers page for `host:slug`, or "" for an unknown host. What `app companies add` stores alongside the board.""" host, _, slug = target.partition(":") module = atshosts.HOSTS.get(host) return module.careers_url(slug) if module is not None else "" #: Pages read from one paged board at most, and the pause between two of #: them. Workable answers ten roles a page, SmartRecruiters a hundred and #: Teamtailor fifty, so the cap is 1,000, 10,000 and 5,000 roles; the pause #: keeps Workable, which answers a burst with 429, from seeing one. The same #: pause separates two detail reads. MAX_PAGES = 100 PAGE_DELAY = 0.5 def _paged(client: httpx.Client, target: str) -> tuple[list[Row], bool]: """Every page of a board whose host pages, `PAGE_DELAY` apart, until the host has no next page, the stated total is reached, or `MAX_PAGES`. Returns the rows and whether the board is complete: as many distinct results read as the host's stated total, or, for a host that states none (Teamtailor), the read ended on an empty page (a page with nothing new in it ends the read short of that, incomplete, so an offset the host ignores cannot loop). Counted on the raw results, so a row the parser drops does not make a board incomplete, and on distinct ids, so a page that shifted under an offset and repeated a role does not make it complete. A 429 after the first page stops the read with what it has, incomplete; one on the first page raises like any other failure.""" host, _, slug = target.partition(":") module = atshosts.HOSTS[host] get = _getter(client, host, slug) rows: dict[str, Row] = {} seen: set[str] = set() total: Any = None cursor: Any = None for page in range(MAX_PAGES): if page: time.sleep(PAGE_DELAY) try: data, results, total, cursor = module.page(get, slug, cursor) except httpx.HTTPStatusError as exc: if not page or exc.response.status_code != 429: raise log.warning("ats: %s: 429 on page %d; kept %d of %s, not complete", target, page + 1, len(seen), total) return list(rows.values()), False before = len(seen) seen |= {str(r.get("id") or r.get("shortcode") or "") for r in results} for row in module.rows(data, slug): rows.setdefault(row["source_id"], row) if (cursor is None or (isinstance(total, int) and len(seen) >= total) or (total is None and len(seen) == before)): break complete = len(seen) == total if isinstance(total, int) else total is None and cursor is None if not complete: log.warning("ats: %s: read %d of %s stated; not complete", target, len(seen), "an unstated total" if total is None else total) return list(rows.values()), complete def _fetch_rows(client: httpx.Client, target: str) -> tuple[list[Row], bool]: """One board's listing, fetched and parsed: its rows and whether they are the whole board. Raises on any failure — refused host, timeout, a feed shape the host's parser does not understand. A host that does not page answers with the whole board, so it is complete whenever it parses; a paged one is complete when `_paged` says so. A body that exceeds `MAX_BYTES` gets one retry at the host's `fallback_url` where it has one (Greenhouse, without `?content=true`): the description is lost, but the titles land rather than the whole board being skipped.""" if not is_known_target(target): raise ValueError(f"{target!r} is not host:slug for a known ATS host") host, _, slug = target.partition(":") module = atshosts.HOSTS[host] if hasattr(module, "page"): return _paged(client, target) get = _getter(client, host, slug) try: body = get(module.feed_url(slug)) except _TooLarge: if not hasattr(module, "fallback_url"): raise log.warning("ats: %s: body exceeded %d bytes; retrying at the smaller feed", target, MAX_BYTES) body = get(module.fallback_url(slug)) return module.parse(body, slug), True # -------------------------------------------------------------- robots.txt #: How long a robots.txt answer holds for one board before it is read again. ROBOTS_TTL = timedelta(days=1) #: This app's product token, the name a robots.txt group addresses it by: #: the `desk` of `desk/ (personal aggregator)`. ROBOTS_TOKEN = USER_AGENT.split("/", 1)[0].strip().lower() #: One rule of a group: `(allow, path pattern)`. _Rule = tuple[bool, str] def _robots_groups(text: str) -> list[tuple[set[str], list[_Rule]]]: """The groups of a robots.txt, per RFC 9309: one or more `user-agent` lines (their tokens lowercased) and the `allow` and `disallow` rules that follow them, up to the next `user-agent` line after a rule. Comments, other keys and rules before any `user-agent` line are left out, and so is an empty rule, which allows everything and so says nothing.""" groups: list[tuple[set[str], list[_Rule]]] = [] in_agents = False for line in text.splitlines(): key, sep, value = line.split("#", 1)[0].partition(":") key, value = key.strip().lower(), value.strip() if not sep: continue if key == "user-agent": if not in_agents: groups.append((set(), [])) in_agents = True groups[-1][0].add(value.lower()) elif key in ("allow", "disallow") and groups: in_agents = False if value: groups[-1][1].append((key == "allow", value)) return groups def _robots_rules(groups: list[tuple[set[str], list[_Rule]]], token: str) -> list[_Rule]: """The rules for `token`: every group that names it (matched whole and ignoring case, as RFC 9309 says, not as a substring), merged; else every `*` group, merged; else none, which allows everything.""" for wanted in (token, "*"): rules = [rule for agents, group in groups if wanted in agents for rule in group] if rules or any(wanted in agents for agents, _group in groups): return rules return [] def _robots_pattern(path: str) -> re.Pattern[str]: """A rule's path as a regex anchored at the start: `*` is any run of characters, and a `$` at the end anchors the end.""" anchored = path.endswith("$") body = ".*".join(re.escape(part) for part in path.rstrip("$").split("*")) return re.compile(body + (r"\Z" if anchored else ""), re.DOTALL) def _robots_verdict(rules: list[_Rule], path: str) -> bool: """Whether `path` is allowed by `rules`: the longest matching rule wins, an allow wins a tie, and no match allows. Both sides are compared percent-decoded, so `%7E` and `~` are one character.""" path = unquote(path) best: tuple[int, bool] | None = None for allow, pattern in rules: if _robots_pattern(unquote(pattern)).match(path): weight = (len(pattern), allow) best = weight if best is None or weight > best else best return best is None or best[1] def _robots_allows(body: bytes, page: str) -> bool: """Whether a robots.txt lets `page` be fetched, by RFC 9309: by its rules for every agent (`*`), and by those it has for this app's own token (`ROBOTS_TOKEN`, which fall back to `*`), so a site that shuts out this app by name is honoured as well. Paths are case-sensitive; a byte order mark is dropped.""" groups = _robots_groups(body.decode("utf-8-sig", "replace")) url = httpx.URL(page) path = (url.raw_path or b"/").decode("ascii", "replace") return all(_robots_verdict(_robots_rules(groups, token), path) for token in ("*", ROBOTS_TOKEN)) def _check_robots(client: httpx.Client, target: str, state: MutableMapping[str, str] | None = None) -> None: """Raise `_Disallowed` when the site of a host with a `robots_page` has a robots.txt that disallows that page, and `_RobotsUnreachable` when it cannot be read. Read through `_get`, so from the board's own allowed hosts only, except that a redirect between the host and its `www.` twin is followed (at most three hops, https only); any other redirect off the site is unreachable. With `state`, the verdict and when it was reached are kept under `ats:robots:` and held for `ROBOTS_TTL`. By RFC 9309: a 4xx other than 429 means there is no robots.txt to obey, which allows, and is kept like any other verdict. A 5xx, a 429, a timeout or a connection error means the file is unreachable, which the RFC says to treat as disallowing everything: the board is not read on this pull, fails with "robots.txt could not be read", and nothing is kept, so the next pull asks again. A 429 counts as unreachable although it is a 4xx, since it says "not now" rather than "there is none".""" host, _, slug = target.partition(":") module = atshosts.HOSTS.get(host) if module is None or not hasattr(module, "robots_page") or not is_known_target(target): return page = module.robots_page(slug) key = f"ats:robots:{target}" now = datetime.now(UTC) if state is not None: try: kept = json.loads(state.get(key) or "{}") fresh = now - datetime.fromisoformat(kept["checked"]) < ROBOTS_TTL except (ValueError, KeyError, TypeError): kept, fresh = {}, False if fresh: if not kept.get("allowed", True): raise _Disallowed(f"robots.txt disallows {page}") return url = f"https://{httpx.URL(page).host}/robots.txt" try: body = _get(client, url, host, slug, headers={"Accept": "text/plain"}, www_twin=True) except httpx.HTTPStatusError as exc: code = exc.response.status_code if not 400 <= code < 500 or code == 429: raise _RobotsUnreachable(f"robots.txt answered {code}") from exc allowed = True except Exception as exc: raise _RobotsUnreachable(f"robots.txt not read: {exc}") from exc else: allowed = _robots_allows(body, page) if state is not None: state[key] = json.dumps({"checked": now.isoformat(timespec="seconds"), "allowed": allowed}) if not allowed: raise _Disallowed(f"robots.txt disallows {page}") # ---------------------------------------------------------------- details #: Roles read in detail from one board in one pull at most. The rest of a #: large new board is read over the next pulls, 200 at a time. DETAIL_CAP = 200 def _read_details(client: httpx.Client, target: str, rows: list[Row], complete: bool, detailed: set[str]) -> tuple[list[Row], set[str]]: """The rows of a board whose host has `detail`, each role not yet in `detailed` (compound ids) filled from its own page, at most `DETAIL_CAP` of them and `PAGE_DELAY` apart. Returns the rows and the ids read in detail so far, for `ats:detailed:`. Every row not read in detail in this call goes out with an empty description: a role read on an earlier pull keeps the one it has, since `upsert_role` never overwrites a filled column with an empty value, and a teaser from the listing must not replace it. A detail that raises is logged and the role left for the next pull; a 429 stops the reads for this board until then. None of this touches completeness, which is the listing's alone. The ids of roles no longer listed are dropped, but only after a complete listing, so one cut short does not make the next pull read its missing roles again.""" host, _, slug = target.partition(":") module = atshosts.HOSTS[host] get = _getter(client, host, slug) done = set(detailed) out: list[Row] = [] tried = 0 stopped = False for row in rows: compound = f"{host}:{slug}:{row['source_id']}" if compound in done or stopped or tried >= DETAIL_CAP: out.append(row | {"description": ""}) continue time.sleep(PAGE_DELAY) tried += 1 try: filled = module.detail(get, slug, row) except Exception as exc: if isinstance(exc, httpx.HTTPStatusError) and exc.response.status_code == 429: log.warning("ats: %s: 429 on a detail read; the rest wait for the next pull", target) stopped = True else: log.warning("ats: %s: detail of %s failed (%s)", target, row["source_id"], exc) out.append(row | {"description": ""}) continue out.append(row | filled | {"source_id": row["source_id"]}) done.add(compound) if complete: done &= {f"{host}:{slug}:{row['source_id']}" for row in rows} return out, done def fetch_board(target: str) -> tuple[list[Row], bool]: """Fetch and parse one board directly: no `Source`, no `self.state`, just "does this board answer, what does it list, and is that all of it". For `app companies add|check`, which ask that question about one board at a time outside any pull. The listing only, with no detail reads, and robots.txt read afresh. Raises the same way `_fetch_rows` does, and `_Disallowed` or `_RobotsUnreachable` for a board robots.txt shuts out or that cannot be checked.""" with httpx.Client() as client: _check_robots(client, target) return _fetch_rows(client, target) def failure_reason(exc: BaseException) -> str: """One short line for why a board did not answer: the status of an HTTP error, else the exception's class and the first line of its message.""" if isinstance(exc, _Disallowed): return "robots.txt disallows it" if isinstance(exc, _RobotsUnreachable): return "robots.txt could not be read" if isinstance(exc, httpx.HTTPStatusError): return f"{exc.response.status_code} {exc.response.reason_phrase}".strip() lines = str(exc).strip().splitlines() text = f"{type(exc).__name__}: {lines[0]}" if lines else type(exc).__name__ return text[:200] # -------------------------------------------------------------- the plugin class ATSSource(Source): name = "ats" description = f"Company job boards: {', '.join(HOSTS.values())}." #: The raw entries under `sources.ats.boards` are `{"url": target, "label": #: company}` — `url` carries the `host:slug` target and `label` the #: company name, which is how `Source.targets()`'s generic dict reading #: hands both back without a plugin-specific override. subscription_field = "boards" spec = PluginSpec( api_version=API_VERSION, id=name, version="1", label="ATS boards", targets=TargetSpec(mode="editable", kind="board", defaults=[]), profiles={"default": Profile( kind="role", tasks=Tasks( summary=SummaryPolicy(enabled=False), tagging=TaggingPolicy(enabled=False), ), presentation=ARTICLE_READER, )}, ) def fetch(self) -> Iterator[Item]: with httpx.Client() as client: # Handed in by the batch that runs this plugin, as the marker of # the pull run that queued it (see `app/jobs/pull.py`), and never # read from state, where a later run may have replaced it. A # plugin run on its own — a test, a dry run — has no run to stamp # itself against, so it makes one up. run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat( timespec="seconds") for target, label in self.targets(): host, _, slug = target.partition(":") failing = f"ats:failing:{target}" try: _check_robots(client, target, self.state) rows, complete = _fetch_rows(client, target) except Exception as exc: log.warning("ats: %s failed (%s)", target, exc) # The first failure in a row sets `since`, later ones only # the reason; `app.jobs.pull` closes the board's roles # once `since` is `STALE_DAYS` old. try: since = json.loads(self.state.get(failing, "") or "{}").get("since") except ValueError: since = None self.state[failing] = json.dumps({ "since": since or datetime.now(UTC).isoformat(timespec="seconds"), "reason": failure_reason(exc)}) continue self.state.pop(failing, None) detailed_key = f"ats:detailed:{target}" detailed: set[str] | None = None if hasattr(atshosts.HOSTS.get(host), "detail"): try: detailed = set(json.loads(self.state.get(detailed_key) or "[]")) except (ValueError, TypeError): detailed = set() rows, detailed = _read_details(client, target, rows, complete, detailed) seen_ids = [] for row in rows: # The compound id, matching `roles.source_id` — what # `Store.mark_closed` compares against, not the board's own # bare job id. compound_id = f"{host}:{slug}:{row['source_id']}" seen_ids.append(compound_id) yield Item( source_id=compound_id, location=row["url"], title=row["title"], excerpt=(row["description"] or "")[:300], body=row["description"], published_at=row["posted"], subscription=label, kind="role", raw={ "company": label, "place": row["location"], "department": row["department"], "function": classify(row["department"], row["title"]), "remote": row["remote"], # Every parser sets salary_text, empty or not (see # "salary_text" in `atshosts/_shared.py`); only # the hosts that state structured pay also set the # normalised figures, so the rest leave them at # row.get's default of no figure. "salary_text": row.get("salary_text", ""), "salary_min": row.get("salary_min"), "salary_max": row.get("salary_max"), "currency": row.get("currency", ""), "board": host, "slug": slug, }, ) # Stamped only once the board has parsed without raising, so a # board that blew up half way through is neither marked ok nor # has a partial id list treated as the whole of it. `complete` # says whether the ids are the whole board, which the pull # needs before it closes what they leave out. The ids read in # detail are kept here too, after every row went out, so a # pull stopped half way reads the rest again rather than # losing their descriptions. self.state[f"ats:ok:{target}"] = run_marker self.state[f"ats:complete:{target}"] = "1" if complete else "" self.state[f"ats:ids:{target}"] = json.dumps(seen_ids) if detailed is not None: self.state[detailed_key] = json.dumps(sorted(detailed))