app/jobs/reach.py derives each board's reach once per pull and the pull hands it to the runner through the batch payload. SmartRecruiters, Eightfold and SuccessFactors filter one country a request, Phenom a list, Workday and Oracle by host ids from facets. Remote beside countries is a series of its own, since two facets intersect. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
414 lines
20 KiB
Python
414 lines
20 KiB
Python
"""ATS boards: a company's own careers page on one of the hosts in
|
|
`plugins/readers/`, one module per host.
|
|
|
|
Every one of these hosts serves a company's own list of roles as public,
|
|
unauthenticated JSON, XML or RSS, and a role that drops off the board is
|
|
*closed*, which is the cheapest staleness check there is. What a host says
|
|
and how its feed reads is its module's, what every reader shares is
|
|
`plugins/readers/_runner.py`'s; this file is the `ats` Source over the runner,
|
|
the classifier, and the names the app and the tests reach for on `ats`.
|
|
|
|
The allow-list and the classifier are ported from `openings.go` and
|
|
`cmd/seedgen/main.go` in
|
|
[jobhunt-hq](https://github.com/rembrandtreyes/jobhunt-hq) (MIT), including
|
|
`classify`'s nine functions and their keywords.
|
|
|
|
A target is `host:slug`, e.g. `greenhouse:databricks`, `ashby:deepl`,
|
|
`lever:eu/tomtom`, `teamtailor:careers.acme.com`; each host module says what
|
|
its slug is. Every board is read independently; one that fails is logged,
|
|
recorded as failing under `ats:failing:<target>`, and skipped. A board's
|
|
filter is the `reach` option the pull hands it (the agents', see
|
|
`app/jobs/reach.py`), over which its subscription `params` (`{"countries":
|
|
["DK"], "remote": true}`, `{"reach": "all"}`) win, narrowed to what its
|
|
reader declares (`_runner.ask_for`). An unfiltered read records the board's
|
|
size under `ats:size:<target>`, which the reach reads to leave small boards
|
|
whole.
|
|
|
|
The limits below are read when a board is read, so a test that replaces one
|
|
on this module reaches the runner.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import re
|
|
import sys
|
|
from collections.abc import Callable, Iterator
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
from desk.sources import (
|
|
API_VERSION, ARTICLE_READER, Item, PluginSpec, Profile, Source,
|
|
SummaryPolicy, TaggingPolicy, TargetSpec, Tasks, USER_AGENT, # noqa: F401
|
|
)
|
|
|
|
try:
|
|
from plugins import readers
|
|
except ModuleNotFoundError:
|
|
# `desk.sources.registry` loads this file by its path, under a name of its
|
|
# own, and the app puts the repo root on `sys.path` before it does; a
|
|
# registry run without the app (a script, another working directory)
|
|
# does not, and the host modules are found as the package beside this
|
|
# file. So the root goes on the path then, and only then.
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
from plugins import readers
|
|
from plugins.readers import _runner
|
|
from plugins.readers._shared import Get, Row
|
|
|
|
# The names the parsers had here before they moved to their host modules,
|
|
# kept for the callers and the tests that still reach for them on `ats`.
|
|
from plugins.readers._shared import ( # noqa: F401
|
|
Page, is_web_url as _is_web_url, parse_dt as _parse_dt, yearly_amount as _yearly_amount,
|
|
)
|
|
from plugins.readers.greenhouse import _salary as _greenhouse_salary # noqa: F401
|
|
from plugins.readers.teamtailor import parse as _parse_teamtailor # noqa: F401
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
#: The runner's limits, kept here as the `ats` Source's own: see `_runner`.
|
|
MAX_BYTES = _runner.MAX_BYTES
|
|
TIMEOUTS = _runner.TIMEOUTS
|
|
DEADLINE = _runner.DEADLINE
|
|
MAX_PAGES = _runner.MAX_PAGES
|
|
PAGE_DELAY = _runner.PAGE_DELAY
|
|
DETAIL_CAP = _runner.DETAIL_CAP
|
|
DETAIL_FIELDS = _runner.DETAIL_FIELDS
|
|
|
|
#: The runner's errors, under the names they had here.
|
|
_Refused = _runner.Refused
|
|
_TooLarge = _runner.TooLarge
|
|
_local_host = _runner.local_host
|
|
|
|
#: host -> the name the host gives itself: the hosts this plugin reads, in
|
|
#: the order a select offers them. What the Sources page's host select is
|
|
#: rendered from, so it cannot offer a host `is_known_target` refuses.
|
|
HOSTS: dict[str, str] = {name: module.LABEL for name, module in readers.HOSTS.items()}
|
|
|
|
#: host -> parser, for the hosts whose module parses a whole body.
|
|
_PARSERS: dict[str, Callable[[bytes, str], list[Row]]] = {
|
|
name: module.parse for name, module in readers.HOSTS.items() if hasattr(module, "parse")}
|
|
|
|
|
|
def _limits() -> _runner.Limits:
|
|
return _runner.Limits(max_pages=MAX_PAGES, page_delay=PAGE_DELAY, detail_cap=DETAIL_CAP)
|
|
|
|
|
|
def _allowed_host(host: str, slug: str) -> Callable[[str], bool]:
|
|
"""The hosts a request or a redirect may land on for one board: what its
|
|
host module's `allowed` says. Anything else is refused, at the first hop
|
|
and at every redirect."""
|
|
module = readers.HOSTS.get(host)
|
|
return module.allowed(slug) if module is not None else lambda h: False
|
|
|
|
|
|
def _get(client: httpx.Client, url: str, host: str, slug: str, *,
|
|
method: str = "GET", payload: dict[str, Any] | None = None,
|
|
headers: dict[str, str] | None = None) -> bytes:
|
|
"""`_runner.fetch` for one board, with this module's `MAX_BYTES` and
|
|
`DEADLINE`."""
|
|
return _runner.fetch(client, url, _allowed_host(host, slug), method=method,
|
|
payload=payload, headers=headers, max_bytes=MAX_BYTES,
|
|
deadline=DEADLINE, name=host)
|
|
|
|
|
|
def _getter(client: httpx.Client, host: str, slug: str) -> Get:
|
|
"""The `get` a host module is handed: `_get` bound to one board. `_get`
|
|
is looked up when a request is made, so a test that replaces `ats._get`
|
|
reaches every host; `payload` and `headers` are passed on only when
|
|
given, which keeps a replacement that predates them working."""
|
|
def get(url: str, *, method: str = "GET", payload: dict[str, Any] | None = None,
|
|
headers: dict[str, str] | None = None) -> bytes:
|
|
extra: dict[str, Any] = {}
|
|
if payload is not None:
|
|
extra["payload"] = payload
|
|
if headers:
|
|
extra["headers"] = headers
|
|
return _get(client, url, host, slug, method=method, **extra)
|
|
return get
|
|
|
|
|
|
# --------------------------------------------------------- the classifier
|
|
|
|
#: The fixed set of functions, and the keyword rules that map a title
|
|
#: (preferred) or a department to one of them. Ported from `cmd/seedgen/
|
|
#: main.go`'s `rules` — first match wins, and "product" is genuinely listed
|
|
#: twice: the first, narrower entry is checked before
|
|
#: recruiting/design/marketing/support/sales/data, and the second, which also
|
|
#: matches the bare word "product", is checked only after all of those, so a
|
|
#: title like "Data Product Manager" keeps classifying as data/product by the
|
|
#: same rule order the Go tool used.
|
|
#:
|
|
#: Not a straight port any more: the Go tool matched substrings, which let
|
|
#: " it" (meant to catch "IT" as a department) match inside "Italy" and
|
|
#: "Italia" — a European department like "Commercial Italy" filed as
|
|
#: engineering. `_classify_text` now matches on word boundaries. A handful of
|
|
#: short tokens (`_WHOLE_WORD`) need a boundary on *both* sides, because they
|
|
#: are themselves the start of an unrelated word ("it" of "Italy", "seo" of
|
|
#: "Seoul", "qa" of "Qatar"); every other keyword keeps a leading boundary
|
|
#: only; most of them are deliberate stems ("manag", "engineer", "recruit")
|
|
#: meant to also catch "management", "engineering", "recruiting", and a
|
|
#: trailing boundary on those would stop them matching the plural and
|
|
#: gerund forms real postings use ("Software Developers", "Account
|
|
#: Managers").
|
|
FUNCTIONS = ("engineering", "data", "product", "design", "sales", "marketing",
|
|
"customer success/support", "recruiting/people", "finance/legal/ops")
|
|
|
|
_RULES: tuple[tuple[str, tuple[str, ...]], ...] = (
|
|
("product", ("product manag", "product management", "product operations",
|
|
"program manag", "technical program")),
|
|
("recruiting/people", ("recruit", "talent", "people ops", "people &",
|
|
"people team", "people operations", "human resources",
|
|
"hr", "hrbp")),
|
|
("design", ("design", "ux", "user experience", "creative")),
|
|
("marketing", ("marketing", "growth", "content", "communications", "brand",
|
|
"seo", "social media", "demand gen", "events")),
|
|
("customer success/support", ("customer success", "customer support",
|
|
"customer experience", "customer service",
|
|
"support", "success", "implementation",
|
|
"onboarding", "professional services",
|
|
"solutions consult", "technical account",
|
|
"customer engineer", "deployment")),
|
|
("sales", ("sales", "account executive", "account manager", "account director",
|
|
"business development", "bdr", "sdr", "revenue", "partnership",
|
|
"go-to-market", "gtm", "solutions engineer", "solution engineer",
|
|
"pre-sales", "presales", "channel",
|
|
# "Commercial" is what French, Italian and Spanish org charts
|
|
# call the sales function ("Commercial Italy" is a sales
|
|
# department; see the docstring above), and the rest are the
|
|
# sales words of the seed's other languages that "sales"
|
|
# itself does not already cover (German, Dutch, Danish,
|
|
# Norwegian, Swedish, French) — a modest set, not a full port;
|
|
# `extract` is the rest.
|
|
"commercial", "vertrieb", "verkauf", "verkoop", "salg",
|
|
"försäljning", "vente")),
|
|
("data", ("data", "analytics", "machine learning", "ml",
|
|
"applied science", "research scientist", "statistic")),
|
|
("product", ("product manag", "product management", "program manag",
|
|
"product operations", "technical program", "product")),
|
|
("finance/legal/ops", ("finance", "financ", "accounting", "legal", "counsel",
|
|
"compliance", "operations", "ops", "administrat",
|
|
"office", "procurement", "strategy", "corporate",
|
|
"facilities", "risk", "audit", "tax", "treasury",
|
|
"payroll", "executive assistant", "chief of staff",
|
|
"workplace", "supply chain", "logistics", "fraud",
|
|
"trust & safety", "trust and safety", "policy")),
|
|
("engineering", ("engineer", "software", "developer", "infrastructure",
|
|
"platform", "devops", "sre", "site reliability", "security",
|
|
"information technology", "it", "technology",
|
|
"technical staff", "hardware", "qa", "quality assurance",
|
|
"architect", "research", "science", "scientist", "systems",
|
|
"network", "cloud", "mobile", "frontend", "backend",
|
|
"full stack", "fullstack", "web",
|
|
# The developer/engineer words of the seed's other
|
|
# languages, same reasoning as sales' above.
|
|
"entwickler", "ingenieur", "ontwikkelaar", "udvikler",
|
|
"utvikler", "utvecklare", "ingenjör", "ingeniør",
|
|
"développeur", "developpeur", "ingénieur")),
|
|
)
|
|
|
|
#: Keywords that must match a whole word, not just its start: each one is
|
|
#: itself the first letters of an unrelated word that would otherwise false-
|
|
#: match ("it" of "Italy", "seo" of "Seoul", "qa" of "Qatar"). Every keyword
|
|
#: not listed here keeps the old prefix behaviour on purpose — see the
|
|
#: `_RULES` docstring above.
|
|
_WHOLE_WORD = frozenset({
|
|
"it", "hr", "hrbp", "ux", "ml", "qa", "seo", "web", "tax", "ops",
|
|
"sre", "bdr", "sdr", "gtm",
|
|
})
|
|
|
|
|
|
def _keyword_pattern(keyword: str) -> re.Pattern[str]:
|
|
trailing = r"\b" if keyword in _WHOLE_WORD else ""
|
|
return re.compile(r"\b" + re.escape(keyword) + trailing, re.IGNORECASE)
|
|
|
|
|
|
#: `_RULES`, compiled once: `(function, patterns)` in the same order.
|
|
_COMPILED_RULES: tuple[tuple[str, tuple[re.Pattern[str], ...]], ...] = tuple(
|
|
(function, tuple(_keyword_pattern(keyword) for keyword in keywords))
|
|
for function, keywords in _RULES)
|
|
|
|
|
|
def _classify_text(text: str) -> str:
|
|
stripped = (text or "").strip()
|
|
if not stripped:
|
|
return ""
|
|
for function, patterns in _COMPILED_RULES:
|
|
if any(pattern.search(stripped) for pattern in patterns):
|
|
return function
|
|
return ""
|
|
|
|
|
|
def classify(department: str, title: str) -> str:
|
|
"""A title (preferred) or a department, to one of `FUNCTIONS`, or `''`
|
|
when neither names one — `''` and not `"other"`, which would claim a
|
|
knowledge the classifier does not have (`extract` may still fill it from
|
|
reading the posting; see `Store.set_extracted`).
|
|
|
|
Title before department: a department is often a generic umbrella
|
|
("Operations", "Engineering & Product") that says less about the role
|
|
than its title does. A hand-checked sample of 60 roles where the two
|
|
orders disagreed (docs/decisions.md) found title-first right about twice
|
|
as often as department-first, mostly because a specific title beats a
|
|
broad department name, so title now wins; department is the fallback
|
|
when the title matches nothing."""
|
|
return _classify_text(title or "") or _classify_text(department or "") or ""
|
|
|
|
|
|
def is_known_target(target: str) -> bool:
|
|
"""`host:slug`, with `host` one of `readers.HOSTS` and a slug its module's
|
|
`slug_ok` accepts. What the target validator refuses: `app companies add`
|
|
and the pull job both call this before touching the network."""
|
|
host, sep, slug = target.partition(":")
|
|
module = readers.HOSTS.get(host)
|
|
return bool(sep) and module is not None and bool(slug.strip()) and module.slug_ok(slug)
|
|
|
|
|
|
def careers_url(target: str) -> str:
|
|
"""The company's own careers page for `host:slug`, or "" for an unknown
|
|
host. What `app companies add` stores alongside the board."""
|
|
host, _, slug = target.partition(":")
|
|
module = readers.HOSTS.get(host)
|
|
return module.careers_url(slug) if module is not None else ""
|
|
|
|
|
|
def _fetch_rows(client: httpx.Client, target: str,
|
|
ask: readers.Ask | None = None) -> tuple[list[Row], bool]:
|
|
"""One board's listing, read and parsed, unfiltered unless `ask` says:
|
|
its rows and whether they are the whole board. Raises on any failure.
|
|
No sweep and no state: what `fetch_board` and the host tests read."""
|
|
if not is_known_target(target):
|
|
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
|
host, _, slug = target.partition(":")
|
|
listing = _runner.read_listing(readers.HOSTS[host], slug, _getter(client, host, slug), ask,
|
|
limits=_limits())
|
|
return listing.rows, listing.complete
|
|
|
|
|
|
def _read_details(client: httpx.Client, target: str, rows: list[Row], complete: bool,
|
|
detailed: set[str], failed: set[str]
|
|
) -> tuple[list[Row], set[str], set[str]]:
|
|
"""`_runner.read_details` for one board, with `rows` as the whole listing
|
|
when `complete`."""
|
|
host, _, slug = target.partition(":")
|
|
listed = {f"{host}:{slug}:{row['source_id']}" for row in rows} if complete else None
|
|
return _runner.read_details(readers.HOSTS[host], host, slug, _getter(client, host, slug),
|
|
rows, detailed, failed, listed=listed, limits=_limits())
|
|
|
|
|
|
def fetch_board(target: str) -> tuple[list[Row], bool]:
|
|
"""Fetch and parse one board directly: no `Source`, no state, just "does
|
|
this board answer, what does it list, and is that all of it". For `app
|
|
companies add|check`. The listing only, unfiltered, no detail reads.
|
|
Raises the same way `_fetch_rows` does."""
|
|
with httpx.Client() as client:
|
|
return _fetch_rows(client, target)
|
|
|
|
|
|
failure_reason = _runner.failure_reason
|
|
|
|
|
|
# -------------------------------------------------------------- the plugin
|
|
|
|
class ATSSource(Source):
|
|
name = "ats"
|
|
description = f"Company job boards: {', '.join(HOSTS.values())}."
|
|
#: The raw entries under `sources.ats.boards` are `{"url": target, "label":
|
|
#: company, "params": {...}}`: `url` carries the `host:slug` target,
|
|
#: `label` the company name and `params` the board's filter.
|
|
subscription_field = "boards"
|
|
|
|
spec = PluginSpec(
|
|
api_version=API_VERSION, id=name, version="1", label="ATS boards",
|
|
targets=TargetSpec(mode="editable", kind="board", defaults=[]),
|
|
profiles={"default": Profile(
|
|
kind="role",
|
|
tasks=Tasks(
|
|
summary=SummaryPolicy(enabled=False),
|
|
tagging=TaggingPolicy(enabled=False),
|
|
),
|
|
presentation=ARTICLE_READER,
|
|
)},
|
|
)
|
|
|
|
def fetch(self) -> Iterator[Item]:
|
|
with httpx.Client() as client:
|
|
# Handed in by the batch that runs this plugin, as the marker of
|
|
# the pull run that queued it (see `app/jobs/pull.py`). A plugin
|
|
# run on its own (a test, a dry run) makes one up.
|
|
run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat(
|
|
timespec="seconds")
|
|
for target, label in self.targets():
|
|
yield from self._board(client, target, label, run_marker)
|
|
|
|
def _board(self, client: httpx.Client, target: str, label: str,
|
|
run_marker: str) -> Iterator[Item]:
|
|
host, _, slug = target.partition(":")
|
|
failing = f"ats:failing:{target}"
|
|
try:
|
|
if not is_known_target(target):
|
|
raise ValueError(f"{target!r} is not host:slug for a known ATS host")
|
|
module = readers.HOSTS[host]
|
|
get = _getter(client, host, slug)
|
|
ask = _runner.ask_for(module, self.params_for(target), self.options.get("reach"),
|
|
keys=self.options.get("keys"))
|
|
listing = _runner.read(self.state, "ats", target, module, slug, get, ask,
|
|
limits=_limits())
|
|
except Exception as exc:
|
|
log.warning("ats: %s failed (%s)", target, exc)
|
|
_runner.record_failure(self.state, failing, exc)
|
|
return
|
|
self.state.pop(failing, None)
|
|
|
|
def compound(source_id: str) -> str:
|
|
# Matches `roles.source_id`, which `Store.mark_closed` compares.
|
|
return f"{host}:{slug}:{source_id}"
|
|
|
|
rows = listing.rows
|
|
detail_keys = (f"ats:detailed:{target}", f"ats:detailfailed:{target}")
|
|
detail_sets: tuple[set[str], set[str]] | None = None
|
|
if hasattr(module, "detail"):
|
|
detailed, failed = (_runner.id_set(self.state, key) for key in detail_keys)
|
|
listed = {compound(i) for i in listing.ids} if listing.complete else None
|
|
rows, detailed, failed = _runner.read_details(
|
|
module, host, slug, get, rows, detailed, failed, listed=listed,
|
|
limits=_limits())
|
|
detail_sets = (detailed, failed)
|
|
|
|
for row in rows:
|
|
description = row.get("description") or ""
|
|
yield Item(
|
|
source_id=compound(row["source_id"]),
|
|
location=row["url"],
|
|
title=row["title"],
|
|
excerpt=description[:300],
|
|
body=description,
|
|
published_at=row.get("posted"),
|
|
subscription=label,
|
|
kind="role",
|
|
raw=_runner.raw_of(row) | {
|
|
"company": label,
|
|
"function": classify(row.get("department") or "", row["title"]),
|
|
"board": host,
|
|
"slug": slug,
|
|
},
|
|
)
|
|
# Stamped only once every row went out, so a board that blew up half
|
|
# way through is neither marked ok nor has a partial id list treated
|
|
# as the whole of it, and a pull stopped half way reads the rest of
|
|
# its details again.
|
|
_runner.stamp(self.state, "ats", target, run_marker, listing,
|
|
[compound(i) for i in listing.ids])
|
|
if not listing.filtered:
|
|
size = len(listing.ids)
|
|
if not listing.complete:
|
|
kept = self.state.get(f"ats:size:{target}", "")
|
|
size = max(size, int(kept) if kept.isdigit() else 0)
|
|
self.state[f"ats:size:{target}"] = str(size)
|
|
if detail_sets is not None:
|
|
for key, ids in zip(detail_keys, detail_sets, strict=True):
|
|
self.state[key] = json.dumps(sorted(ids))
|