"""What the host modules share: the row and page shapes, the fetch they are handed, and the date, URL and salary helpers more than one host needs. Kept apart from `plugins/ats.py` so that a host module and `ats.py` can both import it without either importing the other. The leading underscore keeps `plugins.atshosts` from listing it as a host. """ from __future__ import annotations import re from collections.abc import Callable from datetime import UTC, datetime from email.utils import parsedate_to_datetime from typing import Any, Protocol import httpx #: What every parser turns a feed into: one dict per row, with the keys #: `source_id`, `title`, `location`, `url`, `posted`, `department`, #: `description`, `remote` and `salary_text`, and, where a host states pay, #: `salary_min`, `salary_max` and `currency`. Row = dict[str, Any] #: A paged host's one page: `(decoded, results, total, cursor)`, the decoded #: page, its raw results (dicts with an `id`, counted for completeness), the #: total it states (None for a host that states none), and the cursor of the #: next page (None after the last). The first call gets cursor None. Page = tuple[Any, list[dict[str, Any]], Any, Any] class Get(Protocol): """The fetch a host is handed: `ats.py`'s own, bound to one board, so the allow-list, the redirect rule, the byte cap and the deadline apply to every request a host makes. A POST sends `payload` as JSON; `headers` are added to (and may replace) the default User-Agent and Accept. Returns the body, and raises `httpx.HTTPStatusError` on a non-2xx answer; see "Fetching" in the package's docstring for the rest.""" def __call__(self, url: str, *, method: str = "GET", payload: dict[str, Any] | None = None, headers: dict[str, str] | None = None) -> bytes: ... def is_web_url(url: str) -> bool: """A trimmed http(s) URL, or refuse — a feed is external content and a `javascript:` or `data:` link must never reach an href. Port of openings.go's `webURL`.""" url = (url or "").strip() if not url: return False parsed = httpx.URL(url) return parsed.scheme in ("http", "https") and bool(parsed.host) def only(host: str) -> Callable[[str], bool]: """The allow-list of a board that is fetched from one host name only, which is every host so far. A host name has no case, and a slug that names one (`Acme` of acme.recruitee.com) may be written with capitals.""" want = host.lower() return lambda h: h.lower() == want #: Matched whole (`fullmatch`, so not even a trailing newline slips past `$`). #: A slug that is one URL path segment: what the hosts that put the slug in #: the path accept, and nothing that could add a path, a query or a fragment. _SEGMENT = re.compile(r"[A-Za-z0-9._~%-]+") #: A slug that is one DNS label, for the hosts that put it in the host name. _LABEL = re.compile(r"[A-Za-z0-9-]+") def path_slug_ok(slug: str) -> bool: return bool(_SEGMENT.fullmatch(slug)) and slug not in (".", "..") def label_slug_ok(slug: str) -> bool: return bool(_LABEL.fullmatch(slug)) def parse_dt(value: Any) -> datetime | None: """An aware datetime from whatever a board sends: ISO-8601 with an offset, `Z`, a bare date, epoch milliseconds, "YYYY-MM-DD HH:MM:SS UTC", or an RSS date ("Tue, 29 Sep 2026 13:36:59 +0200"). None for anything else, which `Item` reads as "now".""" if value is None: return None if isinstance(value, int | float): if value <= 0: return None seconds = value / 1000 if value > 10**12 else value return datetime.fromtimestamp(seconds, tz=UTC) text = str(value).strip() if not text: return None try: parsed = datetime.fromisoformat(text.replace("Z", "+00:00")) except ValueError: match = re.match(r"^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})", text) if not match: try: parsed = parsedate_to_datetime(text) except (TypeError, ValueError): return None return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC) parsed = datetime.fromisoformat(f"{match.group(1)}T{match.group(2)}+00:00") return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC) # ---------------------------------------------------------- structured pay # # A board's own salary range, mapped to yearly `salary_min`/`salary_max` # (integers) and `currency` (ISO 4217) — the columns `upsert_role` unpacks # `raw` into, per docs/decisions.md's "raw is transport, columns are # storage". Only Ashby, Greenhouse, Lever, Recruitee and Personio give # structured pay on their list endpoint; Workable, SmartRecruiters and # Teamtailor do not (checked live, 2026-09-29), so their rows leave the # columns out and `ATSSource.fetch` reads a plain '' or None for them. def num(value: Any) -> float | None: """A positive number from whatever a board sends (int, float, or a numeral string), or None for anything else, including zero and a negative — a board's placeholder 0 (an unfilled field, not a real figure) must not become a salary floor of zero.""" try: number = float(value) except (TypeError, ValueError): return None return number if number > 0 else None def yearly_amount(min_value: Any, max_value: Any, currency: Any, interval: Any, factors: dict[str, int]) -> tuple[int | None, int | None, str]: """`(salary_min, salary_max, currency)` as yearly integers, or `(None, None, "")` when there is no currency, no bound, or `interval` is not one of `factors`' own keys — which is how an hourly or a daily rate is left out rather than annualised into a number nobody stated. `factors` maps the board's own interval spelling to the multiplier that makes it yearly: 1 for a figure already yearly, 12 for a monthly one.""" code = str(currency or "").strip().upper() factor = factors.get(str(interval or "").strip()) if not code or not factor: return None, None, "" lo, hi = num(min_value), num(max_value) if lo is None and hi is None: return None, None, "" return (int(lo * factor) if lo is not None else None, int(hi * factor) if hi is not None else None, code) # ------------------------------------------------------------- salary_text # # `salary_min`/`salary_max`/`currency` above are yearly only, on purpose: # hourly and daily rates, a Greenhouse role with several pay zones, and a # range with no currency are all dropped from those columns rather than # guessed into a wrong yearly figure. `salary_text` is where that dropped # information survives — one human-readable line per parser, written # whether or not the pay conformed, so a person can read it and do their own # sums and a stricter analysis can find the roles the columns left out # (`Role.salary_state`, `app/models.py`). #: Formats a number the way a salary line writes it: no decimals for a whole #: figure (the normal case for a yearly or monthly one), two for anything #: else (an hourly or daily rate quoted with cents). def fmt_amount(value: float) -> str: return f"{int(value):,}" if value == int(value) else f"{value:,.2f}" def bounds_words(lo: float | None, hi: float | None) -> str: """"70,000 to 90,000", or "from 70,000"/"up to 90,000" with only one bound stated, or "" with neither.""" if lo is not None and hi is not None: return fmt_amount(lo) if lo == hi else f"{fmt_amount(lo)} to {fmt_amount(hi)}" if lo is not None: return f"from {fmt_amount(lo)}" if hi is not None: return f"up to {fmt_amount(hi)}" return "" def pay_line(lo: float | None, hi: float | None, currency: Any, period: str) -> str: """One statement of pay as a human-readable line — "EUR 70,000 to 90,000 per year" — or "" when there is no bound to report at all. A currency that was not stated does not drop the line, unlike the normalised columns: it is said plainly instead, "80,000 to 100,000 per year (no currency stated)", so the figure is not lost along with the currency.""" bounds = bounds_words(lo, hi) if not bounds: return "" tail = f" per {period}" if period else "" code = str(currency or "").strip().upper() return f"{code} {bounds}{tail}" if code else f"{bounds}{tail} (no currency stated)"