Files
jobboard/plugins/atshosts/_shared.py
T

193 lines
8.3 KiB
Python

"""What the host modules share: the row and page shapes, the fetch they are
handed, and the date, URL and salary helpers more than one host needs.
Kept apart from `plugins/ats.py` so that a host module and `ats.py` can both
import it without either importing the other. The leading underscore keeps
`plugins.atshosts` from listing it as a host.
"""
from __future__ import annotations
import re
from collections.abc import Callable
from datetime import UTC, datetime
from email.utils import parsedate_to_datetime
from typing import Any, Protocol
import httpx
#: What every parser turns a feed into: one dict per row, with the keys
#: `source_id`, `title`, `location`, `url`, `posted`, `department`,
#: `description`, `remote` and `salary_text`, and, where a host states pay,
#: `salary_min`, `salary_max` and `currency`.
Row = dict[str, Any]
#: A paged host's one page: `(decoded, results, total, cursor)`, the decoded
#: page, its raw results (dicts with an `id`, counted for completeness), the
#: total it states (None for a host that states none), and the cursor of the
#: next page (None after the last). The first call gets cursor None.
Page = tuple[Any, list[dict[str, Any]], Any, Any]
class Get(Protocol):
"""The fetch a host is handed: `ats.py`'s own, bound to one board, so the
allow-list, the redirect rule, the byte cap and the deadline apply to
every request a host makes. A POST sends `payload` as JSON; `headers` are
added to (and may replace) the default User-Agent and Accept. Returns the
body, and raises `httpx.HTTPStatusError` on a non-2xx answer; see
"Fetching" in the package's docstring for the rest."""
def __call__(self, url: str, *, method: str = "GET",
payload: dict[str, Any] | None = None,
headers: dict[str, str] | None = None) -> bytes: ...
def is_web_url(url: str) -> bool:
"""A trimmed http(s) URL, or refuse — a feed is external content and a
`javascript:` or `data:` link must never reach an href. Port of
openings.go's `webURL`."""
url = (url or "").strip()
if not url:
return False
parsed = httpx.URL(url)
return parsed.scheme in ("http", "https") and bool(parsed.host)
def only(host: str) -> Callable[[str], bool]:
"""The allow-list of a board that is fetched from one host name only,
which is every host so far. A host name has no case, and a slug that
names one (`Acme` of acme.recruitee.com) may be written with capitals."""
want = host.lower()
return lambda h: h.lower() == want
#: Matched whole (`fullmatch`, so not even a trailing newline slips past `$`).
#: A slug that is one URL path segment: what the hosts that put the slug in
#: the path accept, and nothing that could add a path, a query or a fragment.
_SEGMENT = re.compile(r"[A-Za-z0-9._~%-]+")
#: A slug that is one DNS label, for the hosts that put it in the host name.
_LABEL = re.compile(r"[A-Za-z0-9-]+")
def path_slug_ok(slug: str) -> bool:
return bool(_SEGMENT.fullmatch(slug)) and slug not in (".", "..")
def label_slug_ok(slug: str) -> bool:
return bool(_LABEL.fullmatch(slug))
def parse_dt(value: Any) -> datetime | None:
"""An aware datetime from whatever a board sends: ISO-8601 with an
offset, `Z`, a bare date, epoch milliseconds, "YYYY-MM-DD HH:MM:SS UTC",
or an RSS date ("Tue, 29 Sep 2026 13:36:59 +0200"). None for anything else, which `Item` reads as "now"."""
if value is None:
return None
if isinstance(value, int | float):
if value <= 0:
return None
seconds = value / 1000 if value > 10**12 else value
return datetime.fromtimestamp(seconds, tz=UTC)
text = str(value).strip()
if not text:
return None
try:
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
except ValueError:
match = re.match(r"^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})", text)
if not match:
try:
parsed = parsedate_to_datetime(text)
except (TypeError, ValueError):
return None
return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
parsed = datetime.fromisoformat(f"{match.group(1)}T{match.group(2)}+00:00")
return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
# ---------------------------------------------------------- structured pay
#
# A board's own salary range, mapped to yearly `salary_min`/`salary_max`
# (integers) and `currency` (ISO 4217) — the columns `upsert_role` unpacks
# `raw` into, per docs/decisions.md's "raw is transport, columns are
# storage". Only Ashby, Greenhouse, Lever, Recruitee and Personio give
# structured pay on their list endpoint; Workable, SmartRecruiters and
# Teamtailor do not (checked live, 2026-09-29), so their rows leave the
# columns out and `ATSSource.fetch` reads a plain '' or None for them.
def num(value: Any) -> float | None:
"""A positive number from whatever a board sends (int, float, or a
numeral string), or None for anything else, including zero and a
negative — a board's placeholder 0 (an unfilled field, not a real
figure) must not become a salary floor of zero."""
try:
number = float(value)
except (TypeError, ValueError):
return None
return number if number > 0 else None
def yearly_amount(min_value: Any, max_value: Any, currency: Any, interval: Any,
factors: dict[str, int]) -> tuple[int | None, int | None, str]:
"""`(salary_min, salary_max, currency)` as yearly integers, or `(None,
None, "")` when there is no currency, no bound, or `interval` is not one
of `factors`' own keys — which is how an hourly or a daily rate is left
out rather than annualised into a number nobody stated. `factors` maps
the board's own interval spelling to the multiplier that makes it
yearly: 1 for a figure already yearly, 12 for a monthly one."""
code = str(currency or "").strip().upper()
factor = factors.get(str(interval or "").strip())
if not code or not factor:
return None, None, ""
lo, hi = num(min_value), num(max_value)
if lo is None and hi is None:
return None, None, ""
return (int(lo * factor) if lo is not None else None,
int(hi * factor) if hi is not None else None, code)
# ------------------------------------------------------------- salary_text
#
# `salary_min`/`salary_max`/`currency` above are yearly only, on purpose:
# hourly and daily rates, a Greenhouse role with several pay zones, and a
# range with no currency are all dropped from those columns rather than
# guessed into a wrong yearly figure. `salary_text` is where that dropped
# information survives — one human-readable line per parser, written
# whether or not the pay conformed, so a person can read it and do their own
# sums and a stricter analysis can find the roles the columns left out
# (`Role.salary_state`, `app/models.py`).
#: Formats a number the way a salary line writes it: no decimals for a whole
#: figure (the normal case for a yearly or monthly one), two for anything
#: else (an hourly or daily rate quoted with cents).
def fmt_amount(value: float) -> str:
return f"{int(value):,}" if value == int(value) else f"{value:,.2f}"
def bounds_words(lo: float | None, hi: float | None) -> str:
""""70,000 to 90,000", or "from 70,000"/"up to 90,000" with only one
bound stated, or "" with neither."""
if lo is not None and hi is not None:
return fmt_amount(lo) if lo == hi else f"{fmt_amount(lo)} to {fmt_amount(hi)}"
if lo is not None:
return f"from {fmt_amount(lo)}"
if hi is not None:
return f"up to {fmt_amount(hi)}"
return ""
def pay_line(lo: float | None, hi: float | None, currency: Any, period: str) -> str:
"""One statement of pay as a human-readable line — "EUR 70,000 to 90,000
per year" — or "" when there is no bound to report at all. A currency
that was not stated does not drop the line, unlike the normalised
columns: it is said plainly instead, "80,000 to 100,000 per year (no
currency stated)", so the figure is not lost along with the currency."""
bounds = bounds_words(lo, hi)
if not bounds:
return ""
tail = f" per {period}" if period else ""
code = str(currency or "").strip().upper()
return f"{code} {bounds}{tail}" if code else f"{bounds}{tail} (no currency stated)"