Resolves tests/test_readers_shared.py, adds MATCH_MODE to the pinpoint, recruitee and rippling tables, and removes table_word, country_if_single and remote_word from plugins/readers/_shared.py with their tests and the Helpers paragraph. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
220 lines
9.3 KiB
Python
220 lines
9.3 KiB
Python
"""What the host modules share: the row and page shapes, the fetch they are
|
|
handed, and the date, URL and salary helpers more than one host needs.
|
|
|
|
Kept apart from `plugins/ats.py` so that a host module and `ats.py` can both
|
|
import it without either importing the other. The leading underscore keeps
|
|
`plugins.readers` from listing it as a host.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from collections.abc import Callable
|
|
from datetime import UTC, datetime
|
|
from email.utils import parsedate_to_datetime
|
|
from typing import Any, Protocol
|
|
|
|
import httpx
|
|
|
|
from ._contract import Place
|
|
|
|
#: What an undeclared module's parser turns a feed into; a module that
|
|
#: declares `READS` returns `_contract.Row`, which adds `places` and `teaser`.
|
|
#: One dict per row, with the keys
|
|
#: `source_id`, `title`, `location`, `url`, `posted`, `department`,
|
|
#: `description`, `remote` and `salary_text`, and, where a host states pay,
|
|
#: `salary_min`, `salary_max` and `currency`.
|
|
Row = dict[str, Any]
|
|
|
|
#: A paged host's one page: `(decoded, results, total, cursor)`, the decoded
|
|
#: page, its raw results (dicts with an `id`, counted for completeness), the
|
|
#: total it states (None for a host that states none), and the cursor of the
|
|
#: next page (None after the last). The first call gets cursor None.
|
|
Page = tuple[Any, list[dict[str, Any]], Any, Any]
|
|
|
|
|
|
class Get(Protocol):
|
|
"""The fetch a host is handed: `ats.py`'s own, bound to one board, so the
|
|
allow-list, the redirect rule, the byte cap and the deadline apply to
|
|
every request a host makes. A POST sends `payload` as JSON; `headers` are
|
|
added to (and may replace) the default User-Agent and Accept. Returns the
|
|
body, and raises `httpx.HTTPStatusError` on a non-2xx answer; see
|
|
"Fetching" in the package's docstring for the rest."""
|
|
|
|
def __call__(self, url: str, *, method: str = "GET",
|
|
payload: dict[str, Any] | None = None,
|
|
headers: dict[str, str] | None = None) -> bytes: ...
|
|
|
|
|
|
def is_web_url(url: str) -> bool:
|
|
"""A trimmed http(s) URL, or refuse — a feed is external content and a
|
|
`javascript:` or `data:` link must never reach an href. Port of
|
|
openings.go's `webURL`."""
|
|
url = (url or "").strip()
|
|
if not url:
|
|
return False
|
|
parsed = httpx.URL(url)
|
|
return parsed.scheme in ("http", "https") and bool(parsed.host)
|
|
|
|
|
|
def only(host: str) -> Callable[[str], bool]:
|
|
"""The allow-list of a board that is fetched from one host name only,
|
|
which is every host so far. A host name has no case, and a slug that
|
|
names one (`Acme` of acme.recruitee.com) may be written with capitals."""
|
|
want = host.lower()
|
|
return lambda h: h.lower() == want
|
|
|
|
|
|
#: Matched whole (`fullmatch`, so not even a trailing newline slips past `$`).
|
|
#: A slug that is one URL path segment: what the hosts that put the slug in
|
|
#: the path accept, and nothing that could add a path, a query or a fragment.
|
|
_SEGMENT = re.compile(r"[A-Za-z0-9._~%-]+")
|
|
|
|
#: A slug that is one DNS label, for the hosts that put it in the host name.
|
|
_LABEL = re.compile(r"[A-Za-z0-9-]+")
|
|
|
|
|
|
def path_slug_ok(slug: str) -> bool:
|
|
return bool(_SEGMENT.fullmatch(slug)) and slug not in (".", "..")
|
|
|
|
|
|
def label_slug_ok(slug: str) -> bool:
|
|
return bool(_LABEL.fullmatch(slug))
|
|
|
|
|
|
def parse_dt(value: Any) -> datetime | None:
|
|
"""An aware datetime from whatever a board sends: ISO-8601 with an
|
|
offset, `Z`, a bare date, epoch milliseconds, "YYYY-MM-DD HH:MM:SS UTC",
|
|
or an RSS date ("Tue, 29 Sep 2026 13:36:59 +0200"). None for anything else, which `Item` reads as "now"."""
|
|
if value is None:
|
|
return None
|
|
if isinstance(value, int | float):
|
|
if value <= 0:
|
|
return None
|
|
seconds = value / 1000 if value > 10**12 else value
|
|
return datetime.fromtimestamp(seconds, tz=UTC)
|
|
text = str(value).strip()
|
|
if not text:
|
|
return None
|
|
try:
|
|
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
|
|
except ValueError:
|
|
match = re.match(r"^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})", text)
|
|
if not match:
|
|
try:
|
|
parsed = parsedate_to_datetime(text)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
|
|
parsed = datetime.fromisoformat(f"{match.group(1)}T{match.group(2)}+00:00")
|
|
return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
|
|
|
|
|
|
# ---------------------------------------------------------- structured pay
|
|
#
|
|
# A board's own salary range, mapped to yearly `salary_min`/`salary_max`
|
|
# (integers) and `currency` (ISO 4217) — the columns `upsert_role` unpacks
|
|
# `raw` into, per docs/decisions.md's "raw is transport, columns are
|
|
# storage". Only Ashby, Greenhouse, Lever, Recruitee and Personio give
|
|
# structured pay on their list endpoint; Workable, SmartRecruiters and
|
|
# Teamtailor do not (checked live, 2026-09-29), so their rows leave the
|
|
# columns out and `ATSSource.fetch` reads a plain '' or None for them.
|
|
|
|
|
|
def num(value: Any) -> float | None:
|
|
"""A positive number from whatever a board sends (int, float, or a
|
|
numeral string), or None for anything else, including zero and a
|
|
negative — a board's placeholder 0 (an unfilled field, not a real
|
|
figure) must not become a salary floor of zero."""
|
|
try:
|
|
number = float(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
return number if number > 0 else None
|
|
|
|
|
|
def yearly_amount(min_value: Any, max_value: Any, currency: Any, interval: Any,
|
|
factors: dict[str, int]) -> tuple[int | None, int | None, str]:
|
|
"""`(salary_min, salary_max, currency)` as yearly integers, or `(None,
|
|
None, "")` when there is no currency, no bound, or `interval` is not one
|
|
of `factors`' own keys — which is how an hourly or a daily rate is left
|
|
out rather than annualised into a number nobody stated. `factors` maps
|
|
the board's own interval spelling to the multiplier that makes it
|
|
yearly: 1 for a figure already yearly, 12 for a monthly one."""
|
|
code = str(currency or "").strip().upper()
|
|
factor = factors.get(str(interval or "").strip())
|
|
if not code or not factor:
|
|
return None, None, ""
|
|
lo, hi = num(min_value), num(max_value)
|
|
if lo is None and hi is None:
|
|
return None, None, ""
|
|
return (int(lo * factor) if lo is not None else None,
|
|
int(hi * factor) if hi is not None else None, code)
|
|
|
|
|
|
# ------------------------------------------------------------- salary_text
|
|
#
|
|
# `salary_min`/`salary_max`/`currency` above are yearly only, on purpose:
|
|
# hourly and daily rates, a Greenhouse role with several pay zones, and a
|
|
# range with no currency are all dropped from those columns rather than
|
|
# guessed into a wrong yearly figure. `salary_text` is where that dropped
|
|
# information survives — one human-readable line per parser, written
|
|
# whether or not the pay conformed, so a person can read it and do their own
|
|
# sums and a stricter analysis can find the roles the columns left out
|
|
# (`Role.salary_state`, `app/models.py`).
|
|
|
|
#: Formats a number the way a salary line writes it: no decimals for a whole
|
|
#: figure (the normal case for a yearly or monthly one), two for anything
|
|
#: else (an hourly or daily rate quoted with cents).
|
|
def fmt_amount(value: float) -> str:
|
|
return f"{int(value):,}" if value == int(value) else f"{value:,.2f}"
|
|
|
|
|
|
def bounds_words(lo: float | None, hi: float | None) -> str:
|
|
""""70,000 to 90,000", or "from 70,000"/"up to 90,000" with only one
|
|
bound stated, or "" with neither."""
|
|
if lo is not None and hi is not None:
|
|
return fmt_amount(lo) if lo == hi else f"{fmt_amount(lo)} to {fmt_amount(hi)}"
|
|
if lo is not None:
|
|
return f"from {fmt_amount(lo)}"
|
|
if hi is not None:
|
|
return f"up to {fmt_amount(hi)}"
|
|
return ""
|
|
|
|
|
|
def pay_line(lo: float | None, hi: float | None, currency: Any, period: str) -> str:
|
|
"""One statement of pay as a human-readable line — "EUR 70,000 to 90,000
|
|
per year" — or "" when there is no bound to report at all. A currency
|
|
that was not stated does not drop the line, unlike the normalised
|
|
columns: it is said plainly instead, "80,000 to 100,000 per year (no
|
|
currency stated)", so the figure is not lost along with the currency."""
|
|
bounds = bounds_words(lo, hi)
|
|
if not bounds:
|
|
return ""
|
|
tail = f" per {period}" if period else ""
|
|
code = str(currency or "").strip().upper()
|
|
return f"{code} {bounds}{tail}" if code else f"{bounds}{tail} (no currency stated)"
|
|
|
|
|
|
#: A copy of `app.models.EMPLOYMENTS`, the vocabulary `app.prep` maps a
|
|
#: host's employment word onto, kept for the readers' tests since a plugin
|
|
#: imports nothing from `app`. `tests/test_prep.py` holds the two equal.
|
|
EMPLOYMENTS = ("full_time", "part_time", "contract", "temporary", "internship", "freelance")
|
|
|
|
_REMOTE_WORD = re.compile(r"\bremote\b", re.IGNORECASE)
|
|
|
|
|
|
def place(text: str, remote: bool | None = None) -> Place:
|
|
"""A `Place` for `text`: remote when the text says the word `remote`,
|
|
otherwise as `remote` says (None when the host says nothing)."""
|
|
return {"text": text, "remote": True if _REMOTE_WORD.search(text) else remote}
|
|
|
|
|
|
def add_place(found: list[Place], entry: Place) -> None:
|
|
"""Append `entry` to `found` unless its text is empty or already there;
|
|
the host's order is kept."""
|
|
if entry["text"] and entry["text"] not in (p["text"] for p in found):
|
|
found.append(entry)
|
|
|
|
|