Rename plugins/atshosts to plugins/readers
Every import, test and live doc follows the new path; behaviour is unchanged. tests/test_atshosts.py becomes tests/test_readers.py. plan-readers.md and review-ingest.md keep the old name, since they describe the state before. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
af337ef558
commit
73dec9028e
48 files changed
+104
-104
No files matched your search
@@ -0,0 +1,192 @@
|
||||
"""What the host modules share: the row and page shapes, the fetch they are
|
||||
handed, and the date, URL and salary helpers more than one host needs.
|
||||
|
||||
Kept apart from `plugins/ats.py` so that a host module and `ats.py` can both
|
||||
import it without either importing the other. The leading underscore keeps
|
||||
`plugins.readers` from listing it as a host.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime
|
||||
from email.utils import parsedate_to_datetime
|
||||
from typing import Any, Protocol
|
||||
|
||||
import httpx
|
||||
|
||||
#: What every parser turns a feed into: one dict per row, with the keys
|
||||
#: `source_id`, `title`, `location`, `url`, `posted`, `department`,
|
||||
#: `description`, `remote` and `salary_text`, and, where a host states pay,
|
||||
#: `salary_min`, `salary_max` and `currency`.
|
||||
Row = dict[str, Any]
|
||||
|
||||
#: A paged host's one page: `(decoded, results, total, cursor)`, the decoded
|
||||
#: page, its raw results (dicts with an `id`, counted for completeness), the
|
||||
#: total it states (None for a host that states none), and the cursor of the
|
||||
#: next page (None after the last). The first call gets cursor None.
|
||||
Page = tuple[Any, list[dict[str, Any]], Any, Any]
|
||||
|
||||
|
||||
class Get(Protocol):
|
||||
"""The fetch a host is handed: `ats.py`'s own, bound to one board, so the
|
||||
allow-list, the redirect rule, the byte cap and the deadline apply to
|
||||
every request a host makes. A POST sends `payload` as JSON; `headers` are
|
||||
added to (and may replace) the default User-Agent and Accept. Returns the
|
||||
body, and raises `httpx.HTTPStatusError` on a non-2xx answer; see
|
||||
"Fetching" in the package's docstring for the rest."""
|
||||
|
||||
def __call__(self, url: str, *, method: str = "GET",
|
||||
payload: dict[str, Any] | None = None,
|
||||
headers: dict[str, str] | None = None) -> bytes: ...
|
||||
|
||||
|
||||
def is_web_url(url: str) -> bool:
|
||||
"""A trimmed http(s) URL, or refuse — a feed is external content and a
|
||||
`javascript:` or `data:` link must never reach an href. Port of
|
||||
openings.go's `webURL`."""
|
||||
url = (url or "").strip()
|
||||
if not url:
|
||||
return False
|
||||
parsed = httpx.URL(url)
|
||||
return parsed.scheme in ("http", "https") and bool(parsed.host)
|
||||
|
||||
|
||||
def only(host: str) -> Callable[[str], bool]:
|
||||
"""The allow-list of a board that is fetched from one host name only,
|
||||
which is every host so far. A host name has no case, and a slug that
|
||||
names one (`Acme` of acme.recruitee.com) may be written with capitals."""
|
||||
want = host.lower()
|
||||
return lambda h: h.lower() == want
|
||||
|
||||
|
||||
#: Matched whole (`fullmatch`, so not even a trailing newline slips past `$`).
|
||||
#: A slug that is one URL path segment: what the hosts that put the slug in
|
||||
#: the path accept, and nothing that could add a path, a query or a fragment.
|
||||
_SEGMENT = re.compile(r"[A-Za-z0-9._~%-]+")
|
||||
|
||||
#: A slug that is one DNS label, for the hosts that put it in the host name.
|
||||
_LABEL = re.compile(r"[A-Za-z0-9-]+")
|
||||
|
||||
|
||||
def path_slug_ok(slug: str) -> bool:
|
||||
return bool(_SEGMENT.fullmatch(slug)) and slug not in (".", "..")
|
||||
|
||||
|
||||
def label_slug_ok(slug: str) -> bool:
|
||||
return bool(_LABEL.fullmatch(slug))
|
||||
|
||||
|
||||
def parse_dt(value: Any) -> datetime | None:
|
||||
"""An aware datetime from whatever a board sends: ISO-8601 with an
|
||||
offset, `Z`, a bare date, epoch milliseconds, "YYYY-MM-DD HH:MM:SS UTC",
|
||||
or an RSS date ("Tue, 29 Sep 2026 13:36:59 +0200"). None for anything else, which `Item` reads as "now"."""
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, int | float):
|
||||
if value <= 0:
|
||||
return None
|
||||
seconds = value / 1000 if value > 10**12 else value
|
||||
return datetime.fromtimestamp(seconds, tz=UTC)
|
||||
text = str(value).strip()
|
||||
if not text:
|
||||
return None
|
||||
try:
|
||||
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
match = re.match(r"^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})", text)
|
||||
if not match:
|
||||
try:
|
||||
parsed = parsedate_to_datetime(text)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
|
||||
parsed = datetime.fromisoformat(f"{match.group(1)}T{match.group(2)}+00:00")
|
||||
return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
|
||||
|
||||
|
||||
# ---------------------------------------------------------- structured pay
|
||||
#
|
||||
# A board's own salary range, mapped to yearly `salary_min`/`salary_max`
|
||||
# (integers) and `currency` (ISO 4217) — the columns `upsert_role` unpacks
|
||||
# `raw` into, per docs/decisions.md's "raw is transport, columns are
|
||||
# storage". Only Ashby, Greenhouse, Lever, Recruitee and Personio give
|
||||
# structured pay on their list endpoint; Workable, SmartRecruiters and
|
||||
# Teamtailor do not (checked live, 2026-09-29), so their rows leave the
|
||||
# columns out and `ATSSource.fetch` reads a plain '' or None for them.
|
||||
|
||||
|
||||
def num(value: Any) -> float | None:
|
||||
"""A positive number from whatever a board sends (int, float, or a
|
||||
numeral string), or None for anything else, including zero and a
|
||||
negative — a board's placeholder 0 (an unfilled field, not a real
|
||||
figure) must not become a salary floor of zero."""
|
||||
try:
|
||||
number = float(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return number if number > 0 else None
|
||||
|
||||
|
||||
def yearly_amount(min_value: Any, max_value: Any, currency: Any, interval: Any,
|
||||
factors: dict[str, int]) -> tuple[int | None, int | None, str]:
|
||||
"""`(salary_min, salary_max, currency)` as yearly integers, or `(None,
|
||||
None, "")` when there is no currency, no bound, or `interval` is not one
|
||||
of `factors`' own keys — which is how an hourly or a daily rate is left
|
||||
out rather than annualised into a number nobody stated. `factors` maps
|
||||
the board's own interval spelling to the multiplier that makes it
|
||||
yearly: 1 for a figure already yearly, 12 for a monthly one."""
|
||||
code = str(currency or "").strip().upper()
|
||||
factor = factors.get(str(interval or "").strip())
|
||||
if not code or not factor:
|
||||
return None, None, ""
|
||||
lo, hi = num(min_value), num(max_value)
|
||||
if lo is None and hi is None:
|
||||
return None, None, ""
|
||||
return (int(lo * factor) if lo is not None else None,
|
||||
int(hi * factor) if hi is not None else None, code)
|
||||
|
||||
|
||||
# ------------------------------------------------------------- salary_text
|
||||
#
|
||||
# `salary_min`/`salary_max`/`currency` above are yearly only, on purpose:
|
||||
# hourly and daily rates, a Greenhouse role with several pay zones, and a
|
||||
# range with no currency are all dropped from those columns rather than
|
||||
# guessed into a wrong yearly figure. `salary_text` is where that dropped
|
||||
# information survives — one human-readable line per parser, written
|
||||
# whether or not the pay conformed, so a person can read it and do their own
|
||||
# sums and a stricter analysis can find the roles the columns left out
|
||||
# (`Role.salary_state`, `app/models.py`).
|
||||
|
||||
#: Formats a number the way a salary line writes it: no decimals for a whole
|
||||
#: figure (the normal case for a yearly or monthly one), two for anything
|
||||
#: else (an hourly or daily rate quoted with cents).
|
||||
def fmt_amount(value: float) -> str:
|
||||
return f"{int(value):,}" if value == int(value) else f"{value:,.2f}"
|
||||
|
||||
|
||||
def bounds_words(lo: float | None, hi: float | None) -> str:
|
||||
""""70,000 to 90,000", or "from 70,000"/"up to 90,000" with only one
|
||||
bound stated, or "" with neither."""
|
||||
if lo is not None and hi is not None:
|
||||
return fmt_amount(lo) if lo == hi else f"{fmt_amount(lo)} to {fmt_amount(hi)}"
|
||||
if lo is not None:
|
||||
return f"from {fmt_amount(lo)}"
|
||||
if hi is not None:
|
||||
return f"up to {fmt_amount(hi)}"
|
||||
return ""
|
||||
|
||||
|
||||
def pay_line(lo: float | None, hi: float | None, currency: Any, period: str) -> str:
|
||||
"""One statement of pay as a human-readable line — "EUR 70,000 to 90,000
|
||||
per year" — or "" when there is no bound to report at all. A currency
|
||||
that was not stated does not drop the line, unlike the normalised
|
||||
columns: it is said plainly instead, "80,000 to 100,000 per year (no
|
||||
currency stated)", so the figure is not lost along with the currency."""
|
||||
bounds = bounds_words(lo, hi)
|
||||
if not bounds:
|
||||
return ""
|
||||
tail = f" per {period}" if period else ""
|
||||
code = str(currency or "").strip().upper()
|
||||
return f"{code} {bounds}{tail}" if code else f"{bounds}{tail} (no currency stated)"
|
||||
Reference in new issue
Block a user