The readers flattened a posting's HTML into one line with strip_html, so sections and bullets ran together. _shared.html_text gives a line per paragraph, '• ' before a list item and '## ' before a heading; every reader's description goes through it, sections read per role get their own headings, and the reader pane draws the lines as h3, ul and p. Excerpts leave the marks out. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
341 lines
14 KiB
Python
341 lines
14 KiB
Python
"""What the host modules share: the row and page shapes, the fetch they are
|
|
handed, and the date, URL and salary helpers more than one host needs.
|
|
|
|
Kept apart from `plugins/ats.py` so that a host module and `ats.py` can both
|
|
import it without either importing the other. The leading underscore keeps
|
|
`plugins.readers` from listing it as a host.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import html
|
|
import re
|
|
from collections.abc import Callable
|
|
from datetime import UTC, datetime
|
|
from email.utils import parsedate_to_datetime
|
|
from html.parser import HTMLParser
|
|
from typing import Any, Protocol
|
|
|
|
import httpx
|
|
|
|
from ._contract import Place
|
|
|
|
#: What a reader's parser turns a feed into, as `dict[str, Any]`; the keys a
|
|
#: row carries are `_contract.Row`'s. One dict per row, with the keys
|
|
#: `source_id`, `title`, `location`, `url`, `posted`, `department`,
|
|
#: `description`, `remote` and `salary_text`, and, where a host states pay,
|
|
#: `salary_min`, `salary_max` and `currency`.
|
|
Row = dict[str, Any]
|
|
|
|
#: A paged host's one page: `(decoded, results, total, cursor)`, the decoded
|
|
#: page, its raw results (dicts with an `id`, counted for completeness), the
|
|
#: total it states (None for a host that states none), and the cursor of the
|
|
#: next page (None after the last). The first call gets cursor None.
|
|
Page = tuple[Any, list[dict[str, Any]], Any, Any]
|
|
|
|
|
|
class Get(Protocol):
|
|
"""The fetch a host is handed: `ats.py`'s own, bound to one board, so the
|
|
allow-list, the redirect rule, the byte cap and the deadline apply to
|
|
every request a host makes. A POST sends `payload` as JSON; `headers` are
|
|
added to (and may replace) the default User-Agent and Accept. Returns the
|
|
body, and raises `httpx.HTTPStatusError` on a non-2xx answer; see
|
|
"Fetching" in the package's docstring for the rest."""
|
|
|
|
def __call__(self, url: str, *, method: str = "GET",
|
|
payload: dict[str, Any] | None = None,
|
|
headers: dict[str, str] | None = None) -> bytes: ...
|
|
|
|
|
|
def is_web_url(url: str) -> bool:
|
|
"""A trimmed http(s) URL, or refuse — a feed is external content and a
|
|
`javascript:` or `data:` link must never reach an href. Port of
|
|
openings.go's `webURL`."""
|
|
url = (url or "").strip()
|
|
if not url:
|
|
return False
|
|
parsed = httpx.URL(url)
|
|
return parsed.scheme in ("http", "https") and bool(parsed.host)
|
|
|
|
|
|
def only(host: str) -> Callable[[str], bool]:
|
|
"""The allow-list of a board that is fetched from one host name only,
|
|
which is every host so far. A host name has no case, and a slug that
|
|
names one (`Acme` of acme.recruitee.com) may be written with capitals."""
|
|
want = host.lower()
|
|
return lambda h: h.lower() == want
|
|
|
|
|
|
#: Matched whole (`fullmatch`, so not even a trailing newline slips past `$`).
|
|
#: A slug that is one URL path segment: what the hosts that put the slug in
|
|
#: the path accept, and nothing that could add a path, a query or a fragment.
|
|
_SEGMENT = re.compile(r"[A-Za-z0-9._~%-]+")
|
|
|
|
#: A slug that is one DNS label, for the hosts that put it in the host name.
|
|
_LABEL = re.compile(r"[A-Za-z0-9-]+")
|
|
|
|
|
|
def path_slug_ok(slug: str) -> bool:
|
|
return bool(_SEGMENT.fullmatch(slug)) and slug not in (".", "..")
|
|
|
|
|
|
def label_slug_ok(slug: str) -> bool:
|
|
return bool(_LABEL.fullmatch(slug))
|
|
|
|
|
|
def parse_dt(value: Any) -> datetime | None:
|
|
"""An aware datetime from whatever a board sends: ISO-8601 with an
|
|
offset, `Z`, a bare date, epoch milliseconds, "YYYY-MM-DD HH:MM:SS UTC",
|
|
or an RSS date ("Tue, 29 Sep 2026 13:36:59 +0200"). None for anything else, which `Item` reads as "now"."""
|
|
if value is None:
|
|
return None
|
|
if isinstance(value, int | float):
|
|
if value <= 0:
|
|
return None
|
|
seconds = value / 1000 if value > 10**12 else value
|
|
return datetime.fromtimestamp(seconds, tz=UTC)
|
|
text = str(value).strip()
|
|
if not text:
|
|
return None
|
|
try:
|
|
parsed = datetime.fromisoformat(text.replace("Z", "+00:00"))
|
|
except ValueError:
|
|
match = re.match(r"^(\d{4}-\d{2}-\d{2})[ T](\d{2}:\d{2}:\d{2})", text)
|
|
if not match:
|
|
try:
|
|
parsed = parsedate_to_datetime(text)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
|
|
parsed = datetime.fromisoformat(f"{match.group(1)}T{match.group(2)}+00:00")
|
|
return parsed if parsed.utcoffset() is not None else parsed.replace(tzinfo=UTC)
|
|
|
|
|
|
# ---------------------------------------------------------- structured pay
|
|
#
|
|
# A board's own salary range, mapped to yearly `salary_min`/`salary_max`
|
|
# (integers) and `currency` (ISO 4217) — the columns `upsert_role` unpacks
|
|
# `raw` into, per docs/decisions.md's "raw is transport, columns are
|
|
# storage". Only Ashby, Greenhouse, Lever, Recruitee and Personio give
|
|
# structured pay on their list endpoint; Workable, SmartRecruiters and
|
|
# Teamtailor do not (checked live, 2026-09-29), so their rows leave the
|
|
# columns out and `ATSSource.fetch` reads a plain '' or None for them.
|
|
|
|
|
|
def num(value: Any) -> float | None:
|
|
"""A positive number from whatever a board sends (int, float, or a
|
|
numeral string), or None for anything else, including zero and a
|
|
negative — a board's placeholder 0 (an unfilled field, not a real
|
|
figure) must not become a salary floor of zero."""
|
|
try:
|
|
number = float(value)
|
|
except (TypeError, ValueError):
|
|
return None
|
|
return number if number > 0 else None
|
|
|
|
|
|
def yearly_amount(min_value: Any, max_value: Any, currency: Any, interval: Any,
|
|
factors: dict[str, int]) -> tuple[int | None, int | None, str]:
|
|
"""`(salary_min, salary_max, currency)` as yearly integers, or `(None,
|
|
None, "")` when there is no currency, no bound, or `interval` is not one
|
|
of `factors`' own keys — which is how an hourly or a daily rate is left
|
|
out rather than annualised into a number nobody stated. `factors` maps
|
|
the board's own interval spelling to the multiplier that makes it
|
|
yearly: 1 for a figure already yearly, 12 for a monthly one."""
|
|
code = str(currency or "").strip().upper()
|
|
factor = factors.get(str(interval or "").strip())
|
|
if not code or not factor:
|
|
return None, None, ""
|
|
lo, hi = num(min_value), num(max_value)
|
|
if lo is None and hi is None:
|
|
return None, None, ""
|
|
return (int(lo * factor) if lo is not None else None,
|
|
int(hi * factor) if hi is not None else None, code)
|
|
|
|
|
|
# ------------------------------------------------------------- salary_text
|
|
#
|
|
# `salary_min`/`salary_max`/`currency` above are yearly only, on purpose:
|
|
# hourly and daily rates, a Greenhouse role with several pay zones, and a
|
|
# range with no currency are all dropped from those columns rather than
|
|
# guessed into a wrong yearly figure. `salary_text` is where that dropped
|
|
# information survives — one human-readable line per parser, written
|
|
# whether or not the pay conformed, so a person can read it and do their own
|
|
# sums and a stricter analysis can find the roles the columns left out
|
|
# (`Role.salary_state`, `app/models.py`).
|
|
|
|
#: Formats a number the way a salary line writes it: no decimals for a whole
|
|
#: figure (the normal case for a yearly or monthly one), two for anything
|
|
#: else (an hourly or daily rate quoted with cents).
|
|
def fmt_amount(value: float) -> str:
|
|
return f"{int(value):,}" if value == int(value) else f"{value:,.2f}"
|
|
|
|
|
|
def bounds_words(lo: float | None, hi: float | None) -> str:
|
|
""""70,000 to 90,000", or "from 70,000"/"up to 90,000" with only one
|
|
bound stated, or "" with neither."""
|
|
if lo is not None and hi is not None:
|
|
return fmt_amount(lo) if lo == hi else f"{fmt_amount(lo)} to {fmt_amount(hi)}"
|
|
if lo is not None:
|
|
return f"from {fmt_amount(lo)}"
|
|
if hi is not None:
|
|
return f"up to {fmt_amount(hi)}"
|
|
return ""
|
|
|
|
|
|
def pay_line(lo: float | None, hi: float | None, currency: Any, period: str) -> str:
|
|
"""One statement of pay as a human-readable line — "EUR 70,000 to 90,000
|
|
per year" — or "" when there is no bound to report at all. A currency
|
|
that was not stated does not drop the line, unlike the normalised
|
|
columns: it is said plainly instead, "80,000 to 100,000 per year (no
|
|
currency stated)", so the figure is not lost along with the currency."""
|
|
bounds = bounds_words(lo, hi)
|
|
if not bounds:
|
|
return ""
|
|
tail = f" per {period}" if period else ""
|
|
code = str(currency or "").strip().upper()
|
|
return f"{code} {bounds}{tail}" if code else f"{bounds}{tail} (no currency stated)"
|
|
|
|
|
|
#: A copy of `app.models.EMPLOYMENTS`, the vocabulary `app.prep` maps a
|
|
#: host's employment word onto, kept for the readers' tests since a plugin
|
|
#: imports nothing from `app`. `tests/test_prep.py` holds the two equal.
|
|
EMPLOYMENTS = ("full_time", "part_time", "contract", "temporary", "internship", "freelance")
|
|
|
|
_REMOTE_WORD = re.compile(r"\bremote\b", re.IGNORECASE)
|
|
|
|
|
|
def place(text: str, remote: bool | None = None) -> Place:
|
|
"""A `Place` for `text`: remote when the text says the word `remote`,
|
|
otherwise as `remote` says (None when the host says nothing)."""
|
|
return {"text": text, "remote": True if _REMOTE_WORD.search(text) else remote}
|
|
|
|
|
|
def add_place(found: list[Place], entry: Place) -> None:
|
|
"""Append `entry` to `found` unless its text is empty or already there;
|
|
the host's order is kept."""
|
|
if entry["text"] and entry["text"] not in (p["text"] for p in found):
|
|
found.append(entry)
|
|
|
|
|
|
|
|
|
|
#: A description line that is a heading, and one that is a list item; what
|
|
#: the reader pane draws as `<h3>` and `<li>`. The rest of a description is
|
|
#: plain text, one paragraph a line.
|
|
HEADING = "## "
|
|
BULLET = "\u2022 "
|
|
|
|
#: Tags that end a line. A `<li>` starts a bullet line, an `<h1>`..`<h6>` a
|
|
#: heading line; the rest start a paragraph.
|
|
_BLOCKS = frozenset({
|
|
"address", "article", "aside", "blockquote", "br", "dd", "div", "dl", "dt", "fieldset",
|
|
"figcaption", "figure", "footer", "form", "header", "hr", "li", "main", "nav", "ol", "p",
|
|
"pre", "section", "table", "tbody", "thead", "tfoot", "tr", "ul",
|
|
"h1", "h2", "h3", "h4", "h5", "h6",
|
|
})
|
|
_HEADINGS = frozenset({"h1", "h2", "h3", "h4", "h5", "h6"})
|
|
_BOLD = frozenset({"b", "strong"})
|
|
_SKIPPED = frozenset({"script", "style", "head", "template", "noscript"})
|
|
#: A paragraph set wholly in bold is a heading when it is at most this long;
|
|
#: longer, it is a paragraph somebody made loud.
|
|
_BOLD_HEADING = 80
|
|
_TAG = re.compile(r"</?[A-Za-z][^>]*>")
|
|
_ESCAPED_TAG = re.compile(r"</?[A-Za-z]")
|
|
_ENTITY = re.compile(r"&(#\d+|#x[0-9A-Fa-f]+|[A-Za-z]+);")
|
|
#: A bullet typed as text: `- item`, `* item`, `\u00b7 item` and the like.
|
|
_TYPED_BULLET = re.compile("^[\u2022\u00b7\u25aa\u25cf\u25e6\u2023\u2043*\\-\u2013]\\s+")
|
|
|
|
|
|
class _Lines(HTMLParser):
|
|
"""`html_text`'s parser: a line per block, each a kind ("p", "li" or
|
|
"h"), its text, and whether all of the text was bold."""
|
|
|
|
def __init__(self) -> None:
|
|
super().__init__(convert_charrefs=True)
|
|
self.lines: list[tuple[str, str, bool]] = []
|
|
self.kind = "p"
|
|
self.pieces: list[tuple[str, bool]] = []
|
|
self.bold = 0
|
|
self.skip = 0
|
|
|
|
def has_text(self) -> bool:
|
|
return any(text.strip() for text, _ in self.pieces)
|
|
|
|
def flush(self) -> None:
|
|
if self.has_text():
|
|
text = "".join(text for text, _ in self.pieces)
|
|
bold = all(b for text, b in self.pieces if text.strip())
|
|
self.lines.append((self.kind, text, bold))
|
|
self.pieces = []
|
|
|
|
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
|
if tag in _SKIPPED:
|
|
self.skip += 1
|
|
elif tag in _BOLD:
|
|
self.bold += 1
|
|
elif tag in ("td", "th"):
|
|
self.pieces.append((" ", True))
|
|
elif tag in _BLOCKS:
|
|
kind = "li" if tag == "li" else "h" if tag in _HEADINGS else ""
|
|
if self.has_text():
|
|
self.flush()
|
|
self.kind = kind or "p"
|
|
elif kind:
|
|
self.kind = kind
|
|
# else `<li><p>text</p></li>`: the `<p>` keeps the bullet.
|
|
|
|
def handle_endtag(self, tag: str) -> None:
|
|
if tag in _SKIPPED:
|
|
self.skip = max(0, self.skip - 1)
|
|
elif tag in _BOLD:
|
|
self.bold = max(0, self.bold - 1)
|
|
elif tag in _BLOCKS:
|
|
self.flush()
|
|
self.kind = "p"
|
|
|
|
def handle_data(self, data: str) -> None:
|
|
if not self.skip:
|
|
self.pieces.append((data, self.bold > 0))
|
|
|
|
|
|
def _line(kind: str, text: str, bold: bool) -> str:
|
|
text = text.replace("\u00a0", " ").replace("\u200b", "").replace("\ufeff", "")
|
|
if _ENTITY.search(text):
|
|
# Escaped twice at the host (`&amp;`): once is left after parsing.
|
|
text = html.unescape(text)
|
|
text = " ".join(text.split())
|
|
if typed := _TYPED_BULLET.match(text):
|
|
text, kind = text[typed.end():], "li"
|
|
if not text or text in ("\u2022", "\u00b7", "-", "*"):
|
|
return ""
|
|
if kind == "li":
|
|
return BULLET + text
|
|
if kind == "h" or (bold and len(text) <= _BOLD_HEADING):
|
|
return HEADING + text
|
|
return text
|
|
|
|
|
|
def html_text(markup: Any) -> str:
|
|
"""A host's description HTML as plain text that keeps its shape: a line
|
|
per paragraph, `BULLET` before a list item and `HEADING` before a
|
|
heading (an `<h1>`..`<h6>`, or a short paragraph set wholly in bold).
|
|
Entities are decoded, and markup the host escaped once more
|
|
(`<p>`) is read as markup. Text with no tags keeps its own line
|
|
breaks, and a line typed as a bullet (`- item`) becomes one.
|
|
|
|
This is a row's `description`: what the reader pane draws as headings,
|
|
lists and paragraphs, and what summarise and embed read. A one-line
|
|
field (a title, a teaser) stays `desk.sources.strip_html`."""
|
|
markup = str(markup or "")
|
|
if not _TAG.search(markup):
|
|
if _ESCAPED_TAG.search(markup):
|
|
markup = html.unescape(markup)
|
|
else:
|
|
markup = "<br>".join(html.escape(line, quote=False) for line in markup.splitlines())
|
|
parser = _Lines()
|
|
parser.feed(markup)
|
|
parser.close()
|
|
parser.flush()
|
|
return "\n".join(line for line in (_line(*entry) for entry in parser.lines) if line)
|