Keep a description's headings, bullets and paragraphs, and draw them in the reader
The readers flattened a posting's HTML into one line with strip_html, so sections and bullets ran together. _shared.html_text gives a line per paragraph, '• ' before a list item and '## ' before a heading; every reader's description goes through it, sections read per role get their own headings, and the reader pane draws the lines as h3, ul and p. Excerpts leave the marks out. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
17e7615599
commit
c497fe965d
44 files changed
+300
-147
No files matched your search
@@ -8,10 +8,12 @@ import it without either importing the other. The leading underscore keeps
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from datetime import UTC, datetime
|
||||
from email.utils import parsedate_to_datetime
|
||||
from html.parser import HTMLParser
|
||||
from typing import Any, Protocol
|
||||
|
||||
import httpx
|
||||
@@ -216,3 +218,123 @@ def add_place(found: list[Place], entry: Place) -> None:
|
||||
found.append(entry)
|
||||
|
||||
|
||||
|
||||
|
||||
#: A description line that is a heading, and one that is a list item; what
|
||||
#: the reader pane draws as `<h3>` and `<li>`. The rest of a description is
|
||||
#: plain text, one paragraph a line.
|
||||
HEADING = "## "
|
||||
BULLET = "\u2022 "
|
||||
|
||||
#: Tags that end a line. A `<li>` starts a bullet line, an `<h1>`..`<h6>` a
|
||||
#: heading line; the rest start a paragraph.
|
||||
_BLOCKS = frozenset({
|
||||
"address", "article", "aside", "blockquote", "br", "dd", "div", "dl", "dt", "fieldset",
|
||||
"figcaption", "figure", "footer", "form", "header", "hr", "li", "main", "nav", "ol", "p",
|
||||
"pre", "section", "table", "tbody", "thead", "tfoot", "tr", "ul",
|
||||
"h1", "h2", "h3", "h4", "h5", "h6",
|
||||
})
|
||||
_HEADINGS = frozenset({"h1", "h2", "h3", "h4", "h5", "h6"})
|
||||
_BOLD = frozenset({"b", "strong"})
|
||||
_SKIPPED = frozenset({"script", "style", "head", "template", "noscript"})
|
||||
#: A paragraph set wholly in bold is a heading when it is at most this long;
|
||||
#: longer, it is a paragraph somebody made loud.
|
||||
_BOLD_HEADING = 80
|
||||
_TAG = re.compile(r"</?[A-Za-z][^>]*>")
|
||||
_ESCAPED_TAG = re.compile(r"</?[A-Za-z]")
|
||||
_ENTITY = re.compile(r"&(#\d+|#x[0-9A-Fa-f]+|[A-Za-z]+);")
|
||||
#: A bullet typed as text: `- item`, `* item`, `\u00b7 item` and the like.
|
||||
_TYPED_BULLET = re.compile("^[\u2022\u00b7\u25aa\u25cf\u25e6\u2023\u2043*\\-\u2013]\\s+")
|
||||
|
||||
|
||||
class _Lines(HTMLParser):
|
||||
"""`html_text`'s parser: a line per block, each a kind ("p", "li" or
|
||||
"h"), its text, and whether all of the text was bold."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.lines: list[tuple[str, str, bool]] = []
|
||||
self.kind = "p"
|
||||
self.pieces: list[tuple[str, bool]] = []
|
||||
self.bold = 0
|
||||
self.skip = 0
|
||||
|
||||
def has_text(self) -> bool:
|
||||
return any(text.strip() for text, _ in self.pieces)
|
||||
|
||||
def flush(self) -> None:
|
||||
if self.has_text():
|
||||
text = "".join(text for text, _ in self.pieces)
|
||||
bold = all(b for text, b in self.pieces if text.strip())
|
||||
self.lines.append((self.kind, text, bold))
|
||||
self.pieces = []
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
if tag in _SKIPPED:
|
||||
self.skip += 1
|
||||
elif tag in _BOLD:
|
||||
self.bold += 1
|
||||
elif tag in ("td", "th"):
|
||||
self.pieces.append((" ", True))
|
||||
elif tag in _BLOCKS:
|
||||
kind = "li" if tag == "li" else "h" if tag in _HEADINGS else ""
|
||||
if self.has_text():
|
||||
self.flush()
|
||||
self.kind = kind or "p"
|
||||
elif kind:
|
||||
self.kind = kind
|
||||
# else `<li><p>text</p></li>`: the `<p>` keeps the bullet.
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
if tag in _SKIPPED:
|
||||
self.skip = max(0, self.skip - 1)
|
||||
elif tag in _BOLD:
|
||||
self.bold = max(0, self.bold - 1)
|
||||
elif tag in _BLOCKS:
|
||||
self.flush()
|
||||
self.kind = "p"
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if not self.skip:
|
||||
self.pieces.append((data, self.bold > 0))
|
||||
|
||||
|
||||
def _line(kind: str, text: str, bold: bool) -> str:
|
||||
text = text.replace("\u00a0", " ").replace("\u200b", "").replace("\ufeff", "")
|
||||
if _ENTITY.search(text):
|
||||
# Escaped twice at the host (`&amp;`): once is left after parsing.
|
||||
text = html.unescape(text)
|
||||
text = " ".join(text.split())
|
||||
if typed := _TYPED_BULLET.match(text):
|
||||
text, kind = text[typed.end():], "li"
|
||||
if not text or text in ("\u2022", "\u00b7", "-", "*"):
|
||||
return ""
|
||||
if kind == "li":
|
||||
return BULLET + text
|
||||
if kind == "h" or (bold and len(text) <= _BOLD_HEADING):
|
||||
return HEADING + text
|
||||
return text
|
||||
|
||||
|
||||
def html_text(markup: Any) -> str:
|
||||
"""A host's description HTML as plain text that keeps its shape: a line
|
||||
per paragraph, `BULLET` before a list item and `HEADING` before a
|
||||
heading (an `<h1>`..`<h6>`, or a short paragraph set wholly in bold).
|
||||
Entities are decoded, and markup the host escaped once more
|
||||
(`<p>`) is read as markup. Text with no tags keeps its own line
|
||||
breaks, and a line typed as a bullet (`- item`) becomes one.
|
||||
|
||||
This is a row's `description`: what the reader pane draws as headings,
|
||||
lists and paragraphs, and what summarise and embed read. A one-line
|
||||
field (a title, a teaser) stays `desk.sources.strip_html`."""
|
||||
markup = str(markup or "")
|
||||
if not _TAG.search(markup):
|
||||
if _ESCAPED_TAG.search(markup):
|
||||
markup = html.unescape(markup)
|
||||
else:
|
||||
markup = "<br>".join(html.escape(line, quote=False) for line in markup.splitlines())
|
||||
parser = _Lines()
|
||||
parser.feed(markup)
|
||||
parser.close()
|
||||
parser.flush()
|
||||
return "\n".join(line for line in (_line(*entry) for entry in parser.lines) if line)
|
||||
Reference in new issue
Block a user