The plugin contract, the model task and the tools roledesk imports, copied rather than reached through an editable install of news, which also exposes a package named pipeline. news stays the source of truth for desk. Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
213 lines
8.5 KiB
Python
213 lines
8.5 KiB
Python
"""Structured article extraction, independent of fetching and persistence.
|
|
|
|
The tree is data, never HTML. Code is kept verbatim; plain text is a derived
|
|
view of the same tree. Custom extractors can return ArticleContent; callers
|
|
own persistence and renderers consume the closed vocabulary below.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import hashlib
|
|
from datetime import UTC, datetime
|
|
from html.parser import HTMLParser
|
|
from typing import Literal, Self
|
|
from urllib.parse import urljoin, urlsplit
|
|
from xml.etree import ElementTree as ET
|
|
|
|
from pydantic import Field, model_validator
|
|
|
|
from desk.models import Model
|
|
|
|
|
|
class ContentNode(Model):
|
|
kind: Literal[
|
|
"document", "text", "paragraph", "heading", "link", "code", "list",
|
|
"item", "table", "row", "cell", "strong", "emphasis", "quote", "break",
|
|
"span", "inline_code",
|
|
]
|
|
text: str = ""
|
|
children: list[ContentNode] = Field(default_factory=list)
|
|
level: int = Field(default=1, ge=1, le=6)
|
|
ordered: bool = False
|
|
header: bool = False
|
|
url: str = ""
|
|
|
|
@model_validator(mode="after")
|
|
def consistent_node(self) -> Self:
|
|
if self.kind in {"text", "code", "inline_code", "break"}:
|
|
if self.children:
|
|
raise ValueError("leaf nodes cannot have children")
|
|
elif self.text:
|
|
raise ValueError("container text belongs in text children")
|
|
if self.url and (self.kind != "link" or not _link(self.url, "")):
|
|
raise ValueError("only links can carry absolute HTTP(S) URLs")
|
|
return self
|
|
|
|
def plain_text(self) -> str:
|
|
if self.kind == "break":
|
|
return "\n"
|
|
if self.kind in {"code", "inline_code", "text"}:
|
|
return self.text
|
|
separator = {"document": "\n\n", "list": "\n", "table": "\n", "row": "\t"}.get(self.kind, "")
|
|
return separator.join(child.plain_text() for child in self.children)
|
|
|
|
|
|
class ArticleContent(Model):
|
|
format: Literal["plain", "structured-v1"]
|
|
content: str | ContentNode
|
|
source_url: str
|
|
method: str
|
|
acquired_at: str = Field(default_factory=lambda: datetime.now(UTC).isoformat())
|
|
# Finding a readable article does not prove that it was not truncated by
|
|
# the publisher. The caller must supply evidence before declaring it full.
|
|
completeness: Literal["unknown", "complete", "partial"] = "unknown"
|
|
removed_gutters: int = Field(default=0, ge=0)
|
|
|
|
@model_validator(mode="after")
|
|
def consistent_format(self) -> Self:
|
|
if self.format == "plain":
|
|
if not isinstance(self.content, str):
|
|
raise ValueError("plain content must be a string")
|
|
elif not isinstance(self.content, ContentNode) or self.content.kind != "document":
|
|
raise ValueError("structured-v1 content must be a document node")
|
|
return self
|
|
|
|
@property
|
|
def plain_text(self) -> str:
|
|
return self.content if isinstance(self.content, str) else self.content.plain_text()
|
|
|
|
@property
|
|
def revision(self) -> str:
|
|
value = self.content if isinstance(self.content, str) else self.content.model_dump_json()
|
|
return hashlib.sha256(f"{self.format}\n{value}".encode()).hexdigest()
|
|
|
|
|
|
class _Gutters(HTMLParser):
|
|
"""Remove only explicit code-gutter nodes, retaining their following text.
|
|
|
|
No numeric-line heuristics. Unknown publisher markup is preserved. The
|
|
void-element handling matters: a br inside a gutter must not eat the rest
|
|
of the article by leaving the suppression depth unbalanced.
|
|
"""
|
|
|
|
VOID = frozenset({"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"})
|
|
MARKERS = frozenset({"line-number", "line-numbers-rows", "lineno", "linenos"})
|
|
|
|
def __init__(self) -> None:
|
|
super().__init__(convert_charrefs=False)
|
|
self.parts: list[str] = []
|
|
self.stack: list[tuple[str, bool, bool]] = []
|
|
self.removed = 0
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
classes = set((dict(attrs).get("class") or "").split())
|
|
context = tag in {"pre", "code"} or bool(classes & {"highlight", "highlighttable"}) or any(c for _, c, _ in self.stack)
|
|
suppressed = any(s for _, _, s in self.stack)
|
|
gutter = context and bool(classes & self.MARKERS)
|
|
if gutter and not suppressed:
|
|
self.removed += 1
|
|
if not (gutter or suppressed):
|
|
self.parts.append(self.get_starttag_text() or "")
|
|
if tag not in self.VOID:
|
|
self.stack.append((tag, context, gutter or suppressed))
|
|
|
|
def handle_startendtag(self, tag, attrs):
|
|
self.handle_starttag(tag, attrs)
|
|
if tag not in self.VOID:
|
|
self.handle_endtag(tag)
|
|
|
|
def handle_endtag(self, tag):
|
|
suppressed = any(s for _, _, s in self.stack)
|
|
for index in range(len(self.stack) - 1, -1, -1):
|
|
if self.stack[index][0] == tag:
|
|
del self.stack[index:]
|
|
break
|
|
if not suppressed:
|
|
self.parts.append(f"</{tag}>")
|
|
|
|
def handle_data(self, data):
|
|
if not any(s for _, _, s in self.stack):
|
|
self.parts.append(data)
|
|
|
|
def handle_entityref(self, name):
|
|
self.handle_data(f"&{name};")
|
|
|
|
def handle_charref(self, name):
|
|
self.handle_data(f"&#{name};")
|
|
|
|
|
|
def _link(value: str, source_url: str) -> str:
|
|
try:
|
|
resolved = urljoin(source_url, value)
|
|
return resolved if urlsplit(resolved).scheme in {"http", "https"} else ""
|
|
except ValueError:
|
|
return ""
|
|
|
|
|
|
def _node(element: ET.Element, source_url: str) -> ContentNode:
|
|
children: list[ContentNode] = []
|
|
if element.text:
|
|
children.append(ContentNode(kind="text", text=element.text))
|
|
for child in element:
|
|
children.append(_node(child, source_url))
|
|
if child.tail:
|
|
children.append(ContentNode(kind="text", text=child.tail))
|
|
match element.tag:
|
|
case "code":
|
|
return ContentNode(kind="code", text="".join(element.itertext()))
|
|
case "head":
|
|
level = element.get("rend", "h1").removeprefix("h")
|
|
return ContentNode(kind="heading", level=int(level) if level in "123456" and len(level) == 1 else 1, children=children)
|
|
case "ref":
|
|
return ContentNode(kind="link", url=_link(element.get("target", ""), source_url), children=children)
|
|
case "list":
|
|
return ContentNode(kind="list", ordered=element.get("rend") == "ol", children=children)
|
|
case "cell":
|
|
return ContentNode(kind="cell", header=element.get("role") == "head", children=children)
|
|
case "hi":
|
|
if element.get("rend") == "#t":
|
|
return ContentNode(kind="inline_code", text="".join(element.itertext()))
|
|
kind = {"#b": "strong", "#i": "emphasis"}.get(element.get("rend", ""), "span")
|
|
return ContentNode.model_validate({"kind": kind, "children": children})
|
|
case "lb":
|
|
return ContentNode(kind="break")
|
|
case _:
|
|
kinds = {"main": "document", "p": "paragraph", "item": "item", "table": "table", "row": "row", "quote": "quote"}
|
|
return ContentNode.model_validate({"kind": kinds.get(element.tag, "paragraph"), "children": children})
|
|
|
|
|
|
def extract_article(html: str, source_url: str) -> ArticleContent | None:
|
|
"""Extract supplied HTML without I/O; return None when no article is found.
|
|
|
|
XML is an intermediate representation only. Its tags and attributes never
|
|
escape into the content contract, and no raw HTML is passed to a renderer.
|
|
"""
|
|
import trafilatura
|
|
|
|
cleaned = _Gutters()
|
|
cleaned.feed(html)
|
|
cleaned.close()
|
|
extracted = trafilatura.extract(
|
|
"".join(cleaned.parts), url=source_url, output_format="xml",
|
|
include_comments=False, include_tables=True, include_links=True,
|
|
include_formatting=True,
|
|
)
|
|
if not extracted:
|
|
return None
|
|
main = ET.fromstring(extracted).find("main")
|
|
if main is None:
|
|
return None
|
|
# XML pretty-printing inserts whitespace between blocks. It is not source
|
|
# content; inside code, however, every byte of whitespace belongs to it.
|
|
for element in main.iter():
|
|
if element.tag in {"main", "table", "row", "list"}:
|
|
if not (element.text or "").strip():
|
|
element.text = None
|
|
for child in element:
|
|
if not (child.tail or "").strip():
|
|
child.tail = None
|
|
return ArticleContent(
|
|
format="structured-v1", content=_node(main, source_url), source_url=source_url,
|
|
method="trafilatura-xml", removed_gutters=cleaned.removed,
|
|
)
|