Bring in desk from news
The plugin contract, the model task and the tools roledesk imports, copied rather than reached through an editable install of news, which also exposes a package named pipeline. news stays the source of truth for desk. Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
9d666d0937
commit
248ea1636d
36 files changed
+8393
No files matched your search
@@ -0,0 +1,212 @@
|
||||
"""Structured article extraction, independent of fetching and persistence.
|
||||
|
||||
The tree is data, never HTML. Code is kept verbatim; plain text is a derived
|
||||
view of the same tree. Custom extractors can return ArticleContent; callers
|
||||
own persistence and renderers consume the closed vocabulary below.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from datetime import UTC, datetime
|
||||
from html.parser import HTMLParser
|
||||
from typing import Literal, Self
|
||||
from urllib.parse import urljoin, urlsplit
|
||||
from xml.etree import ElementTree as ET
|
||||
|
||||
from pydantic import Field, model_validator
|
||||
|
||||
from desk.models import Model
|
||||
|
||||
|
||||
class ContentNode(Model):
|
||||
kind: Literal[
|
||||
"document", "text", "paragraph", "heading", "link", "code", "list",
|
||||
"item", "table", "row", "cell", "strong", "emphasis", "quote", "break",
|
||||
"span", "inline_code",
|
||||
]
|
||||
text: str = ""
|
||||
children: list[ContentNode] = Field(default_factory=list)
|
||||
level: int = Field(default=1, ge=1, le=6)
|
||||
ordered: bool = False
|
||||
header: bool = False
|
||||
url: str = ""
|
||||
|
||||
@model_validator(mode="after")
|
||||
def consistent_node(self) -> Self:
|
||||
if self.kind in {"text", "code", "inline_code", "break"}:
|
||||
if self.children:
|
||||
raise ValueError("leaf nodes cannot have children")
|
||||
elif self.text:
|
||||
raise ValueError("container text belongs in text children")
|
||||
if self.url and (self.kind != "link" or not _link(self.url, "")):
|
||||
raise ValueError("only links can carry absolute HTTP(S) URLs")
|
||||
return self
|
||||
|
||||
def plain_text(self) -> str:
|
||||
if self.kind == "break":
|
||||
return "\n"
|
||||
if self.kind in {"code", "inline_code", "text"}:
|
||||
return self.text
|
||||
separator = {"document": "\n\n", "list": "\n", "table": "\n", "row": "\t"}.get(self.kind, "")
|
||||
return separator.join(child.plain_text() for child in self.children)
|
||||
|
||||
|
||||
class ArticleContent(Model):
|
||||
format: Literal["plain", "structured-v1"]
|
||||
content: str | ContentNode
|
||||
source_url: str
|
||||
method: str
|
||||
acquired_at: str = Field(default_factory=lambda: datetime.now(UTC).isoformat())
|
||||
# Finding a readable article does not prove that it was not truncated by
|
||||
# the publisher. The caller must supply evidence before declaring it full.
|
||||
completeness: Literal["unknown", "complete", "partial"] = "unknown"
|
||||
removed_gutters: int = Field(default=0, ge=0)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def consistent_format(self) -> Self:
|
||||
if self.format == "plain":
|
||||
if not isinstance(self.content, str):
|
||||
raise ValueError("plain content must be a string")
|
||||
elif not isinstance(self.content, ContentNode) or self.content.kind != "document":
|
||||
raise ValueError("structured-v1 content must be a document node")
|
||||
return self
|
||||
|
||||
@property
|
||||
def plain_text(self) -> str:
|
||||
return self.content if isinstance(self.content, str) else self.content.plain_text()
|
||||
|
||||
@property
|
||||
def revision(self) -> str:
|
||||
value = self.content if isinstance(self.content, str) else self.content.model_dump_json()
|
||||
return hashlib.sha256(f"{self.format}\n{value}".encode()).hexdigest()
|
||||
|
||||
|
||||
class _Gutters(HTMLParser):
|
||||
"""Remove only explicit code-gutter nodes, retaining their following text.
|
||||
|
||||
No numeric-line heuristics. Unknown publisher markup is preserved. The
|
||||
void-element handling matters: a br inside a gutter must not eat the rest
|
||||
of the article by leaving the suppression depth unbalanced.
|
||||
"""
|
||||
|
||||
VOID = frozenset({"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"})
|
||||
MARKERS = frozenset({"line-number", "line-numbers-rows", "lineno", "linenos"})
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(convert_charrefs=False)
|
||||
self.parts: list[str] = []
|
||||
self.stack: list[tuple[str, bool, bool]] = []
|
||||
self.removed = 0
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
classes = set((dict(attrs).get("class") or "").split())
|
||||
context = tag in {"pre", "code"} or bool(classes & {"highlight", "highlighttable"}) or any(c for _, c, _ in self.stack)
|
||||
suppressed = any(s for _, _, s in self.stack)
|
||||
gutter = context and bool(classes & self.MARKERS)
|
||||
if gutter and not suppressed:
|
||||
self.removed += 1
|
||||
if not (gutter or suppressed):
|
||||
self.parts.append(self.get_starttag_text() or "")
|
||||
if tag not in self.VOID:
|
||||
self.stack.append((tag, context, gutter or suppressed))
|
||||
|
||||
def handle_startendtag(self, tag, attrs):
|
||||
self.handle_starttag(tag, attrs)
|
||||
if tag not in self.VOID:
|
||||
self.handle_endtag(tag)
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
suppressed = any(s for _, _, s in self.stack)
|
||||
for index in range(len(self.stack) - 1, -1, -1):
|
||||
if self.stack[index][0] == tag:
|
||||
del self.stack[index:]
|
||||
break
|
||||
if not suppressed:
|
||||
self.parts.append(f"</{tag}>")
|
||||
|
||||
def handle_data(self, data):
|
||||
if not any(s for _, _, s in self.stack):
|
||||
self.parts.append(data)
|
||||
|
||||
def handle_entityref(self, name):
|
||||
self.handle_data(f"&{name};")
|
||||
|
||||
def handle_charref(self, name):
|
||||
self.handle_data(f"&#{name};")
|
||||
|
||||
|
||||
def _link(value: str, source_url: str) -> str:
|
||||
try:
|
||||
resolved = urljoin(source_url, value)
|
||||
return resolved if urlsplit(resolved).scheme in {"http", "https"} else ""
|
||||
except ValueError:
|
||||
return ""
|
||||
|
||||
|
||||
def _node(element: ET.Element, source_url: str) -> ContentNode:
|
||||
children: list[ContentNode] = []
|
||||
if element.text:
|
||||
children.append(ContentNode(kind="text", text=element.text))
|
||||
for child in element:
|
||||
children.append(_node(child, source_url))
|
||||
if child.tail:
|
||||
children.append(ContentNode(kind="text", text=child.tail))
|
||||
match element.tag:
|
||||
case "code":
|
||||
return ContentNode(kind="code", text="".join(element.itertext()))
|
||||
case "head":
|
||||
level = element.get("rend", "h1").removeprefix("h")
|
||||
return ContentNode(kind="heading", level=int(level) if level in "123456" and len(level) == 1 else 1, children=children)
|
||||
case "ref":
|
||||
return ContentNode(kind="link", url=_link(element.get("target", ""), source_url), children=children)
|
||||
case "list":
|
||||
return ContentNode(kind="list", ordered=element.get("rend") == "ol", children=children)
|
||||
case "cell":
|
||||
return ContentNode(kind="cell", header=element.get("role") == "head", children=children)
|
||||
case "hi":
|
||||
if element.get("rend") == "#t":
|
||||
return ContentNode(kind="inline_code", text="".join(element.itertext()))
|
||||
kind = {"#b": "strong", "#i": "emphasis"}.get(element.get("rend", ""), "span")
|
||||
return ContentNode.model_validate({"kind": kind, "children": children})
|
||||
case "lb":
|
||||
return ContentNode(kind="break")
|
||||
case _:
|
||||
kinds = {"main": "document", "p": "paragraph", "item": "item", "table": "table", "row": "row", "quote": "quote"}
|
||||
return ContentNode.model_validate({"kind": kinds.get(element.tag, "paragraph"), "children": children})
|
||||
|
||||
|
||||
def extract_article(html: str, source_url: str) -> ArticleContent | None:
|
||||
"""Extract supplied HTML without I/O; return None when no article is found.
|
||||
|
||||
XML is an intermediate representation only. Its tags and attributes never
|
||||
escape into the content contract, and no raw HTML is passed to a renderer.
|
||||
"""
|
||||
import trafilatura
|
||||
|
||||
cleaned = _Gutters()
|
||||
cleaned.feed(html)
|
||||
cleaned.close()
|
||||
extracted = trafilatura.extract(
|
||||
"".join(cleaned.parts), url=source_url, output_format="xml",
|
||||
include_comments=False, include_tables=True, include_links=True,
|
||||
include_formatting=True,
|
||||
)
|
||||
if not extracted:
|
||||
return None
|
||||
main = ET.fromstring(extracted).find("main")
|
||||
if main is None:
|
||||
return None
|
||||
# XML pretty-printing inserts whitespace between blocks. It is not source
|
||||
# content; inside code, however, every byte of whitespace belongs to it.
|
||||
for element in main.iter():
|
||||
if element.tag in {"main", "table", "row", "list"}:
|
||||
if not (element.text or "").strip():
|
||||
element.text = None
|
||||
for child in element:
|
||||
if not (child.tail or "").strip():
|
||||
child.tail = None
|
||||
return ArticleContent(
|
||||
format="structured-v1", content=_node(main, source_url), source_url=source_url,
|
||||
method="trafilatura-xml", removed_gutters=cleaned.removed,
|
||||
)
|
||||
Reference in new issue
Block a user