Files
jobboard/desk/tools/article.py
T
WolfandClaude Opus 5.5 248ea1636d Bring in desk from news
The plugin contract, the model task and the tools roledesk imports, copied
rather than reached through an editable install of news, which also exposes
a package named pipeline. news stays the source of truth for desk.

Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-23 12:29:17 +02:00

213 lines
8.5 KiB
Python

"""Structured article extraction, independent of fetching and persistence.
The tree is data, never HTML. Code is kept verbatim; plain text is a derived
view of the same tree. Custom extractors can return ArticleContent; callers
own persistence and renderers consume the closed vocabulary below.
"""
from __future__ import annotations
import hashlib
from datetime import UTC, datetime
from html.parser import HTMLParser
from typing import Literal, Self
from urllib.parse import urljoin, urlsplit
from xml.etree import ElementTree as ET
from pydantic import Field, model_validator
from desk.models import Model
class ContentNode(Model):
kind: Literal[
"document", "text", "paragraph", "heading", "link", "code", "list",
"item", "table", "row", "cell", "strong", "emphasis", "quote", "break",
"span", "inline_code",
]
text: str = ""
children: list[ContentNode] = Field(default_factory=list)
level: int = Field(default=1, ge=1, le=6)
ordered: bool = False
header: bool = False
url: str = ""
@model_validator(mode="after")
def consistent_node(self) -> Self:
if self.kind in {"text", "code", "inline_code", "break"}:
if self.children:
raise ValueError("leaf nodes cannot have children")
elif self.text:
raise ValueError("container text belongs in text children")
if self.url and (self.kind != "link" or not _link(self.url, "")):
raise ValueError("only links can carry absolute HTTP(S) URLs")
return self
def plain_text(self) -> str:
if self.kind == "break":
return "\n"
if self.kind in {"code", "inline_code", "text"}:
return self.text
separator = {"document": "\n\n", "list": "\n", "table": "\n", "row": "\t"}.get(self.kind, "")
return separator.join(child.plain_text() for child in self.children)
class ArticleContent(Model):
format: Literal["plain", "structured-v1"]
content: str | ContentNode
source_url: str
method: str
acquired_at: str = Field(default_factory=lambda: datetime.now(UTC).isoformat())
# Finding a readable article does not prove that it was not truncated by
# the publisher. The caller must supply evidence before declaring it full.
completeness: Literal["unknown", "complete", "partial"] = "unknown"
removed_gutters: int = Field(default=0, ge=0)
@model_validator(mode="after")
def consistent_format(self) -> Self:
if self.format == "plain":
if not isinstance(self.content, str):
raise ValueError("plain content must be a string")
elif not isinstance(self.content, ContentNode) or self.content.kind != "document":
raise ValueError("structured-v1 content must be a document node")
return self
@property
def plain_text(self) -> str:
return self.content if isinstance(self.content, str) else self.content.plain_text()
@property
def revision(self) -> str:
value = self.content if isinstance(self.content, str) else self.content.model_dump_json()
return hashlib.sha256(f"{self.format}\n{value}".encode()).hexdigest()
class _Gutters(HTMLParser):
"""Remove only explicit code-gutter nodes, retaining their following text.
No numeric-line heuristics. Unknown publisher markup is preserved. The
void-element handling matters: a br inside a gutter must not eat the rest
of the article by leaving the suppression depth unbalanced.
"""
VOID = frozenset({"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta", "param", "source", "track", "wbr"})
MARKERS = frozenset({"line-number", "line-numbers-rows", "lineno", "linenos"})
def __init__(self) -> None:
super().__init__(convert_charrefs=False)
self.parts: list[str] = []
self.stack: list[tuple[str, bool, bool]] = []
self.removed = 0
def handle_starttag(self, tag, attrs):
classes = set((dict(attrs).get("class") or "").split())
context = tag in {"pre", "code"} or bool(classes & {"highlight", "highlighttable"}) or any(c for _, c, _ in self.stack)
suppressed = any(s for _, _, s in self.stack)
gutter = context and bool(classes & self.MARKERS)
if gutter and not suppressed:
self.removed += 1
if not (gutter or suppressed):
self.parts.append(self.get_starttag_text() or "")
if tag not in self.VOID:
self.stack.append((tag, context, gutter or suppressed))
def handle_startendtag(self, tag, attrs):
self.handle_starttag(tag, attrs)
if tag not in self.VOID:
self.handle_endtag(tag)
def handle_endtag(self, tag):
suppressed = any(s for _, _, s in self.stack)
for index in range(len(self.stack) - 1, -1, -1):
if self.stack[index][0] == tag:
del self.stack[index:]
break
if not suppressed:
self.parts.append(f"</{tag}>")
def handle_data(self, data):
if not any(s for _, _, s in self.stack):
self.parts.append(data)
def handle_entityref(self, name):
self.handle_data(f"&{name};")
def handle_charref(self, name):
self.handle_data(f"&#{name};")
def _link(value: str, source_url: str) -> str:
try:
resolved = urljoin(source_url, value)
return resolved if urlsplit(resolved).scheme in {"http", "https"} else ""
except ValueError:
return ""
def _node(element: ET.Element, source_url: str) -> ContentNode:
children: list[ContentNode] = []
if element.text:
children.append(ContentNode(kind="text", text=element.text))
for child in element:
children.append(_node(child, source_url))
if child.tail:
children.append(ContentNode(kind="text", text=child.tail))
match element.tag:
case "code":
return ContentNode(kind="code", text="".join(element.itertext()))
case "head":
level = element.get("rend", "h1").removeprefix("h")
return ContentNode(kind="heading", level=int(level) if level in "123456" and len(level) == 1 else 1, children=children)
case "ref":
return ContentNode(kind="link", url=_link(element.get("target", ""), source_url), children=children)
case "list":
return ContentNode(kind="list", ordered=element.get("rend") == "ol", children=children)
case "cell":
return ContentNode(kind="cell", header=element.get("role") == "head", children=children)
case "hi":
if element.get("rend") == "#t":
return ContentNode(kind="inline_code", text="".join(element.itertext()))
kind = {"#b": "strong", "#i": "emphasis"}.get(element.get("rend", ""), "span")
return ContentNode.model_validate({"kind": kind, "children": children})
case "lb":
return ContentNode(kind="break")
case _:
kinds = {"main": "document", "p": "paragraph", "item": "item", "table": "table", "row": "row", "quote": "quote"}
return ContentNode.model_validate({"kind": kinds.get(element.tag, "paragraph"), "children": children})
def extract_article(html: str, source_url: str) -> ArticleContent | None:
"""Extract supplied HTML without I/O; return None when no article is found.
XML is an intermediate representation only. Its tags and attributes never
escape into the content contract, and no raw HTML is passed to a renderer.
"""
import trafilatura
cleaned = _Gutters()
cleaned.feed(html)
cleaned.close()
extracted = trafilatura.extract(
"".join(cleaned.parts), url=source_url, output_format="xml",
include_comments=False, include_tables=True, include_links=True,
include_formatting=True,
)
if not extracted:
return None
main = ET.fromstring(extracted).find("main")
if main is None:
return None
# XML pretty-printing inserts whitespace between blocks. It is not source
# content; inside code, however, every byte of whitespace belongs to it.
for element in main.iter():
if element.tag in {"main", "table", "row", "list"}:
if not (element.text or "").strip():
element.text = None
for child in element:
if not (child.tail or "").strip():
child.tail = None
return ArticleContent(
format="structured-v1", content=_node(main, source_url), source_url=source_url,
method="trafilatura-xml", removed_gutters=cleaned.removed,
)