The plugin contract, the model task and the tools roledesk imports, copied rather than reached through an editable install of news, which also exposes a package named pipeline. news stays the source of truth for desk. Refreshed from news/desk at a275c38a1aa0fe80a7ec7dd5781ecccf683f71c6 Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
116 lines
4.3 KiB
Python
116 lines
4.3 KiB
Python
"""Turning an article into a vector.
|
|
|
|
A local sentence-transformer turns each article into ~384 floats. What it
|
|
buys, and why it is worth an optional 100MB download:
|
|
|
|
**It generalises across phrasings, which TF-IDF cannot.** The bag-of-words
|
|
model learns the token "semiconductor". It learns nothing at all from that
|
|
about an article on "chip fabrication", because the two share no words. The
|
|
embedding puts them next to each other, so a vote on one moves the other.
|
|
|
|
**It works from about five votes.** The taste vector is
|
|
`centroid(upvoted) - centroid(downvoted)` — a direction in the space, scored
|
|
by cosine. There is no training step, no fitting, and no threshold below which
|
|
it does nothing useful; it just gets sharper as votes arrive.
|
|
|
|
This module is only the encoder. Where the vectors go, and the arithmetic over
|
|
them, is `newsdesk/storage/vectors/`; which articles get encoded, and when, is
|
|
`newsdesk/jobs/ingest.py`.
|
|
|
|
`ranking.embedding_model` is recorded on every stored row, because a vector is
|
|
only comparable to another from the same model. Change the setting and the
|
|
old vectors are not wrong, they are *meaningless* — a different geometry
|
|
entirely — so they are dropped and re-computed rather than silently compared.
|
|
|
|
Not to be confused with `desk/tools/framing.py`, which is about whether
|
|
an outlet lets its pages be put in an iframe — that used to be called
|
|
`embed.py`, and the collision with `ranking.embedding_model` is why both were
|
|
renamed.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from collections.abc import Sequence
|
|
from typing import Protocol
|
|
|
|
import numpy as np
|
|
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
#: How much of the article body the encoder sees. These models truncate at a
|
|
#: few hundred tokens anyway, and the headline plus the opening paragraphs is
|
|
#: what actually says what a story is about.
|
|
BODY_CHARS = 1500
|
|
|
|
|
|
class Embedder(Protocol):
|
|
"""Anything that can turn strings into vectors.
|
|
|
|
A protocol rather than the concrete class, so the tests can hand the rest
|
|
of the system a deterministic encoder and never download a model. That is
|
|
not a testing hack — it is also what a future backend (an API embedder, a
|
|
different local library) would implement.
|
|
"""
|
|
|
|
model_id: str
|
|
|
|
def encode(self, texts: Sequence[str]) -> np.ndarray: ...
|
|
|
|
|
|
class SentenceTransformer:
|
|
"""The real one. Loads the model on first use, not on import.
|
|
|
|
A pull with `backend: tfidf` must not pay for a torch import, so
|
|
nothing here is imported at module level and this class is only constructed
|
|
by `load()`, which the embeddings path calls and the TF-IDF path does not.
|
|
"""
|
|
|
|
def __init__(self, model_id: str) -> None:
|
|
try:
|
|
from sentence_transformers import SentenceTransformer as ST
|
|
except ImportError as exc:
|
|
raise ImportError(
|
|
"ranking.backend is 'embeddings', which needs sentence-transformers.\n"
|
|
" uv sync --extra embeddings\n"
|
|
"or set ranking.backend back to 'tfidf' in config.yaml."
|
|
) from exc
|
|
self.model_id = model_id
|
|
log.info("loading embedding model %s (first run downloads it)", model_id)
|
|
self._model = ST(model_id)
|
|
|
|
def encode(self, texts: Sequence[str]) -> np.ndarray:
|
|
# Normalised on the way out, so cosine similarity is a dot product and
|
|
# every consumer can stop worrying about magnitudes.
|
|
return np.asarray(
|
|
self._model.encode(list(texts), normalize_embeddings=True,
|
|
show_progress_bar=False),
|
|
dtype=np.float32,
|
|
)
|
|
|
|
|
|
def load(model_id: str) -> Embedder:
|
|
return SentenceTransformer(model_id)
|
|
|
|
|
|
class Embeddable(Protocol):
|
|
"""What `document` reads: a title and a body, however the app stores them."""
|
|
|
|
title: str
|
|
excerpt: str
|
|
body: str
|
|
|
|
|
|
def document(row: Embeddable) -> str:
|
|
"""What gets embedded.
|
|
|
|
Title and body only — deliberately no `__source_` tokens. Those exist in
|
|
`rank._document` because TF-IDF has no other way to learn "I like this
|
|
outlet", and they would be noise here: the encoder would try to find
|
|
meaning in a made-up word. Outlet preference belongs in the linear model of
|
|
Stage 2, as its own one-hot feature, not smuggled into the text.
|
|
"""
|
|
body = row.body or row.excerpt
|
|
return f"{row.title}\n\n{' '.join((body or '').split())[:BODY_CHARS]}"
|