Read searches through the runner, so a search has the 10% guard, the shared stamp and a search:failing record when it does not answer

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-09-30 15:53:47 +02:00
1 parent 36bc639384
commit 24a165818c
1 file changed
+40 -21
+40 -21
View File
@@ -17,14 +17,19 @@ one run is not closed here. A search that read everything that matches was
complete (`complete="stated"`, as many distinct results as the stated total), complete (`complete="stated"`, as many distinct results as the stated total),
and the pull closes what it stopped listing; the rest close once no search and the pull closes what it stopped listing; the rest close once no search
has seen them for `STALE_DAYS` (see `app/jobs/pull.py`). has seen them for `STALE_DAYS` (see `app/jobs/pull.py`).
In a pull a search is read by `_runner.read` under the prefix `search`, as a
board is under `ats`: the 10% guard holds it, `_runner.stamp` records it,
and a search that fails or is suspect is kept as failing under
`search:failing:<target>` (`_runner.record_failure`) until it answers, which
the settle reads as it reads a failing board's.
""" """
from __future__ import annotations from __future__ import annotations
import json
import logging import logging
import sys import sys
from collections.abc import Iterator, Mapping from collections.abc import Iterator, Mapping, MutableMapping
from datetime import UTC, datetime from datetime import UTC, datetime
from pathlib import Path from pathlib import Path
from types import ModuleType from types import ModuleType
@@ -65,8 +70,9 @@ def _pages(params: Mapping[str, Any]) -> int:
return DEFAULT_PAGES return DEFAULT_PAGES
def _limits(params: Mapping[str, Any]) -> _runner.Limits: def _limits(params: Mapping[str, Any], stale_days: int | None = None) -> _runner.Limits:
return _runner.Limits(max_pages=_pages(params), page_delay=PAGE_DELAY) return _runner.Limits(max_pages=_pages(params), page_delay=PAGE_DELAY,
stale_days=stale_days)
def provider_of(target: str) -> str: def provider_of(target: str) -> str:
@@ -76,10 +82,13 @@ def provider_of(target: str) -> str:
def _read(client: httpx.Client, target: str, params: Mapping[str, Any], def _read(client: httpx.Client, target: str, params: Mapping[str, Any],
keys: Mapping[str, Any] | None) -> tuple[ModuleType, _runner.Listing]: keys: Mapping[str, Any] | None, state: MutableMapping[str, str] | None = None,
stale_days: int | None = None) -> tuple[ModuleType, _runner.Listing]:
"""One search read through the runner: its provider and its listing. """One search read through the runner: its provider and its listing.
Raises ValueError for an unknown provider or a missing key, before any Raises ValueError for an unknown provider or a missing key, before any
request.""" request. With `state` (a pull) it is `_runner.read` under `search`, the
10% guard included and never swept; without (a search tried before it is
saved) a bare `read_listing`, since there is nothing to compare with."""
provider = provider_of(target) provider = provider_of(target)
if not provider: if not provider:
raise ValueError(f"{target!r} is not provider:name with provider one of " raise ValueError(f"{target!r} is not provider:name with provider one of "
@@ -88,7 +97,11 @@ def _read(client: httpx.Client, target: str, params: Mapping[str, Any],
slug = target.partition(":")[2] slug = target.partition(":")[2]
ask = _runner.ask_for(module, params, keys=keys) ask = _runner.ask_for(module, params, keys=keys)
get = _runner.getter(client, module.allowed(slug), provider) get = _runner.getter(client, module.allowed(slug), provider)
return module, _runner.read_listing(module, slug, get, ask, limits=_limits(params)) limits = _limits(params, stale_days)
if state is None:
return module, _runner.read_listing(module, slug, get, ask, limits=limits)
return module, _runner.read(state, "search", target, module, slug, get, ask,
limits=limits, can_sweep=False)
def run_search(target: str, params: dict[str, Any], def run_search(target: str, params: dict[str, Any],
@@ -130,29 +143,35 @@ class SearchSource(Source):
run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat( run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat(
timespec="seconds") timespec="seconds")
for target, label in self.targets(): for target, label in self.targets():
yield from self._search(client, target, label, run_marker)
def _search(self, client: httpx.Client, target: str, label: str,
run_marker: str) -> Iterator[Item]:
"""One search: read, its rows out as items, then stamped. A search
that fails, a suspect answer included, is recorded under
`search:failing:<target>` and stamps nothing; the record goes once a
read is stamped, after every row went out, as for a board."""
failing = f"search:failing:{target}"
provider = provider_of(target) provider = provider_of(target)
if not provider: stale_days = self.options.get("stale_days")
log.warning("search: %r is not provider:name; skipped", target)
continue
try: try:
_module, listing = _read(client, target, self.params_for(target), _module, listing = _read(client, target, self.params_for(target), self.options,
self.options) self.state, int(stale_days) if stale_days else None)
except Exception as exc: except Exception as exc:
log.warning("search: %s failed (%s)", target, exc) log.warning("search: %s failed (%s)", target, exc)
continue _runner.record_failure(self.state, failing, exc)
return
seen_ids = [] now = datetime.now(UTC)
for row in listing.rows: for row in listing.rows:
source_id = f"{provider}:{row['source_id']}"
seen_ids.append(source_id)
description = row.get("description") or "" description = row.get("description") or ""
yield Item( yield Item(
source_id=source_id, source_id=f"{provider}:{row['source_id']}",
location=row["url"], location=row["url"],
title=row["title"], title=row["title"],
excerpt=description[:300], excerpt=description[:300],
body=description, body=description,
published_at=row.get("posted"), published_at=row.get("posted") or now,
subscription=label, subscription=label,
kind="role", kind="role",
raw=_runner.raw_of(row) | { raw=_runner.raw_of(row) | {
@@ -161,6 +180,6 @@ class SearchSource(Source):
"search": target, "search": target,
}, },
) )
self.state[f"search:ok:{target}"] = run_marker _runner.stamp(self.state, "search", target, run_marker, listing,
self.state[f"search:complete:{target}"] = "1" if listing.complete else "" [f"{provider}:{i}" for i in listing.ids])
self.state[f"search:ids:{target}"] = json.dumps(seen_ids) self.state.pop(failing, None)