From 24a165818c7f4d72c54299a97ebdcd8fd68c2303 Mon Sep 17 00:00:00 2001 From: Wolf Date: Wed, 30 Sep 2026 15:53:47 +0200 Subject: [PATCH] Read searches through the runner, so a search has the 10% guard, the shared stamp and a search:failing record when it does not answer Co-Authored-By: Claude Fable 5.1 --- plugins/search.py | 99 ++++++++++++++++++++++++++++------------------- 1 file changed, 59 insertions(+), 40 deletions(-) diff --git a/plugins/search.py b/plugins/search.py index 01a163f..50baf9d 100644 --- a/plugins/search.py +++ b/plugins/search.py @@ -17,14 +17,19 @@ one run is not closed here. A search that read everything that matches was complete (`complete="stated"`, as many distinct results as the stated total), and the pull closes what it stopped listing; the rest close once no search has seen them for `STALE_DAYS` (see `app/jobs/pull.py`). + +In a pull a search is read by `_runner.read` under the prefix `search`, as a +board is under `ats`: the 10% guard holds it, `_runner.stamp` records it, +and a search that fails or is suspect is kept as failing under +`search:failing:` (`_runner.record_failure`) until it answers, which +the settle reads as it reads a failing board's. """ from __future__ import annotations -import json import logging import sys -from collections.abc import Iterator, Mapping +from collections.abc import Iterator, Mapping, MutableMapping from datetime import UTC, datetime from pathlib import Path from types import ModuleType @@ -65,8 +70,9 @@ def _pages(params: Mapping[str, Any]) -> int: return DEFAULT_PAGES -def _limits(params: Mapping[str, Any]) -> _runner.Limits: - return _runner.Limits(max_pages=_pages(params), page_delay=PAGE_DELAY) +def _limits(params: Mapping[str, Any], stale_days: int | None = None) -> _runner.Limits: + return _runner.Limits(max_pages=_pages(params), page_delay=PAGE_DELAY, + stale_days=stale_days) def provider_of(target: str) -> str: @@ -76,10 +82,13 @@ def provider_of(target: str) -> str: def _read(client: httpx.Client, target: str, params: Mapping[str, Any], - keys: Mapping[str, Any] | None) -> tuple[ModuleType, _runner.Listing]: + keys: Mapping[str, Any] | None, state: MutableMapping[str, str] | None = None, + stale_days: int | None = None) -> tuple[ModuleType, _runner.Listing]: """One search read through the runner: its provider and its listing. Raises ValueError for an unknown provider or a missing key, before any - request.""" + request. With `state` (a pull) it is `_runner.read` under `search`, the + 10% guard included and never swept; without (a search tried before it is + saved) a bare `read_listing`, since there is nothing to compare with.""" provider = provider_of(target) if not provider: raise ValueError(f"{target!r} is not provider:name with provider one of " @@ -88,7 +97,11 @@ def _read(client: httpx.Client, target: str, params: Mapping[str, Any], slug = target.partition(":")[2] ask = _runner.ask_for(module, params, keys=keys) get = _runner.getter(client, module.allowed(slug), provider) - return module, _runner.read_listing(module, slug, get, ask, limits=_limits(params)) + limits = _limits(params, stale_days) + if state is None: + return module, _runner.read_listing(module, slug, get, ask, limits=limits) + return module, _runner.read(state, "search", target, module, slug, get, ask, + limits=limits, can_sweep=False) def run_search(target: str, params: dict[str, Any], @@ -130,37 +143,43 @@ class SearchSource(Source): run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat( timespec="seconds") for target, label in self.targets(): - provider = provider_of(target) - if not provider: - log.warning("search: %r is not provider:name; skipped", target) - continue - try: - _module, listing = _read(client, target, self.params_for(target), - self.options) - except Exception as exc: - log.warning("search: %s failed (%s)", target, exc) - continue + yield from self._search(client, target, label, run_marker) - seen_ids = [] - for row in listing.rows: - source_id = f"{provider}:{row['source_id']}" - seen_ids.append(source_id) - description = row.get("description") or "" - yield Item( - source_id=source_id, - location=row["url"], - title=row["title"], - excerpt=description[:300], - body=description, - published_at=row.get("posted"), - subscription=label, - kind="role", - raw=_runner.raw_of(row) | { - "company": row.get("company") or "", - "provider": provider, - "search": target, - }, - ) - self.state[f"search:ok:{target}"] = run_marker - self.state[f"search:complete:{target}"] = "1" if listing.complete else "" - self.state[f"search:ids:{target}"] = json.dumps(seen_ids) + def _search(self, client: httpx.Client, target: str, label: str, + run_marker: str) -> Iterator[Item]: + """One search: read, its rows out as items, then stamped. A search + that fails, a suspect answer included, is recorded under + `search:failing:` and stamps nothing; the record goes once a + read is stamped, after every row went out, as for a board.""" + failing = f"search:failing:{target}" + provider = provider_of(target) + stale_days = self.options.get("stale_days") + try: + _module, listing = _read(client, target, self.params_for(target), self.options, + self.state, int(stale_days) if stale_days else None) + except Exception as exc: + log.warning("search: %s failed (%s)", target, exc) + _runner.record_failure(self.state, failing, exc) + return + + now = datetime.now(UTC) + for row in listing.rows: + description = row.get("description") or "" + yield Item( + source_id=f"{provider}:{row['source_id']}", + location=row["url"], + title=row["title"], + excerpt=description[:300], + body=description, + published_at=row.get("posted") or now, + subscription=label, + kind="role", + raw=_runner.raw_of(row) | { + "company": row.get("company") or "", + "provider": provider, + "search": target, + }, + ) + _runner.stamp(self.state, "search", target, run_marker, listing, + [f"{provider}:{i}" for i in listing.ids]) + self.state.pop(failing, None)