Read searches through the runner, so a search has the 10% guard, the shared stamp and a search:failing record when it does not answer

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-09-30 15:53:47 +02:00
1 parent 36bc639384
commit 24a165818c
1 file changed
+59 -40
+59 -40
View File
@@ -17,14 +17,19 @@ one run is not closed here. A search that read everything that matches was
complete (`complete="stated"`, as many distinct results as the stated total),
and the pull closes what it stopped listing; the rest close once no search
has seen them for `STALE_DAYS` (see `app/jobs/pull.py`).
In a pull a search is read by `_runner.read` under the prefix `search`, as a
board is under `ats`: the 10% guard holds it, `_runner.stamp` records it,
and a search that fails or is suspect is kept as failing under
`search:failing:<target>` (`_runner.record_failure`) until it answers, which
the settle reads as it reads a failing board's.
"""
from __future__ import annotations
import json
import logging
import sys
from collections.abc import Iterator, Mapping
from collections.abc import Iterator, Mapping, MutableMapping
from datetime import UTC, datetime
from pathlib import Path
from types import ModuleType
@@ -65,8 +70,9 @@ def _pages(params: Mapping[str, Any]) -> int:
return DEFAULT_PAGES
def _limits(params: Mapping[str, Any]) -> _runner.Limits:
return _runner.Limits(max_pages=_pages(params), page_delay=PAGE_DELAY)
def _limits(params: Mapping[str, Any], stale_days: int | None = None) -> _runner.Limits:
return _runner.Limits(max_pages=_pages(params), page_delay=PAGE_DELAY,
stale_days=stale_days)
def provider_of(target: str) -> str:
@@ -76,10 +82,13 @@ def provider_of(target: str) -> str:
def _read(client: httpx.Client, target: str, params: Mapping[str, Any],
keys: Mapping[str, Any] | None) -> tuple[ModuleType, _runner.Listing]:
keys: Mapping[str, Any] | None, state: MutableMapping[str, str] | None = None,
stale_days: int | None = None) -> tuple[ModuleType, _runner.Listing]:
"""One search read through the runner: its provider and its listing.
Raises ValueError for an unknown provider or a missing key, before any
request."""
request. With `state` (a pull) it is `_runner.read` under `search`, the
10% guard included and never swept; without (a search tried before it is
saved) a bare `read_listing`, since there is nothing to compare with."""
provider = provider_of(target)
if not provider:
raise ValueError(f"{target!r} is not provider:name with provider one of "
@@ -88,7 +97,11 @@ def _read(client: httpx.Client, target: str, params: Mapping[str, Any],
slug = target.partition(":")[2]
ask = _runner.ask_for(module, params, keys=keys)
get = _runner.getter(client, module.allowed(slug), provider)
return module, _runner.read_listing(module, slug, get, ask, limits=_limits(params))
limits = _limits(params, stale_days)
if state is None:
return module, _runner.read_listing(module, slug, get, ask, limits=limits)
return module, _runner.read(state, "search", target, module, slug, get, ask,
limits=limits, can_sweep=False)
def run_search(target: str, params: dict[str, Any],
@@ -130,37 +143,43 @@ class SearchSource(Source):
run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat(
timespec="seconds")
for target, label in self.targets():
provider = provider_of(target)
if not provider:
log.warning("search: %r is not provider:name; skipped", target)
continue
try:
_module, listing = _read(client, target, self.params_for(target),
self.options)
except Exception as exc:
log.warning("search: %s failed (%s)", target, exc)
continue
yield from self._search(client, target, label, run_marker)
seen_ids = []
for row in listing.rows:
source_id = f"{provider}:{row['source_id']}"
seen_ids.append(source_id)
description = row.get("description") or ""
yield Item(
source_id=source_id,
location=row["url"],
title=row["title"],
excerpt=description[:300],
body=description,
published_at=row.get("posted"),
subscription=label,
kind="role",
raw=_runner.raw_of(row) | {
"company": row.get("company") or "",
"provider": provider,
"search": target,
},
)
self.state[f"search:ok:{target}"] = run_marker
self.state[f"search:complete:{target}"] = "1" if listing.complete else ""
self.state[f"search:ids:{target}"] = json.dumps(seen_ids)
def _search(self, client: httpx.Client, target: str, label: str,
run_marker: str) -> Iterator[Item]:
"""One search: read, its rows out as items, then stamped. A search
that fails, a suspect answer included, is recorded under
`search:failing:<target>` and stamps nothing; the record goes once a
read is stamped, after every row went out, as for a board."""
failing = f"search:failing:{target}"
provider = provider_of(target)
stale_days = self.options.get("stale_days")
try:
_module, listing = _read(client, target, self.params_for(target), self.options,
self.state, int(stale_days) if stale_days else None)
except Exception as exc:
log.warning("search: %s failed (%s)", target, exc)
_runner.record_failure(self.state, failing, exc)
return
now = datetime.now(UTC)
for row in listing.rows:
description = row.get("description") or ""
yield Item(
source_id=f"{provider}:{row['source_id']}",
location=row["url"],
title=row["title"],
excerpt=description[:300],
body=description,
published_at=row.get("posted") or now,
subscription=label,
kind="role",
raw=_runner.raw_of(row) | {
"company": row.get("company") or "",
"provider": provider,
"search": target,
},
)
_runner.stamp(self.state, "search", target, run_marker, listing,
[f"{provider}:{i}" for i in listing.ids])
self.state.pop(failing, None)