Read searches through the runner, so a search has the 10% guard, the shared stamp and a search:failing record when it does not answer

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-09-30 15:53:47 +02:00
1 parent 36bc639384
commit 24a165818c
1 file changed
+59 -40
+59 -40
View File
@@ -17,14 +17,19 @@ one run is not closed here. A search that read everything that matches was
complete (`complete="stated"`, as many distinct results as the stated total), complete (`complete="stated"`, as many distinct results as the stated total),
and the pull closes what it stopped listing; the rest close once no search and the pull closes what it stopped listing; the rest close once no search
has seen them for `STALE_DAYS` (see `app/jobs/pull.py`). has seen them for `STALE_DAYS` (see `app/jobs/pull.py`).
In a pull a search is read by `_runner.read` under the prefix `search`, as a
board is under `ats`: the 10% guard holds it, `_runner.stamp` records it,
and a search that fails or is suspect is kept as failing under
`search:failing:<target>` (`_runner.record_failure`) until it answers, which
the settle reads as it reads a failing board's.
""" """
from __future__ import annotations from __future__ import annotations
import json
import logging import logging
import sys import sys
from collections.abc import Iterator, Mapping from collections.abc import Iterator, Mapping, MutableMapping
from datetime import UTC, datetime from datetime import UTC, datetime
from pathlib import Path from pathlib import Path
from types import ModuleType from types import ModuleType
@@ -65,8 +70,9 @@ def _pages(params: Mapping[str, Any]) -> int:
return DEFAULT_PAGES return DEFAULT_PAGES
def _limits(params: Mapping[str, Any]) -> _runner.Limits: def _limits(params: Mapping[str, Any], stale_days: int | None = None) -> _runner.Limits:
return _runner.Limits(max_pages=_pages(params), page_delay=PAGE_DELAY) return _runner.Limits(max_pages=_pages(params), page_delay=PAGE_DELAY,
stale_days=stale_days)
def provider_of(target: str) -> str: def provider_of(target: str) -> str:
@@ -76,10 +82,13 @@ def provider_of(target: str) -> str:
def _read(client: httpx.Client, target: str, params: Mapping[str, Any], def _read(client: httpx.Client, target: str, params: Mapping[str, Any],
keys: Mapping[str, Any] | None) -> tuple[ModuleType, _runner.Listing]: keys: Mapping[str, Any] | None, state: MutableMapping[str, str] | None = None,
stale_days: int | None = None) -> tuple[ModuleType, _runner.Listing]:
"""One search read through the runner: its provider and its listing. """One search read through the runner: its provider and its listing.
Raises ValueError for an unknown provider or a missing key, before any Raises ValueError for an unknown provider or a missing key, before any
request.""" request. With `state` (a pull) it is `_runner.read` under `search`, the
10% guard included and never swept; without (a search tried before it is
saved) a bare `read_listing`, since there is nothing to compare with."""
provider = provider_of(target) provider = provider_of(target)
if not provider: if not provider:
raise ValueError(f"{target!r} is not provider:name with provider one of " raise ValueError(f"{target!r} is not provider:name with provider one of "
@@ -88,7 +97,11 @@ def _read(client: httpx.Client, target: str, params: Mapping[str, Any],
slug = target.partition(":")[2] slug = target.partition(":")[2]
ask = _runner.ask_for(module, params, keys=keys) ask = _runner.ask_for(module, params, keys=keys)
get = _runner.getter(client, module.allowed(slug), provider) get = _runner.getter(client, module.allowed(slug), provider)
return module, _runner.read_listing(module, slug, get, ask, limits=_limits(params)) limits = _limits(params, stale_days)
if state is None:
return module, _runner.read_listing(module, slug, get, ask, limits=limits)
return module, _runner.read(state, "search", target, module, slug, get, ask,
limits=limits, can_sweep=False)
def run_search(target: str, params: dict[str, Any], def run_search(target: str, params: dict[str, Any],
@@ -130,37 +143,43 @@ class SearchSource(Source):
run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat( run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat(
timespec="seconds") timespec="seconds")
for target, label in self.targets(): for target, label in self.targets():
provider = provider_of(target) yield from self._search(client, target, label, run_marker)
if not provider:
log.warning("search: %r is not provider:name; skipped", target)
continue
try:
_module, listing = _read(client, target, self.params_for(target),
self.options)
except Exception as exc:
log.warning("search: %s failed (%s)", target, exc)
continue
seen_ids = [] def _search(self, client: httpx.Client, target: str, label: str,
for row in listing.rows: run_marker: str) -> Iterator[Item]:
source_id = f"{provider}:{row['source_id']}" """One search: read, its rows out as items, then stamped. A search
seen_ids.append(source_id) that fails, a suspect answer included, is recorded under
description = row.get("description") or "" `search:failing:<target>` and stamps nothing; the record goes once a
yield Item( read is stamped, after every row went out, as for a board."""
source_id=source_id, failing = f"search:failing:{target}"
location=row["url"], provider = provider_of(target)
title=row["title"], stale_days = self.options.get("stale_days")
excerpt=description[:300], try:
body=description, _module, listing = _read(client, target, self.params_for(target), self.options,
published_at=row.get("posted"), self.state, int(stale_days) if stale_days else None)
subscription=label, except Exception as exc:
kind="role", log.warning("search: %s failed (%s)", target, exc)
raw=_runner.raw_of(row) | { _runner.record_failure(self.state, failing, exc)
"company": row.get("company") or "", return
"provider": provider,
"search": target, now = datetime.now(UTC)
}, for row in listing.rows:
) description = row.get("description") or ""
self.state[f"search:ok:{target}"] = run_marker yield Item(
self.state[f"search:complete:{target}"] = "1" if listing.complete else "" source_id=f"{provider}:{row['source_id']}",
self.state[f"search:ids:{target}"] = json.dumps(seen_ids) location=row["url"],
title=row["title"],
excerpt=description[:300],
body=description,
published_at=row.get("posted") or now,
subscription=label,
kind="role",
raw=_runner.raw_of(row) | {
"company": row.get("company") or "",
"provider": provider,
"search": target,
},
)
_runner.stamp(self.state, "search", target, run_marker, listing,
[f"{provider}:{i}" for i in listing.ids])
self.state.pop(failing, None)