Read searches through the runner, so a search has the 10% guard, the shared stamp and a search:failing record when it does not answer
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
36bc639384
commit
24a165818c
1 file changed
+59
-40
+59
-40
@@ -17,14 +17,19 @@ one run is not closed here. A search that read everything that matches was
|
||||
complete (`complete="stated"`, as many distinct results as the stated total),
|
||||
and the pull closes what it stopped listing; the rest close once no search
|
||||
has seen them for `STALE_DAYS` (see `app/jobs/pull.py`).
|
||||
|
||||
In a pull a search is read by `_runner.read` under the prefix `search`, as a
|
||||
board is under `ats`: the 10% guard holds it, `_runner.stamp` records it,
|
||||
and a search that fails or is suspect is kept as failing under
|
||||
`search:failing:<target>` (`_runner.record_failure`) until it answers, which
|
||||
the settle reads as it reads a failing board's.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import sys
|
||||
from collections.abc import Iterator, Mapping
|
||||
from collections.abc import Iterator, Mapping, MutableMapping
|
||||
from datetime import UTC, datetime
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
@@ -65,8 +70,9 @@ def _pages(params: Mapping[str, Any]) -> int:
|
||||
return DEFAULT_PAGES
|
||||
|
||||
|
||||
def _limits(params: Mapping[str, Any]) -> _runner.Limits:
|
||||
return _runner.Limits(max_pages=_pages(params), page_delay=PAGE_DELAY)
|
||||
def _limits(params: Mapping[str, Any], stale_days: int | None = None) -> _runner.Limits:
|
||||
return _runner.Limits(max_pages=_pages(params), page_delay=PAGE_DELAY,
|
||||
stale_days=stale_days)
|
||||
|
||||
|
||||
def provider_of(target: str) -> str:
|
||||
@@ -76,10 +82,13 @@ def provider_of(target: str) -> str:
|
||||
|
||||
|
||||
def _read(client: httpx.Client, target: str, params: Mapping[str, Any],
|
||||
keys: Mapping[str, Any] | None) -> tuple[ModuleType, _runner.Listing]:
|
||||
keys: Mapping[str, Any] | None, state: MutableMapping[str, str] | None = None,
|
||||
stale_days: int | None = None) -> tuple[ModuleType, _runner.Listing]:
|
||||
"""One search read through the runner: its provider and its listing.
|
||||
Raises ValueError for an unknown provider or a missing key, before any
|
||||
request."""
|
||||
request. With `state` (a pull) it is `_runner.read` under `search`, the
|
||||
10% guard included and never swept; without (a search tried before it is
|
||||
saved) a bare `read_listing`, since there is nothing to compare with."""
|
||||
provider = provider_of(target)
|
||||
if not provider:
|
||||
raise ValueError(f"{target!r} is not provider:name with provider one of "
|
||||
@@ -88,7 +97,11 @@ def _read(client: httpx.Client, target: str, params: Mapping[str, Any],
|
||||
slug = target.partition(":")[2]
|
||||
ask = _runner.ask_for(module, params, keys=keys)
|
||||
get = _runner.getter(client, module.allowed(slug), provider)
|
||||
return module, _runner.read_listing(module, slug, get, ask, limits=_limits(params))
|
||||
limits = _limits(params, stale_days)
|
||||
if state is None:
|
||||
return module, _runner.read_listing(module, slug, get, ask, limits=limits)
|
||||
return module, _runner.read(state, "search", target, module, slug, get, ask,
|
||||
limits=limits, can_sweep=False)
|
||||
|
||||
|
||||
def run_search(target: str, params: dict[str, Any],
|
||||
@@ -130,37 +143,43 @@ class SearchSource(Source):
|
||||
run_marker = str(self.options.get("run") or "") or datetime.now(UTC).isoformat(
|
||||
timespec="seconds")
|
||||
for target, label in self.targets():
|
||||
provider = provider_of(target)
|
||||
if not provider:
|
||||
log.warning("search: %r is not provider:name; skipped", target)
|
||||
continue
|
||||
try:
|
||||
_module, listing = _read(client, target, self.params_for(target),
|
||||
self.options)
|
||||
except Exception as exc:
|
||||
log.warning("search: %s failed (%s)", target, exc)
|
||||
continue
|
||||
yield from self._search(client, target, label, run_marker)
|
||||
|
||||
seen_ids = []
|
||||
for row in listing.rows:
|
||||
source_id = f"{provider}:{row['source_id']}"
|
||||
seen_ids.append(source_id)
|
||||
description = row.get("description") or ""
|
||||
yield Item(
|
||||
source_id=source_id,
|
||||
location=row["url"],
|
||||
title=row["title"],
|
||||
excerpt=description[:300],
|
||||
body=description,
|
||||
published_at=row.get("posted"),
|
||||
subscription=label,
|
||||
kind="role",
|
||||
raw=_runner.raw_of(row) | {
|
||||
"company": row.get("company") or "",
|
||||
"provider": provider,
|
||||
"search": target,
|
||||
},
|
||||
)
|
||||
self.state[f"search:ok:{target}"] = run_marker
|
||||
self.state[f"search:complete:{target}"] = "1" if listing.complete else ""
|
||||
self.state[f"search:ids:{target}"] = json.dumps(seen_ids)
|
||||
def _search(self, client: httpx.Client, target: str, label: str,
|
||||
run_marker: str) -> Iterator[Item]:
|
||||
"""One search: read, its rows out as items, then stamped. A search
|
||||
that fails, a suspect answer included, is recorded under
|
||||
`search:failing:<target>` and stamps nothing; the record goes once a
|
||||
read is stamped, after every row went out, as for a board."""
|
||||
failing = f"search:failing:{target}"
|
||||
provider = provider_of(target)
|
||||
stale_days = self.options.get("stale_days")
|
||||
try:
|
||||
_module, listing = _read(client, target, self.params_for(target), self.options,
|
||||
self.state, int(stale_days) if stale_days else None)
|
||||
except Exception as exc:
|
||||
log.warning("search: %s failed (%s)", target, exc)
|
||||
_runner.record_failure(self.state, failing, exc)
|
||||
return
|
||||
|
||||
now = datetime.now(UTC)
|
||||
for row in listing.rows:
|
||||
description = row.get("description") or ""
|
||||
yield Item(
|
||||
source_id=f"{provider}:{row['source_id']}",
|
||||
location=row["url"],
|
||||
title=row["title"],
|
||||
excerpt=description[:300],
|
||||
body=description,
|
||||
published_at=row.get("posted") or now,
|
||||
subscription=label,
|
||||
kind="role",
|
||||
raw=_runner.raw_of(row) | {
|
||||
"company": row.get("company") or "",
|
||||
"provider": provider,
|
||||
"search": target,
|
||||
},
|
||||
)
|
||||
_runner.stamp(self.state, "search", target, run_marker, listing,
|
||||
[f"{provider}:{i}" for i in listing.ids])
|
||||
self.state.pop(failing, None)
|
||||
Reference in new issue
Block a user