Runs is the queue's elements and a Problems log; a Kill lands between boards
Run one stage goes: the cascade is the unit of work and app job keeps the per-stage entry points. The pull button is disabled while a pull runs and the lede no longer hardcodes the cadence. Messages folds into Runs as Problems. pull_batch checks for a cancel before every board, one plugin per target, and both plugins now close their HTTP client. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
cebe62ddc9
commit
8532e4e739
15 files changed
+284
-242
No files matched your search
+42
-42
@@ -308,46 +308,46 @@ class SearchSource(Source):
|
||||
)
|
||||
|
||||
def fetch(self) -> Iterator[Item]:
|
||||
client = httpx.Client()
|
||||
run_marker = self.state.get("search:run", "") or datetime.now(UTC).isoformat(
|
||||
timespec="seconds")
|
||||
for target, label in self.targets():
|
||||
provider = provider_of(target)
|
||||
if not provider:
|
||||
log.warning("search: %r is not provider:name; skipped", target)
|
||||
continue
|
||||
try:
|
||||
rows, complete = _SEARCHES[provider](
|
||||
client, self.params_for(target), self.options.get(provider) or {})
|
||||
except Exception as exc:
|
||||
log.warning("search: %s failed (%s)", target, exc)
|
||||
continue
|
||||
with httpx.Client() as client:
|
||||
run_marker = self.state.get("search:run", "") or datetime.now(UTC).isoformat(
|
||||
timespec="seconds")
|
||||
for target, label in self.targets():
|
||||
provider = provider_of(target)
|
||||
if not provider:
|
||||
log.warning("search: %r is not provider:name; skipped", target)
|
||||
continue
|
||||
try:
|
||||
rows, complete = _SEARCHES[provider](
|
||||
client, self.params_for(target), self.options.get(provider) or {})
|
||||
except Exception as exc:
|
||||
log.warning("search: %s failed (%s)", target, exc)
|
||||
continue
|
||||
|
||||
seen_ids = []
|
||||
for row in rows:
|
||||
source_id = f"{provider}:{row['id']}"
|
||||
seen_ids.append(source_id)
|
||||
yield Item(
|
||||
source_id=source_id,
|
||||
location=row["url"],
|
||||
title=row["title"],
|
||||
excerpt=(row["description"] or "")[:300],
|
||||
body=row["description"],
|
||||
published_at=row["posted"],
|
||||
subscription=label,
|
||||
kind="role",
|
||||
raw={
|
||||
"company": row["company"],
|
||||
"place": row["place"],
|
||||
"country": row["country"],
|
||||
"remote": row["remote"],
|
||||
"salary_min": row["salary_min"],
|
||||
"salary_max": row["salary_max"],
|
||||
"currency": row["currency"],
|
||||
"provider": provider,
|
||||
"search": target,
|
||||
},
|
||||
)
|
||||
self.state[f"search:ok:{target}"] = run_marker
|
||||
self.state[f"search:complete:{target}"] = "1" if complete else ""
|
||||
self.state[f"search:ids:{target}"] = json.dumps(seen_ids)
|
||||
seen_ids = []
|
||||
for row in rows:
|
||||
source_id = f"{provider}:{row['id']}"
|
||||
seen_ids.append(source_id)
|
||||
yield Item(
|
||||
source_id=source_id,
|
||||
location=row["url"],
|
||||
title=row["title"],
|
||||
excerpt=(row["description"] or "")[:300],
|
||||
body=row["description"],
|
||||
published_at=row["posted"],
|
||||
subscription=label,
|
||||
kind="role",
|
||||
raw={
|
||||
"company": row["company"],
|
||||
"place": row["place"],
|
||||
"country": row["country"],
|
||||
"remote": row["remote"],
|
||||
"salary_min": row["salary_min"],
|
||||
"salary_max": row["salary_max"],
|
||||
"currency": row["currency"],
|
||||
"provider": provider,
|
||||
"search": target,
|
||||
},
|
||||
)
|
||||
self.state[f"search:ok:{target}"] = run_marker
|
||||
self.state[f"search:complete:{target}"] = "1" if complete else ""
|
||||
self.state[f"search:ids:{target}"] = json.dumps(seen_ids)
|
||||
Reference in new issue
Block a user