Runs is the queue's elements and a Problems log; a Kill lands between boards

Run one stage goes: the cascade is the unit of work and app job keeps the
per-stage entry points. The pull button is disabled while a pull runs and
the lede no longer hardcodes the cadence. Messages folds into Runs as
Problems. pull_batch checks for a cancel before every board, one plugin per
target, and both plugins now close their HTTP client.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-09-28 16:10:38 +02:00
1 parent cebe62ddc9
commit 8532e4e739
15 files changed
+284 -242

No files matched your search

+3 -3
View File
@@ -142,9 +142,9 @@ def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str
return {"started": " > ".join(CASCADES["pull"])}
#: What `app job <name>` and the Pulls page offer besides `pull`: one stage
#: What `app job <name>` and `/api/jobs` start besides `pull`: one stage
#: each, on its own. `pull` is not a task here, since it is a job: a name that
#: is both would be two buttons on the Pulls page.
#: is both would be listed twice by `/api/jobs/kinds`.
for _name in ("extract", "embed", "score", "summarise"):
TASKS[_name] = partial(task, _name)
@@ -152,7 +152,7 @@ for _name in ("extract", "embed", "score", "summarise"):
# ---------------------------------------------------------------- schedules
# The default, next to the job it fires. A starting point rather than a
# setting: a cadence edited or switched off on the Pulls page survives
# setting: a cadence edited or switched off under Runs survives
# every restart (see pipeline/schedule.py).
declare("pull", "pull", "every 6h",
label="Pull every board, then embed, score, extract, rescore and summarise")
+23 -24
View File
@@ -78,41 +78,40 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
they list.
`targets` is keyword-only without a default, which is what makes it a
required payload key to the queue: the Pulls page offers no button for a
batch that could only fail without one.
required payload key to the queue: `/api/jobs/kinds` lists no batch that
could only fail without one.
Builds the plugin scoped to just these targets — `Source.targets()` reads
`options[subscription_field]`, so handing it only this batch's entries is
what keeps one job from re-fetching every other job's boards.
One plugin per target, scoped to just that entry — `Source.targets()`
reads `options[subscription_field]` — so one job never re-fetches another
job's boards, and `context.check()` runs before each fetch: a Kill lands
between two boards rather than after the tenth.
"""
from plugins.ats import ATSSource
from plugins.search import SearchSource
# Fetchable now, not when the pull was queued: a board switched off on the
# Searches or Companies page since then is skipped.
# Job boards or Company boards since then is skipped.
subs = {s.target: s for s in store.subscriptions(source=source, fetchable=True)
if s.target in targets}
options = dict(config.sources.get(source) or {})
plugin: ATSSource | SearchSource
if source == "search":
options["searches"] = [{"url": t, "label": subs[t].label, "params": subs[t].params}
for t in targets if t in subs]
plugin = SearchSource(options)
else:
options["boards"] = [{"url": t, "label": subs[t].company or subs[t].label}
for t in targets if t in subs]
plugin = ATSSource(options)
plugin.state = store.state
stored = 0
for item in plugin.fetch():
item.source = source
for target in targets:
sub = subs.get(target)
if sub is None:
continue
context.check()
options = dict(config.sources.get(source) or {})
plugin: ATSSource | SearchSource
if source == "search":
target = str(item.raw.get("search", ""))
options["searches"] = [{"url": target, "label": sub.label, "params": sub.params}]
plugin = SearchSource(options)
else:
target = f"{item.raw.get('board', '')}:{item.raw.get('slug', '')}"
store.upsert_role(item, subs.get(target))
stored += 1
options["boards"] = [{"url": target, "label": sub.company or sub.label}]
plugin = ATSSource(options)
plugin.state = store.state
for item in plugin.fetch():
item.source = source
store.upsert_role(item, sub)
stored += 1
return {"roles": stored}
+3 -3
View File
@@ -186,7 +186,7 @@ def run_batch(config: Config, store: Store, *, context: RunContext = NULL,
def summarise_task() -> Task:
"""Finds nothing while `summarise.enabled` is false, so the task, the
button on the Pulls page and the cascade stage are all no-ops with the
switch off."""
"""Finds nothing while `summarise.enabled` is false, so the task, `app
job summarise` and the cascade stage are all no-ops with the switch
off."""
return replace(modeltask.of(TASK), label=TASK.label)