Runs is the queue's elements and a Problems log; a Kill lands between boards
Run one stage goes: the cascade is the unit of work and app job keeps the per-stage entry points. The pull button is disabled while a pull runs and the lede no longer hardcodes the cadence. Messages folds into Runs as Problems. pull_batch checks for a cancel before every board, one plugin per target, and both plugins now close their HTTP client. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
cebe62ddc9
commit
8532e4e739
15 files changed
+284
-242
No files matched your search
+3
-3
@@ -142,9 +142,9 @@ def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str
|
||||
return {"started": " > ".join(CASCADES["pull"])}
|
||||
|
||||
|
||||
#: What `app job <name>` and the Pulls page offer besides `pull`: one stage
|
||||
#: What `app job <name>` and `/api/jobs` start besides `pull`: one stage
|
||||
#: each, on its own. `pull` is not a task here, since it is a job: a name that
|
||||
#: is both would be two buttons on the Pulls page.
|
||||
#: is both would be listed twice by `/api/jobs/kinds`.
|
||||
for _name in ("extract", "embed", "score", "summarise"):
|
||||
TASKS[_name] = partial(task, _name)
|
||||
|
||||
@@ -152,7 +152,7 @@ for _name in ("extract", "embed", "score", "summarise"):
|
||||
# ---------------------------------------------------------------- schedules
|
||||
|
||||
# The default, next to the job it fires. A starting point rather than a
|
||||
# setting: a cadence edited or switched off on the Pulls page survives
|
||||
# setting: a cadence edited or switched off under Runs survives
|
||||
# every restart (see pipeline/schedule.py).
|
||||
declare("pull", "pull", "every 6h",
|
||||
label="Pull every board, then embed, score, extract, rescore and summarise")
|
||||
+23
-24
@@ -78,41 +78,40 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
|
||||
they list.
|
||||
|
||||
`targets` is keyword-only without a default, which is what makes it a
|
||||
required payload key to the queue: the Pulls page offers no button for a
|
||||
batch that could only fail without one.
|
||||
required payload key to the queue: `/api/jobs/kinds` lists no batch that
|
||||
could only fail without one.
|
||||
|
||||
Builds the plugin scoped to just these targets — `Source.targets()` reads
|
||||
`options[subscription_field]`, so handing it only this batch's entries is
|
||||
what keeps one job from re-fetching every other job's boards.
|
||||
One plugin per target, scoped to just that entry — `Source.targets()`
|
||||
reads `options[subscription_field]` — so one job never re-fetches another
|
||||
job's boards, and `context.check()` runs before each fetch: a Kill lands
|
||||
between two boards rather than after the tenth.
|
||||
"""
|
||||
from plugins.ats import ATSSource
|
||||
from plugins.search import SearchSource
|
||||
|
||||
# Fetchable now, not when the pull was queued: a board switched off on the
|
||||
# Searches or Companies page since then is skipped.
|
||||
# Job boards or Company boards since then is skipped.
|
||||
subs = {s.target: s for s in store.subscriptions(source=source, fetchable=True)
|
||||
if s.target in targets}
|
||||
options = dict(config.sources.get(source) or {})
|
||||
plugin: ATSSource | SearchSource
|
||||
if source == "search":
|
||||
options["searches"] = [{"url": t, "label": subs[t].label, "params": subs[t].params}
|
||||
for t in targets if t in subs]
|
||||
plugin = SearchSource(options)
|
||||
else:
|
||||
options["boards"] = [{"url": t, "label": subs[t].company or subs[t].label}
|
||||
for t in targets if t in subs]
|
||||
plugin = ATSSource(options)
|
||||
plugin.state = store.state
|
||||
|
||||
stored = 0
|
||||
for item in plugin.fetch():
|
||||
item.source = source
|
||||
for target in targets:
|
||||
sub = subs.get(target)
|
||||
if sub is None:
|
||||
continue
|
||||
context.check()
|
||||
options = dict(config.sources.get(source) or {})
|
||||
plugin: ATSSource | SearchSource
|
||||
if source == "search":
|
||||
target = str(item.raw.get("search", ""))
|
||||
options["searches"] = [{"url": target, "label": sub.label, "params": sub.params}]
|
||||
plugin = SearchSource(options)
|
||||
else:
|
||||
target = f"{item.raw.get('board', '')}:{item.raw.get('slug', '')}"
|
||||
store.upsert_role(item, subs.get(target))
|
||||
stored += 1
|
||||
options["boards"] = [{"url": target, "label": sub.company or sub.label}]
|
||||
plugin = ATSSource(options)
|
||||
plugin.state = store.state
|
||||
for item in plugin.fetch():
|
||||
item.source = source
|
||||
store.upsert_role(item, sub)
|
||||
stored += 1
|
||||
return {"roles": stored}
|
||||
|
||||
|
||||
|
||||
@@ -186,7 +186,7 @@ def run_batch(config: Config, store: Store, *, context: RunContext = NULL,
|
||||
|
||||
|
||||
def summarise_task() -> Task:
|
||||
"""Finds nothing while `summarise.enabled` is false, so the task, the
|
||||
button on the Pulls page and the cascade stage are all no-ops with the
|
||||
switch off."""
|
||||
"""Finds nothing while `summarise.enabled` is false, so the task, `app
|
||||
job summarise` and the cascade stage are all no-ops with the switch
|
||||
off."""
|
||||
return replace(modeltask.of(TASK), label=TASK.label)
|
||||
Reference in new issue
Block a user