Runs is the queue's elements and a Problems log; a Kill lands between boards
Run one stage goes: the cascade is the unit of work and app job keeps the per-stage entry points. The pull button is disabled while a pull runs and the lede no longer hardcodes the cadence. Messages folds into Runs as Problems. pull_batch checks for a cancel before every board, one plugin per target, and both plugins now close their HTTP client. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
cebe62ddc9
commit
8532e4e739
15 files changed
+284
-242
No files matched your search
+3
-3
@@ -142,9 +142,9 @@ def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str
|
||||
return {"started": " > ".join(CASCADES["pull"])}
|
||||
|
||||
|
||||
#: What `app job <name>` and the Pulls page offer besides `pull`: one stage
|
||||
#: What `app job <name>` and `/api/jobs` start besides `pull`: one stage
|
||||
#: each, on its own. `pull` is not a task here, since it is a job: a name that
|
||||
#: is both would be two buttons on the Pulls page.
|
||||
#: is both would be listed twice by `/api/jobs/kinds`.
|
||||
for _name in ("extract", "embed", "score", "summarise"):
|
||||
TASKS[_name] = partial(task, _name)
|
||||
|
||||
@@ -152,7 +152,7 @@ for _name in ("extract", "embed", "score", "summarise"):
|
||||
# ---------------------------------------------------------------- schedules
|
||||
|
||||
# The default, next to the job it fires. A starting point rather than a
|
||||
# setting: a cadence edited or switched off on the Pulls page survives
|
||||
# setting: a cadence edited or switched off under Runs survives
|
||||
# every restart (see pipeline/schedule.py).
|
||||
declare("pull", "pull", "every 6h",
|
||||
label="Pull every board, then embed, score, extract, rescore and summarise")
|
||||
+23
-24
@@ -78,41 +78,40 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
|
||||
they list.
|
||||
|
||||
`targets` is keyword-only without a default, which is what makes it a
|
||||
required payload key to the queue: the Pulls page offers no button for a
|
||||
batch that could only fail without one.
|
||||
required payload key to the queue: `/api/jobs/kinds` lists no batch that
|
||||
could only fail without one.
|
||||
|
||||
Builds the plugin scoped to just these targets — `Source.targets()` reads
|
||||
`options[subscription_field]`, so handing it only this batch's entries is
|
||||
what keeps one job from re-fetching every other job's boards.
|
||||
One plugin per target, scoped to just that entry — `Source.targets()`
|
||||
reads `options[subscription_field]` — so one job never re-fetches another
|
||||
job's boards, and `context.check()` runs before each fetch: a Kill lands
|
||||
between two boards rather than after the tenth.
|
||||
"""
|
||||
from plugins.ats import ATSSource
|
||||
from plugins.search import SearchSource
|
||||
|
||||
# Fetchable now, not when the pull was queued: a board switched off on the
|
||||
# Searches or Companies page since then is skipped.
|
||||
# Job boards or Company boards since then is skipped.
|
||||
subs = {s.target: s for s in store.subscriptions(source=source, fetchable=True)
|
||||
if s.target in targets}
|
||||
options = dict(config.sources.get(source) or {})
|
||||
plugin: ATSSource | SearchSource
|
||||
if source == "search":
|
||||
options["searches"] = [{"url": t, "label": subs[t].label, "params": subs[t].params}
|
||||
for t in targets if t in subs]
|
||||
plugin = SearchSource(options)
|
||||
else:
|
||||
options["boards"] = [{"url": t, "label": subs[t].company or subs[t].label}
|
||||
for t in targets if t in subs]
|
||||
plugin = ATSSource(options)
|
||||
plugin.state = store.state
|
||||
|
||||
stored = 0
|
||||
for item in plugin.fetch():
|
||||
item.source = source
|
||||
for target in targets:
|
||||
sub = subs.get(target)
|
||||
if sub is None:
|
||||
continue
|
||||
context.check()
|
||||
options = dict(config.sources.get(source) or {})
|
||||
plugin: ATSSource | SearchSource
|
||||
if source == "search":
|
||||
target = str(item.raw.get("search", ""))
|
||||
options["searches"] = [{"url": target, "label": sub.label, "params": sub.params}]
|
||||
plugin = SearchSource(options)
|
||||
else:
|
||||
target = f"{item.raw.get('board', '')}:{item.raw.get('slug', '')}"
|
||||
store.upsert_role(item, subs.get(target))
|
||||
stored += 1
|
||||
options["boards"] = [{"url": target, "label": sub.company or sub.label}]
|
||||
plugin = ATSSource(options)
|
||||
plugin.state = store.state
|
||||
for item in plugin.fetch():
|
||||
item.source = source
|
||||
store.upsert_role(item, sub)
|
||||
stored += 1
|
||||
return {"roles": stored}
|
||||
|
||||
|
||||
|
||||
@@ -186,7 +186,7 @@ def run_batch(config: Config, store: Store, *, context: RunContext = NULL,
|
||||
|
||||
|
||||
def summarise_task() -> Task:
|
||||
"""Finds nothing while `summarise.enabled` is false, so the task, the
|
||||
button on the Pulls page and the cascade stage are all no-ops with the
|
||||
switch off."""
|
||||
"""Finds nothing while `summarise.enabled` is false, so the task, `app
|
||||
job summarise` and the cascade stage are all no-ops with the switch
|
||||
off."""
|
||||
return replace(modeltask.of(TASK), label=TASK.label)
|
||||
@@ -275,13 +275,6 @@ button.secondary {
|
||||
font-size: 0.8rem;
|
||||
}
|
||||
|
||||
/* -- pulls ------------------------------------------------------------------ */
|
||||
/* "Run one stage" on the Pulls page: extract, embed and score on their own,
|
||||
under the schedule rather than beside Pull now. */
|
||||
|
||||
details.stage summary { cursor: pointer; color: var(--dim); font-size: 0.95rem; font-weight: 600; }
|
||||
details.stage[open] summary { margin-bottom: 0.75rem; }
|
||||
|
||||
/* -- the notice log ------------------------------------------------------- */
|
||||
|
||||
.notice {
|
||||
|
||||
+50
-55
@@ -1,11 +1,12 @@
|
||||
/* The Runs section of Sources: Pull now on top, one button per remaining
|
||||
* kind under "Run one stage".
|
||||
/* The Runs section of Sources: "Run the pull now" and the idle line.
|
||||
*
|
||||
* The list comes from `/api/jobs/kinds`, which is the registry, so a new
|
||||
* `@job` in `app/jobs.py` is a new button and this file is not edited --
|
||||
* except that `pull` itself is pulled out of the list and wired to the
|
||||
* dedicated button instead of appearing twice. What the queue is doing is
|
||||
* `<desk-jobs>`'s business, and it draws itself.
|
||||
* What the queue is doing and when it runs next are `<desk-jobs>`' and
|
||||
* `<desk-schedules>`' business, and they draw themselves. The single stages
|
||||
* have no buttons: the cascade is the unit of work, and `app job <stage>`
|
||||
* keeps the per-stage entry points.
|
||||
*
|
||||
* Both read the `app:jobs` snapshot from live.js rather than asking the
|
||||
* elements, which keep to a shadow root and owe nobody an answer.
|
||||
*
|
||||
* Failures are the server's sentence through `api()`, which toasts it -- so
|
||||
* there is no error handling here beyond keeping the button usable.
|
||||
@@ -14,63 +15,57 @@
|
||||
(function () {
|
||||
"use strict";
|
||||
|
||||
const host = document.getElementById("kinds");
|
||||
const pullButton = document.getElementById("pull-now");
|
||||
|
||||
async function start(kind, button) {
|
||||
button.disabled = true;
|
||||
try {
|
||||
const job = await window.api("/api/jobs", { method: "POST", body: { kind } });
|
||||
window.toast.good(`${job.label || kind}: queued.`);
|
||||
} catch (e) {
|
||||
/* already toasted */
|
||||
} finally {
|
||||
button.disabled = false;
|
||||
}
|
||||
let pending = false; // the POST is in the air
|
||||
let pulling = false; // the latest snapshot has a live pull cascade
|
||||
|
||||
/* A pull queued by the button or a schedule is the `pull` job, which
|
||||
begins a cascade of its own named `pull on its own`, and the chain's
|
||||
tasks join it; `pull` is the name a cascade started as one would have.
|
||||
Live while any task has a job queued or running: the pipeline keeps a
|
||||
finished cascade in the snapshot for a few seconds, and that is not a
|
||||
pull still going. As newsdesk's Fetch: the whole cascade, not the `pull`
|
||||
task, which is over seconds in with five tasks to go. */
|
||||
const isPull = (c) => c.name === "pull" || c.name === "pull on its own";
|
||||
const live = (c) => (c.tasks || []).some((t) => t.running || t.queued);
|
||||
|
||||
function paint() {
|
||||
if (!pullButton) return;
|
||||
pullButton.disabled = pending || pulling;
|
||||
pullButton.title = pulling ? "A pull is running." : "";
|
||||
}
|
||||
|
||||
if (pullButton) {
|
||||
pullButton.addEventListener("click", () => start("pull", pullButton));
|
||||
}
|
||||
|
||||
async function draw() {
|
||||
if (!host) return;
|
||||
let kinds;
|
||||
try {
|
||||
kinds = await window.api("/api/jobs/kinds");
|
||||
} catch (e) {
|
||||
host.innerHTML = '<span class="muted">could not read the job kinds</span>';
|
||||
return;
|
||||
}
|
||||
kinds = kinds.filter((k) => k.kind !== "pull");
|
||||
if (!kinds.length) {
|
||||
host.innerHTML = '<span class="muted">nothing is registered</span>';
|
||||
return;
|
||||
}
|
||||
host.innerHTML = "";
|
||||
kinds.forEach((k) => {
|
||||
const button = document.createElement("button");
|
||||
button.type = "button";
|
||||
button.className = "secondary";
|
||||
button.textContent = k.label || k.kind;
|
||||
button.title = k.solo ? `${k.kind} -- one at a time` : k.kind;
|
||||
button.dataset.kind = k.kind;
|
||||
button.addEventListener("click", () => start(k.kind, button));
|
||||
host.appendChild(button);
|
||||
pullButton.addEventListener("click", async () => {
|
||||
pending = true;
|
||||
paint();
|
||||
try {
|
||||
const job = await window.api("/api/jobs", { method: "POST", body: { kind: "pull" } });
|
||||
window.toast.good(`${job.label || "pull"}: queued.`);
|
||||
} catch (e) {
|
||||
/* already toasted */
|
||||
} finally {
|
||||
pending = false;
|
||||
paint();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/* `<desk-jobs>` takes up no space while the queue is idle, so the line that
|
||||
says so is the app's to show and hide. Off the same socket the element
|
||||
reads, through live.js, rather than by asking the element: it keeps to a
|
||||
shadow root and owes nobody an answer about its own height. */
|
||||
document.addEventListener("app:jobs", (e) => {
|
||||
says so is the app's to show and hide. It starts hidden: until the first
|
||||
snapshot nobody knows whether the queue is idle. */
|
||||
function update(state) {
|
||||
const idle = document.getElementById("idle");
|
||||
if (!idle) return;
|
||||
const state = e.detail || {};
|
||||
const busy = (state.running || []).length + (state.queued || []).length;
|
||||
idle.hidden = busy > 0;
|
||||
});
|
||||
if (idle) {
|
||||
const busy = (state.running || []).length + (state.queued || []).length;
|
||||
idle.hidden = busy > 0;
|
||||
}
|
||||
pulling = (state.cascades || []).some((c) => isPull(c) && live(c));
|
||||
paint();
|
||||
}
|
||||
|
||||
document.addEventListener("DOMContentLoaded", draw);
|
||||
document.addEventListener("app:jobs", (e) => update(e.detail || {}));
|
||||
// A snapshot that arrived before this script loaded.
|
||||
if (window.live && window.live.jobs) update(window.live.jobs);
|
||||
})();
|
||||
+14
-9
@@ -206,17 +206,19 @@
|
||||
|
||||
// ------------------------------------------------------ one section shown
|
||||
|
||||
// Sources shows one section: the one the fragment names, or the first when
|
||||
// there is none or it names none, so `/settings/sources#runs` (a rail link,
|
||||
// an old path's 301, the Search agents page's "follow it under Runs") opens
|
||||
// Runs. The rest are hidden, not removed, so their scripts still find their
|
||||
// roots. It runs as the script loads rather than on DOMContentLoaded, so the
|
||||
// page never paints all four first.
|
||||
// Sources shows one section: the one the fragment names or contains, or the
|
||||
// first when there is none or it names none, so `/settings/sources#runs` (a
|
||||
// rail link, an old path's 301, the Search agents page's "follow it under
|
||||
// Runs") opens Runs, and so does `#messages`, the log at its end. The rest
|
||||
// are hidden, not removed, so their scripts still find their roots. It runs
|
||||
// as the script loads rather than on DOMContentLoaded, so the page never
|
||||
// paints all three first.
|
||||
function showSection() {
|
||||
const sections = [...document.querySelectorAll("section[data-section]")];
|
||||
if (!sections.length) return;
|
||||
const want = decodeURIComponent(location.hash.slice(1));
|
||||
const shown = sections.find((s) => s.id === want) || sections[0];
|
||||
const target = want && document.getElementById(want);
|
||||
const shown = sections.find((s) => target && s.contains(target)) || sections[0];
|
||||
for (const section of sections) section.hidden = section !== shown;
|
||||
for (const link of document.querySelectorAll(".settings-rail a[data-section]")) {
|
||||
if (link.dataset.section === shown.id) {
|
||||
@@ -229,10 +231,13 @@
|
||||
}
|
||||
|
||||
showSection();
|
||||
// A switch starts the section at the top of the page, as opening it would.
|
||||
// A switch starts the section at the top of the page, as opening it would;
|
||||
// a fragment inside one (`#messages`) scrolls to itself.
|
||||
window.addEventListener("hashchange", () => {
|
||||
showSection();
|
||||
window.scrollTo(0, 0);
|
||||
const target = document.getElementById(decodeURIComponent(location.hash.slice(1)));
|
||||
if (target && !target.matches("section[data-section]")) target.scrollIntoView();
|
||||
else window.scrollTo(0, 0);
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------- init
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
<p class="lede">The one password, the devices signed in with it, and the passkeys that sign in without it.</p>
|
||||
|
||||
<section class="card">
|
||||
{# As on the Pulls page: the element and its script belong to
|
||||
{# As in Sources' Runs: the element and its script belong to
|
||||
the module, so this page is two lines and `auth/` owns what they draw.
|
||||
|
||||
It says its own failures in place, beside the button that caused them,
|
||||
|
||||
@@ -1,14 +1,15 @@
|
||||
{% extends "settings/_layout.html" %}
|
||||
{% block title %}Sources — roledesk{% endblock %}
|
||||
|
||||
{# One page, four sections: the `search` plugin's job board queries, the
|
||||
`ats` plugin's company boards, the runs that read both, and the message
|
||||
log. One is shown at a time, the one the URL fragment names (`#runs`), or
|
||||
Job boards without one; sources.js hides the rest and the rail picks one.
|
||||
Hidden rather than separate pages, because each section keeps the element
|
||||
ids it had as a page of its own and its script (sources.js, companies.js,
|
||||
jobs.js, messages.js) is unchanged: it finds its root or does nothing. The
|
||||
ids are `SECTIONS` in app/web/pages.py, which the rail links to. #}
|
||||
{# One page, three sections: the `search` plugin's job board queries, the
|
||||
`ats` plugin's company boards, and the runs that read both, with the
|
||||
message log as their last block. One is shown at a time, the one the URL
|
||||
fragment names (`#runs`) or contains (`#messages`), or Job boards without
|
||||
one; sources.js hides the rest and the rail picks one. Hidden rather than
|
||||
separate pages, because each section keeps the element ids it had as a
|
||||
page of its own and its script (sources.js, companies.js, jobs.js,
|
||||
messages.js) is unchanged: it finds its root or does nothing. The ids are
|
||||
`SECTIONS` in app/web/pages.py, which the rail links to. #}
|
||||
{% block content %}
|
||||
<h1>Sources</h1>
|
||||
<p class="lede">Sources are where roles come from: queries on job boards that span companies, and the boards of the companies you follow. Each section is one plugin; the runs read them all.</p>
|
||||
@@ -109,18 +110,18 @@
|
||||
<section class="card" id="runs" data-section>
|
||||
<div class="row-between">
|
||||
<h2>Runs</h2>
|
||||
<button type="button" id="pull-now" class="secondary">Pull now</button>
|
||||
<button type="button" id="pull-now" class="secondary">Run the pull now</button>
|
||||
</div>
|
||||
<p class="lede">Every 6 hours the pull reads every board, embed and score rank every role, extract fills in what the best postings say, score runs again with what it filled, and summarise writes the summaries. Change the cadence below.</p>
|
||||
<p class="lede">On the schedule below the pull reads every board, embed and score rank every role, extract fills in what the best postings say, score runs again with what it filled, and summarise writes the summaries.</p>
|
||||
|
||||
<h3>What is running</h3>
|
||||
{# The heading and the idle line are the app's: the element takes up no
|
||||
space while the queue is idle — that is its `idle="hide"` default, and a
|
||||
strip that is always there is furniture — and a heading with nothing
|
||||
under it reads as something that failed to load. The element and its
|
||||
script are the queue's own, served by its own router — see
|
||||
pipeline/web/static/jobs.js. #}
|
||||
<p class="muted small" id="idle">Nothing at the moment.</p>
|
||||
under it reads as something that failed to load. The line is hidden
|
||||
until jobs.js has the first snapshot. The element and its script are the
|
||||
queue's own, served by its own router — see pipeline/web/static/jobs.js. #}
|
||||
<p class="muted small" id="idle" hidden>Nothing at the moment.</p>
|
||||
<script type="module" src="/_jobs/jobs.js"></script>
|
||||
<desk-jobs></desk-jobs>
|
||||
|
||||
@@ -132,22 +133,16 @@
|
||||
<script type="module" src="/_jobs/schedules.js"></script>
|
||||
<desk-schedules></desk-schedules>
|
||||
|
||||
<details class="stage">
|
||||
<summary>Run one stage</summary>
|
||||
<div id="kinds" class="buttons">
|
||||
<span class="muted">loading…</span>
|
||||
{# The message log, the queue's own through `/api/notices`. `#messages` is
|
||||
where the old Messages paths and the toasts' "all messages" link land. #}
|
||||
<div id="messages">
|
||||
<div class="row-between">
|
||||
<h3>Problems</h3>
|
||||
<button type="button" id="clear-all" class="secondary">Clear all</button>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
</section>
|
||||
|
||||
<section class="card" id="messages" data-section>
|
||||
<div class="row-between">
|
||||
<h2>Messages</h2>
|
||||
<button type="button" id="clear-all" class="secondary">Clear all</button>
|
||||
<p class="lede">What the server said that nobody has cleared, newest first. A message goes when you clear it, or a fortnight after it was last said.</p>
|
||||
<div id="notices"><span class="muted">loading…</span></div>
|
||||
</div>
|
||||
<p class="lede">What the server said that nobody has cleared, newest first. A message goes when you clear it, or a fortnight after it was last said.</p>
|
||||
<div id="notices"><span class="muted">loading…</span></div>
|
||||
</section>
|
||||
{% endblock %}
|
||||
|
||||
|
||||
+10
-8
@@ -16,17 +16,19 @@ one page, so a link to a role and the role opened from the feed are the same
|
||||
screen. What the server renders is only what a select needs to offer, so the
|
||||
options cannot drift from what the store accepts.
|
||||
|
||||
Sources is one page in four sections: job boards (the `search` plugin's
|
||||
queries), company boards (`ats`), the runs that read them, and the message
|
||||
log. One is shown at a time, the one the URL fragment names; the others stay
|
||||
in the page, hidden, so each section's script (the ones they had as pages of
|
||||
their own, each guarding on its own root element) keeps working.
|
||||
Sources is one page in three sections: job boards (the `search` plugin's
|
||||
queries), company boards (`ats`), and the runs that read them, which end in
|
||||
the message log. One is shown at a time, the one the URL fragment names or
|
||||
contains; the others stay in the page, hidden, so each section's script (the
|
||||
ones they had as pages of their own, each guarding on its own root element)
|
||||
keeps working.
|
||||
|
||||
Every path a page has had before (`/agents`, `/companies`, `/sources`,
|
||||
`/jobs`, `/schedules`, `/messages`, `/account`, and the six sub-tabs the
|
||||
settings shell started with) answers 301 to its place now, query string
|
||||
kept, so a bookmark still lands. A merged page's old path lands on its
|
||||
section's anchor.
|
||||
section's anchor; the old Messages paths land on `#messages`, the log's
|
||||
block at the end of Runs.
|
||||
|
||||
`FUNCTIONS` is `plugins.ats`'s, not `app`'s: the nine job functions are the
|
||||
classifier's vocabulary, and this reaches `plugins/` the same way `app.cli`
|
||||
@@ -88,12 +90,12 @@ SECTIONS: dict[str, tuple[tuple[str, str], ...]] = {
|
||||
("job-boards", "Job boards"),
|
||||
("company-boards", "Company boards"),
|
||||
("runs", "Runs"),
|
||||
("messages", "Messages"),
|
||||
),
|
||||
}
|
||||
|
||||
#: Where each page that used to be a tab or a sub-tab lives now. The fragment
|
||||
#: is the section on the merged page; a browser keeps it across the 301.
|
||||
#: is the section on the merged page, or an element in one (`#messages`, in
|
||||
#: Runs); a browser keeps it across the 301.
|
||||
MOVED: dict[str, str] = {
|
||||
"/agents": "/settings/agents",
|
||||
"/settings/ranking": "/settings/agents",
|
||||
|
||||
Reference in new issue
Block a user