Runs is the queue's elements and a Problems log; a Kill lands between boards

Run one stage goes: the cascade is the unit of work and app job keeps the
per-stage entry points. The pull button is disabled while a pull runs and
the lede no longer hardcodes the cadence. Messages folds into Runs as
Problems. pull_batch checks for a cancel before every board, one plugin per
target, and both plugins now close their HTTP client.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-09-28 16:10:38 +02:00
1 parent cebe62ddc9
commit 8532e4e739
15 files changed
+284 -242

No files matched your search

+1 -1
View File
@@ -44,7 +44,7 @@ The cog opens `/settings`: a rail of pages on the left, the page beside it, stic
| | |
|---|---|
| **Search agents** | a card per agent, newest first: name, an On/Off switch, what it looks for, and its weights. Edit turns the card into the form in place (name, intent, weights, require/prefer gates behind an Advanced disclosure); New agent puts a blank card on top. Saving or switching on rescores; deleting one takes its votes, liked companies and scores with it |
| **Sources** | one page showing one section at a time, picked in the rail or by the URL fragment: job boards (queries on Jobindex or Adzuna: try, save, edit, remove), company boards (every company whose board is read, its open roles and its affinity for the picked agent, following a company) and runs (Pull now, what is running, the schedule, one stage at a time, and the message log) |
| **Sources** | one page showing one section at a time, picked in the rail or by the URL fragment: job boards (queries on Jobindex or Adzuna: try, save, edit, remove), company boards (every company whose board is read, its open roles and its affinity for the picked agent, following a company) and runs (Run the pull now, what is running, the schedule, and the Problems log) |
| **Account** | the password, sessions and passkeys. Open on loopback, locked on anything else |
Likes, hidden companies and votes belong to a search agent, not to the account: two agents rank the same roles differently, and one may hide a company the other favours. An agent also learns from votes: its `taste` weight scores a role by how close it is to the roles the agent liked minus the ones it passed.
+3 -3
View File
@@ -142,9 +142,9 @@ def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str
return {"started": " > ".join(CASCADES["pull"])}
#: What `app job <name>` and the Pulls page offer besides `pull`: one stage
#: What `app job <name>` and `/api/jobs` start besides `pull`: one stage
#: each, on its own. `pull` is not a task here, since it is a job: a name that
#: is both would be two buttons on the Pulls page.
#: is both would be listed twice by `/api/jobs/kinds`.
for _name in ("extract", "embed", "score", "summarise"):
TASKS[_name] = partial(task, _name)
@@ -152,7 +152,7 @@ for _name in ("extract", "embed", "score", "summarise"):
# ---------------------------------------------------------------- schedules
# The default, next to the job it fires. A starting point rather than a
# setting: a cadence edited or switched off on the Pulls page survives
# setting: a cadence edited or switched off under Runs survives
# every restart (see pipeline/schedule.py).
declare("pull", "pull", "every 6h",
label="Pull every board, then embed, score, extract, rescore and summarise")
+23 -24
View File
@@ -78,41 +78,40 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
they list.
`targets` is keyword-only without a default, which is what makes it a
required payload key to the queue: the Pulls page offers no button for a
batch that could only fail without one.
required payload key to the queue: `/api/jobs/kinds` lists no batch that
could only fail without one.
Builds the plugin scoped to just these targets — `Source.targets()` reads
`options[subscription_field]`, so handing it only this batch's entries is
what keeps one job from re-fetching every other job's boards.
One plugin per target, scoped to just that entry — `Source.targets()`
reads `options[subscription_field]` — so one job never re-fetches another
job's boards, and `context.check()` runs before each fetch: a Kill lands
between two boards rather than after the tenth.
"""
from plugins.ats import ATSSource
from plugins.search import SearchSource
# Fetchable now, not when the pull was queued: a board switched off on the
# Searches or Companies page since then is skipped.
# Job boards or Company boards since then is skipped.
subs = {s.target: s for s in store.subscriptions(source=source, fetchable=True)
if s.target in targets}
options = dict(config.sources.get(source) or {})
plugin: ATSSource | SearchSource
if source == "search":
options["searches"] = [{"url": t, "label": subs[t].label, "params": subs[t].params}
for t in targets if t in subs]
plugin = SearchSource(options)
else:
options["boards"] = [{"url": t, "label": subs[t].company or subs[t].label}
for t in targets if t in subs]
plugin = ATSSource(options)
plugin.state = store.state
stored = 0
for item in plugin.fetch():
item.source = source
for target in targets:
sub = subs.get(target)
if sub is None:
continue
context.check()
options = dict(config.sources.get(source) or {})
plugin: ATSSource | SearchSource
if source == "search":
target = str(item.raw.get("search", ""))
options["searches"] = [{"url": target, "label": sub.label, "params": sub.params}]
plugin = SearchSource(options)
else:
target = f"{item.raw.get('board', '')}:{item.raw.get('slug', '')}"
store.upsert_role(item, subs.get(target))
stored += 1
options["boards"] = [{"url": target, "label": sub.company or sub.label}]
plugin = ATSSource(options)
plugin.state = store.state
for item in plugin.fetch():
item.source = source
store.upsert_role(item, sub)
stored += 1
return {"roles": stored}
+3 -3
View File
@@ -186,7 +186,7 @@ def run_batch(config: Config, store: Store, *, context: RunContext = NULL,
def summarise_task() -> Task:
"""Finds nothing while `summarise.enabled` is false, so the task, the
button on the Pulls page and the cascade stage are all no-ops with the
switch off."""
"""Finds nothing while `summarise.enabled` is false, so the task, `app
job summarise` and the cascade stage are all no-ops with the switch
off."""
return replace(modeltask.of(TASK), label=TASK.label)
-7
View File
@@ -275,13 +275,6 @@ button.secondary {
font-size: 0.8rem;
}
/* -- pulls ------------------------------------------------------------------ */
/* "Run one stage" on the Pulls page: extract, embed and score on their own,
under the schedule rather than beside Pull now. */
details.stage summary { cursor: pointer; color: var(--dim); font-size: 0.95rem; font-weight: 600; }
details.stage[open] summary { margin-bottom: 0.75rem; }
/* -- the notice log ------------------------------------------------------- */
.notice {
+50 -55
View File
@@ -1,11 +1,12 @@
/* The Runs section of Sources: Pull now on top, one button per remaining
* kind under "Run one stage".
/* The Runs section of Sources: "Run the pull now" and the idle line.
*
* The list comes from `/api/jobs/kinds`, which is the registry, so a new
* `@job` in `app/jobs.py` is a new button and this file is not edited --
* except that `pull` itself is pulled out of the list and wired to the
* dedicated button instead of appearing twice. What the queue is doing is
* `<desk-jobs>`'s business, and it draws itself.
* What the queue is doing and when it runs next are `<desk-jobs>`' and
* `<desk-schedules>`' business, and they draw themselves. The single stages
* have no buttons: the cascade is the unit of work, and `app job <stage>`
* keeps the per-stage entry points.
*
* Both read the `app:jobs` snapshot from live.js rather than asking the
* elements, which keep to a shadow root and owe nobody an answer.
*
* Failures are the server's sentence through `api()`, which toasts it -- so
* there is no error handling here beyond keeping the button usable.
@@ -14,63 +15,57 @@
(function () {
"use strict";
const host = document.getElementById("kinds");
const pullButton = document.getElementById("pull-now");
async function start(kind, button) {
button.disabled = true;
try {
const job = await window.api("/api/jobs", { method: "POST", body: { kind } });
window.toast.good(`${job.label || kind}: queued.`);
} catch (e) {
/* already toasted */
} finally {
button.disabled = false;
}
let pending = false; // the POST is in the air
let pulling = false; // the latest snapshot has a live pull cascade
/* A pull queued by the button or a schedule is the `pull` job, which
begins a cascade of its own named `pull on its own`, and the chain's
tasks join it; `pull` is the name a cascade started as one would have.
Live while any task has a job queued or running: the pipeline keeps a
finished cascade in the snapshot for a few seconds, and that is not a
pull still going. As newsdesk's Fetch: the whole cascade, not the `pull`
task, which is over seconds in with five tasks to go. */
const isPull = (c) => c.name === "pull" || c.name === "pull on its own";
const live = (c) => (c.tasks || []).some((t) => t.running || t.queued);
function paint() {
if (!pullButton) return;
pullButton.disabled = pending || pulling;
pullButton.title = pulling ? "A pull is running." : "";
}
if (pullButton) {
pullButton.addEventListener("click", () => start("pull", pullButton));
}
async function draw() {
if (!host) return;
let kinds;
try {
kinds = await window.api("/api/jobs/kinds");
} catch (e) {
host.innerHTML = '<span class="muted">could not read the job kinds</span>';
return;
}
kinds = kinds.filter((k) => k.kind !== "pull");
if (!kinds.length) {
host.innerHTML = '<span class="muted">nothing is registered</span>';
return;
}
host.innerHTML = "";
kinds.forEach((k) => {
const button = document.createElement("button");
button.type = "button";
button.className = "secondary";
button.textContent = k.label || k.kind;
button.title = k.solo ? `${k.kind} -- one at a time` : k.kind;
button.dataset.kind = k.kind;
button.addEventListener("click", () => start(k.kind, button));
host.appendChild(button);
pullButton.addEventListener("click", async () => {
pending = true;
paint();
try {
const job = await window.api("/api/jobs", { method: "POST", body: { kind: "pull" } });
window.toast.good(`${job.label || "pull"}: queued.`);
} catch (e) {
/* already toasted */
} finally {
pending = false;
paint();
}
});
}
/* `<desk-jobs>` takes up no space while the queue is idle, so the line that
says so is the app's to show and hide. Off the same socket the element
reads, through live.js, rather than by asking the element: it keeps to a
shadow root and owes nobody an answer about its own height. */
document.addEventListener("app:jobs", (e) => {
says so is the app's to show and hide. It starts hidden: until the first
snapshot nobody knows whether the queue is idle. */
function update(state) {
const idle = document.getElementById("idle");
if (!idle) return;
const state = e.detail || {};
const busy = (state.running || []).length + (state.queued || []).length;
idle.hidden = busy > 0;
});
if (idle) {
const busy = (state.running || []).length + (state.queued || []).length;
idle.hidden = busy > 0;
}
pulling = (state.cascades || []).some((c) => isPull(c) && live(c));
paint();
}
document.addEventListener("DOMContentLoaded", draw);
document.addEventListener("app:jobs", (e) => update(e.detail || {}));
// A snapshot that arrived before this script loaded.
if (window.live && window.live.jobs) update(window.live.jobs);
})();
+14 -9
View File
@@ -206,17 +206,19 @@
// ------------------------------------------------------ one section shown
// Sources shows one section: the one the fragment names, or the first when
// there is none or it names none, so `/settings/sources#runs` (a rail link,
// an old path's 301, the Search agents page's "follow it under Runs") opens
// Runs. The rest are hidden, not removed, so their scripts still find their
// roots. It runs as the script loads rather than on DOMContentLoaded, so the
// page never paints all four first.
// Sources shows one section: the one the fragment names or contains, or the
// first when there is none or it names none, so `/settings/sources#runs` (a
// rail link, an old path's 301, the Search agents page's "follow it under
// Runs") opens Runs, and so does `#messages`, the log at its end. The rest
// are hidden, not removed, so their scripts still find their roots. It runs
// as the script loads rather than on DOMContentLoaded, so the page never
// paints all three first.
function showSection() {
const sections = [...document.querySelectorAll("section[data-section]")];
if (!sections.length) return;
const want = decodeURIComponent(location.hash.slice(1));
const shown = sections.find((s) => s.id === want) || sections[0];
const target = want && document.getElementById(want);
const shown = sections.find((s) => target && s.contains(target)) || sections[0];
for (const section of sections) section.hidden = section !== shown;
for (const link of document.querySelectorAll(".settings-rail a[data-section]")) {
if (link.dataset.section === shown.id) {
@@ -229,10 +231,13 @@
}
showSection();
// A switch starts the section at the top of the page, as opening it would.
// A switch starts the section at the top of the page, as opening it would;
// a fragment inside one (`#messages`) scrolls to itself.
window.addEventListener("hashchange", () => {
showSection();
window.scrollTo(0, 0);
const target = document.getElementById(decodeURIComponent(location.hash.slice(1)));
if (target && !target.matches("section[data-section]")) target.scrollIntoView();
else window.scrollTo(0, 0);
});
// -------------------------------------------------------------------- init
+1 -1
View File
@@ -6,7 +6,7 @@
<p class="lede">The one password, the devices signed in with it, and the passkeys that sign in without it.</p>
<section class="card">
{# As on the Pulls page: the element and its script belong to
{# As in Sources' Runs: the element and its script belong to
the module, so this page is two lines and `auth/` owns what they draw.
It says its own failures in place, beside the button that caused them,
+23 -28
View File
@@ -1,14 +1,15 @@
{% extends "settings/_layout.html" %}
{% block title %}Sources — roledesk{% endblock %}
{# One page, four sections: the `search` plugin's job board queries, the
`ats` plugin's company boards, the runs that read both, and the message
log. One is shown at a time, the one the URL fragment names (`#runs`), or
Job boards without one; sources.js hides the rest and the rail picks one.
Hidden rather than separate pages, because each section keeps the element
ids it had as a page of its own and its script (sources.js, companies.js,
jobs.js, messages.js) is unchanged: it finds its root or does nothing. The
ids are `SECTIONS` in app/web/pages.py, which the rail links to. #}
{# One page, three sections: the `search` plugin's job board queries, the
`ats` plugin's company boards, and the runs that read both, with the
message log as their last block. One is shown at a time, the one the URL
fragment names (`#runs`) or contains (`#messages`), or Job boards without
one; sources.js hides the rest and the rail picks one. Hidden rather than
separate pages, because each section keeps the element ids it had as a
page of its own and its script (sources.js, companies.js, jobs.js,
messages.js) is unchanged: it finds its root or does nothing. The ids are
`SECTIONS` in app/web/pages.py, which the rail links to. #}
{% block content %}
<h1>Sources</h1>
<p class="lede">Sources are where roles come from: queries on job boards that span companies, and the boards of the companies you follow. Each section is one plugin; the runs read them all.</p>
@@ -109,18 +110,18 @@
<section class="card" id="runs" data-section>
<div class="row-between">
<h2>Runs</h2>
<button type="button" id="pull-now" class="secondary">Pull now</button>
<button type="button" id="pull-now" class="secondary">Run the pull now</button>
</div>
<p class="lede">Every 6 hours the pull reads every board, embed and score rank every role, extract fills in what the best postings say, score runs again with what it filled, and summarise writes the summaries. Change the cadence below.</p>
<p class="lede">On the schedule below the pull reads every board, embed and score rank every role, extract fills in what the best postings say, score runs again with what it filled, and summarise writes the summaries.</p>
<h3>What is running</h3>
{# The heading and the idle line are the app's: the element takes up no
space while the queue is idle — that is its `idle="hide"` default, and a
strip that is always there is furniture — and a heading with nothing
under it reads as something that failed to load. The element and its
script are the queue's own, served by its own router — see
pipeline/web/static/jobs.js. #}
<p class="muted small" id="idle">Nothing at the moment.</p>
under it reads as something that failed to load. The line is hidden
until jobs.js has the first snapshot. The element and its script are the
queue's own, served by its own router — see pipeline/web/static/jobs.js. #}
<p class="muted small" id="idle" hidden>Nothing at the moment.</p>
<script type="module" src="/_jobs/jobs.js"></script>
<desk-jobs></desk-jobs>
@@ -132,22 +133,16 @@
<script type="module" src="/_jobs/schedules.js"></script>
<desk-schedules></desk-schedules>
<details class="stage">
<summary>Run one stage</summary>
<div id="kinds" class="buttons">
<span class="muted">loading…</span>
{# The message log, the queue's own through `/api/notices`. `#messages` is
where the old Messages paths and the toasts' "all messages" link land. #}
<div id="messages">
<div class="row-between">
<h3>Problems</h3>
<button type="button" id="clear-all" class="secondary">Clear all</button>
</div>
</details>
</section>
<section class="card" id="messages" data-section>
<div class="row-between">
<h2>Messages</h2>
<button type="button" id="clear-all" class="secondary">Clear all</button>
<p class="lede">What the server said that nobody has cleared, newest first. A message goes when you clear it, or a fortnight after it was last said.</p>
<div id="notices"><span class="muted">loading…</span></div>
</div>
<p class="lede">What the server said that nobody has cleared, newest first. A message goes when you clear it, or a fortnight after it was last said.</p>
<div id="notices"><span class="muted">loading…</span></div>
</section>
{% endblock %}
+10 -8
View File
@@ -16,17 +16,19 @@ one page, so a link to a role and the role opened from the feed are the same
screen. What the server renders is only what a select needs to offer, so the
options cannot drift from what the store accepts.
Sources is one page in four sections: job boards (the `search` plugin's
queries), company boards (`ats`), the runs that read them, and the message
log. One is shown at a time, the one the URL fragment names; the others stay
in the page, hidden, so each section's script (the ones they had as pages of
their own, each guarding on its own root element) keeps working.
Sources is one page in three sections: job boards (the `search` plugin's
queries), company boards (`ats`), and the runs that read them, which end in
the message log. One is shown at a time, the one the URL fragment names or
contains; the others stay in the page, hidden, so each section's script (the
ones they had as pages of their own, each guarding on its own root element)
keeps working.
Every path a page has had before (`/agents`, `/companies`, `/sources`,
`/jobs`, `/schedules`, `/messages`, `/account`, and the six sub-tabs the
settings shell started with) answers 301 to its place now, query string
kept, so a bookmark still lands. A merged page's old path lands on its
section's anchor.
section's anchor; the old Messages paths land on `#messages`, the log's
block at the end of Runs.
`FUNCTIONS` is `plugins.ats`'s, not `app`'s: the nine job functions are the
classifier's vocabulary, and this reaches `plugins/` the same way `app.cli`
@@ -88,12 +90,12 @@ SECTIONS: dict[str, tuple[tuple[str, str], ...]] = {
("job-boards", "Job boards"),
("company-boards", "Company boards"),
("runs", "Runs"),
("messages", "Messages"),
),
}
#: Where each page that used to be a tab or a sub-tab lives now. The fragment
#: is the section on the merged page; a browser keeps it across the 301.
#: is the section on the merged page, or an element in one (`#messages`, in
#: Runs); a browser keeps it across the 301.
MOVED: dict[str, str] = {
"/agents": "/settings/agents",
"/settings/ranking": "/settings/agents",
+2
View File
@@ -112,6 +112,8 @@ What was decided for roledesk and the one reason for each. The longer reasoning
**Settings pages sit beside a rail, and Sources shows one section at a time, chosen by the URL fragment.** The company boards list runs to 50 or 60 companies, so Runs at the end of a stacked page was out of reach and jump links still meant scrolling back up. The rail (`settings/_rail.html`, from `SETTINGS` and `SECTIONS` in `app/web/pages.py`) is sticky beside the content, and a strip under the header below 60rem. The fragment rather than a path per section, because the old paths already 301 to `/settings/sources#runs` and the like, and a hidden section stays in the DOM, so each section's script runs unchanged.
**Runs is the queue's own elements plus a Problems log.** `<desk-jobs>` and `<desk-schedules>` draw what is running and when the pull runs next, the app adds only "Run the pull now", disabled while a pull cascade is live, and the message log is the section's last block rather than a section of its own; `#messages` still lands on it. "Run one stage" went because the cascade is the unit of work: a stage run alone is a partial pull whose result the next stage never sees. `app job <stage>` and `POST /api/jobs` keep the per-stage entry points, which is what "Save and score" uses.
**Search agents are cards, edited in place.** An agent is defined by what it looks for, so the intent is the card's body and the weights a small line under it; a list of names and weights hid the part that matters. Edit turns that card into the form and New puts a blank one on top, because the new agent sorts first by id and a form at the bottom of the page was out of sight of the card it changed.
**Affinity and votes are per agent.** A company hidden for one hunt may be a favourite for another, and a vote says what one agent should learn from, so both carry the agent id, and the Roles tab and the company list ask for one agent's. Without an agent id the API uses the default agent, the enabled one with the lowest id, so the CLI and older callers keep working.
+45 -45
View File
@@ -532,50 +532,50 @@ class ATSSource(Source):
)
def fetch(self) -> Iterator[Item]:
client = httpx.Client()
# Written by the pull job before any board is fetched (see
# `app/jobs/pull.py`); a plugin run on its own — a test, a dry
# run — has no run to stamp itself against, so it makes one up.
run_marker = self.state.get("ats:run", "") or datetime.now(UTC).isoformat(
timespec="seconds")
with httpx.Client() as client:
# Written by the pull job before any board is fetched (see
# `app/jobs/pull.py`); a plugin run on its own — a test, a dry
# run — has no run to stamp itself against, so it makes one up.
run_marker = self.state.get("ats:run", "") or datetime.now(UTC).isoformat(
timespec="seconds")
for target, label in self.targets():
host, _, slug = target.partition(":")
try:
rows = _fetch_rows(client, target)
except Exception as exc:
log.warning("ats: %s failed (%s)", target, exc)
continue
for target, label in self.targets():
host, _, slug = target.partition(":")
try:
rows = _fetch_rows(client, target)
except Exception as exc:
log.warning("ats: %s failed (%s)", target, exc)
continue
seen_ids = []
for row in rows:
# The compound id, matching `roles.source_id` — what
# `Store.mark_closed` compares against, not the board's own
# bare job id.
compound_id = f"{host}:{slug}:{row['source_id']}"
seen_ids.append(compound_id)
yield Item(
source_id=compound_id,
location=row["url"],
title=row["title"],
excerpt=(row["description"] or "")[:300],
body=row["description"],
published_at=row["posted"],
subscription=label,
kind="role",
raw={
"company": label,
"place": row["location"],
"department": row["department"],
"function": classify(row["department"], row["title"]),
"remote": row["remote"],
"salary": row["salary"],
"board": host,
"slug": slug,
},
)
# Stamped only once the board has parsed without raising, so a
# board that blew up half way through is neither marked ok nor
# has a partial id list treated as the whole of it.
self.state[f"ats:ok:{target}"] = run_marker
self.state[f"ats:ids:{target}"] = json.dumps(seen_ids)
seen_ids = []
for row in rows:
# The compound id, matching `roles.source_id` — what
# `Store.mark_closed` compares against, not the board's own
# bare job id.
compound_id = f"{host}:{slug}:{row['source_id']}"
seen_ids.append(compound_id)
yield Item(
source_id=compound_id,
location=row["url"],
title=row["title"],
excerpt=(row["description"] or "")[:300],
body=row["description"],
published_at=row["posted"],
subscription=label,
kind="role",
raw={
"company": label,
"place": row["location"],
"department": row["department"],
"function": classify(row["department"], row["title"]),
"remote": row["remote"],
"salary": row["salary"],
"board": host,
"slug": slug,
},
)
# Stamped only once the board has parsed without raising, so a
# board that blew up half way through is neither marked ok nor
# has a partial id list treated as the whole of it.
self.state[f"ats:ok:{target}"] = run_marker
self.state[f"ats:ids:{target}"] = json.dumps(seen_ids)
+42 -42
View File
@@ -308,46 +308,46 @@ class SearchSource(Source):
)
def fetch(self) -> Iterator[Item]:
client = httpx.Client()
run_marker = self.state.get("search:run", "") or datetime.now(UTC).isoformat(
timespec="seconds")
for target, label in self.targets():
provider = provider_of(target)
if not provider:
log.warning("search: %r is not provider:name; skipped", target)
continue
try:
rows, complete = _SEARCHES[provider](
client, self.params_for(target), self.options.get(provider) or {})
except Exception as exc:
log.warning("search: %s failed (%s)", target, exc)
continue
with httpx.Client() as client:
run_marker = self.state.get("search:run", "") or datetime.now(UTC).isoformat(
timespec="seconds")
for target, label in self.targets():
provider = provider_of(target)
if not provider:
log.warning("search: %r is not provider:name; skipped", target)
continue
try:
rows, complete = _SEARCHES[provider](
client, self.params_for(target), self.options.get(provider) or {})
except Exception as exc:
log.warning("search: %s failed (%s)", target, exc)
continue
seen_ids = []
for row in rows:
source_id = f"{provider}:{row['id']}"
seen_ids.append(source_id)
yield Item(
source_id=source_id,
location=row["url"],
title=row["title"],
excerpt=(row["description"] or "")[:300],
body=row["description"],
published_at=row["posted"],
subscription=label,
kind="role",
raw={
"company": row["company"],
"place": row["place"],
"country": row["country"],
"remote": row["remote"],
"salary_min": row["salary_min"],
"salary_max": row["salary_max"],
"currency": row["currency"],
"provider": provider,
"search": target,
},
)
self.state[f"search:ok:{target}"] = run_marker
self.state[f"search:complete:{target}"] = "1" if complete else ""
self.state[f"search:ids:{target}"] = json.dumps(seen_ids)
seen_ids = []
for row in rows:
source_id = f"{provider}:{row['id']}"
seen_ids.append(source_id)
yield Item(
source_id=source_id,
location=row["url"],
title=row["title"],
excerpt=(row["description"] or "")[:300],
body=row["description"],
published_at=row["posted"],
subscription=label,
kind="role",
raw={
"company": row["company"],
"place": row["place"],
"country": row["country"],
"remote": row["remote"],
"salary_min": row["salary_min"],
"salary_max": row["salary_max"],
"currency": row["currency"],
"provider": provider,
"search": target,
},
)
self.state[f"search:ok:{target}"] = run_marker
self.state[f"search:complete:{target}"] = "1" if complete else ""
self.state[f"search:ids:{target}"] = json.dumps(seen_ids)
+30
View File
@@ -16,6 +16,7 @@ if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT))
from desk.models import Item # noqa: E402
from pipeline import Cancelled, RunContext # noqa: E402
from plugins.ats import ATSSource # noqa: E402
from app.jobs import pull # noqa: E402
from app.storage import Store # noqa: E402
@@ -81,6 +82,35 @@ def test_run_batch_upserts_every_item_the_plugin_yields(
assert store.roles()[0].subscription_id is not None
def test_a_kill_stops_the_batch_before_the_next_board(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
"""`check()` before each board: a Kill pressed while the first is read
ends the job before the second is fetched, and keeps what the first
stored."""
for slug in ("acme", "beta"):
store.add_subscription("ats", f"greenhouse:{slug}", label=slug.title(),
company=slug.title(), board="greenhouse")
fetched: list[str] = []
killed = False
def fake_fetch(self: ATSSource) -> object:
nonlocal killed
[(target, _)] = self.targets()
fetched.append(target)
yield _item("greenhouse", target.partition(":")[2], "1")
killed = True
monkeypatch.setattr(ATSSource, "fetch", fake_fetch)
context = RunContext(is_cancelled=lambda: killed)
with pytest.raises(Cancelled):
pull.run_batch(FakeConfig(), store, targets=["greenhouse:acme", "greenhouse:beta"],
context=context)
assert fetched == ["greenhouse:acme"]
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"}
def test_close_boards_closes_what_a_completed_board_no_longer_lists(
store: Store) -> None:
sub = store.add_subscription("ats", "greenhouse:acme", label="Acme",
+37 -16
View File
@@ -549,18 +549,22 @@ def test_every_settings_page_has_the_rail_and_not_the_tab_row(client: TestClient
def test_a_moved_section_is_one_the_sources_page_shows(client: TestClient) -> None:
"""Every old path that lands on a fragment of Sources names a section of
`SECTIONS`, and each is a `<section data-section>` on the page, which is
what sources.js shows one of."""
`SECTIONS` or an element inside one, and each section is a `<section
data-section>` on the page, which is what sources.js shows one of. The
old Messages paths land on `#messages`, the log at the end of Runs."""
anchors = [anchor for anchor, _ in SECTIONS["/settings/sources"]]
assert anchors == ["job-boards", "company-boards", "runs", "messages"]
for to in MOVED.values():
path, _, anchor = to.partition("#")
if anchor:
assert path == "/settings/sources" and anchor in anchors, to
assert anchors == ["job-boards", "company-boards", "runs"]
page = client.get("/settings/sources").text
assert page.count("data-section>") == len(anchors)
for anchor in anchors:
assert f'<section class="card" id="{anchor}" data-section>' in page
runs = page[page.index('id="runs"'):]
for to in MOVED.values():
path, _, anchor = to.partition("#")
if anchor:
assert path == "/settings/sources", to
assert anchor in anchors or f'id="{anchor}"' in runs, to
assert MOVED["/messages"] == MOVED["/settings/messages"] == "/settings/sources#messages"
def test_settings_is_search_agents(client: TestClient) -> None:
@@ -586,8 +590,9 @@ def test_an_old_tab_moves_permanently_and_keeps_its_query(
def test_each_page_loads_only_its_own_script(client: TestClient) -> None:
"""Sources is four sections' scripts on one page, each guarding on its own
root element; every other page is one."""
"""Sources is four scripts on one page, one per section and one for the
log in Runs, each guarding on its own root element; every other page is
one."""
scripts = {"roles.js", "applications.js", "agents.js", "companies.js", "sources.js",
"jobs.js", "messages.js"}
for path, container, own in [
@@ -632,14 +637,15 @@ def test_the_agents_page_lists_and_edits_search_agents(client: TestClient) -> No
def test_the_sources_page_has_a_section_per_plugin(client: TestClient) -> None:
"""Job boards, company boards, the runs and the message log, in that
order, each linked from the rail; no jump row any more."""
"""Job boards, company boards and the runs, in that order, each linked
from the rail; no jump row any more."""
page = client.get("/settings/sources").text
assert '<aside class="settings-rail"' in page and 'class="jump"' not in page
for anchor in ("job-boards", "company-boards", "runs", "messages"):
for anchor in ("job-boards", "company-boards", "runs"):
assert f'href="/settings/sources#{anchor}"' in page and f'id="{anchor}"' in page
assert page.index('id="job-boards"') < page.index('id="company-boards"') \
< page.index('id="runs"') < page.index('id="messages"')
< page.index('id="runs"')
assert 'href="/settings/sources#messages"' not in page
# Job boards: the Jobindex areas and the Adzuna countries, from plugins.search.
assert 'id="searches-list"' in page and 'id="search-form"' in page
assert 'value="1" selected>Region Hovedstaden' in page
@@ -650,9 +656,16 @@ def test_the_sources_page_has_a_section_per_plugin(client: TestClient) -> None:
assert '<option value="greenhouse">' in page and '<option value="personio">' in page
boards = page[page.index('id="company-boards"'):page.index('id="runs"')].lower()
assert "hidden" in boards and "mute" not in boards
# Runs: Pull now, the queue's elements, one stage at a time, the log.
assert 'id="pull-now"' in page and 'id="kinds"' in page
assert 'id="notices"' in page and 'id="clear-all"' in page
# Runs: the pull button, the queue's elements, and the log as its last
# block. No button per stage, and no cadence in the lede.
runs = page[page.index('id="runs"'):page.index("</section>", page.index('id="runs"'))]
assert '<button type="button" id="pull-now" class="secondary">Run the pull now</button>' in runs
assert 'id="kinds"' not in page and "Run one stage" not in page
assert "On the schedule below the pull reads every board" in runs
assert "Every 6 hours" not in runs
assert '<p class="muted small" id="idle" hidden>' in runs
assert runs.index("<desk-schedules>") < runs.index('<div id="messages">') \
< runs.index("<h3>Problems</h3>") < runs.index('id="clear-all"') < runs.index('id="notices"')
def test_the_modules_bring_their_own_elements(client: TestClient) -> None:
@@ -685,6 +698,14 @@ def test_the_jobs_tab_offers_pull_once_and_no_batch(client: TestClient) -> None:
assert not {"pull_batch", "extract_batch", "advance"} & set(kinds)
def test_save_and_score_queues_a_score(client: TestClient, queue: Any) -> None:
"""The Search agents page's "Save and score" posts `{kind: "score"}`: a
task on its own, now that no stage has a button of its own."""
res = client.post("/api/jobs", json={"kind": "score", "payload": {}})
assert res.status_code == 202
assert queue.jobs.last("score", states=("queued", "running", "done")) is not None
def test_an_error_is_one_sentence(client: TestClient) -> None:
"""`api()` toasts `detail` as it is, so a 422 is a sentence, not a list."""
res = client.post("/api/agents", json={"name": "Bad",