Merge main: the Runs cleanup meets the queue's own idle text
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
commit
c63f348dc9
15 files changed
+275
-229
No files matched your search
@@ -44,7 +44,7 @@ The cog opens `/settings`: a rail of pages on the left, the page beside it, stic
|
||||
| | |
|
||||
|---|---|
|
||||
| **Search agents** | a card per agent, newest first: name, an On/Off switch, what it looks for, and its weights. Edit turns the card into the form in place (name, intent, weights, require/prefer gates behind an Advanced disclosure); New agent puts a blank card on top. Saving or switching on rescores; deleting one takes its votes, liked companies and scores with it |
|
||||
| **Sources** | one page showing one section at a time, picked in the rail or by the URL fragment: job boards (queries on Jobindex or Adzuna: try, save, edit, remove), company boards (every company whose board is read, its open roles and its affinity for the picked agent, following a company) and runs (Pull now, what is running, the schedule, one stage at a time, and the message log) |
|
||||
| **Sources** | one page showing one section at a time, picked in the rail or by the URL fragment: job boards (queries on Jobindex or Adzuna: try, save, edit, remove), company boards (every company whose board is read, its open roles and its affinity for the picked agent, following a company) and runs (Run the pull now, what is running, the schedule, and the Problems log) |
|
||||
| **Account** | the password, sessions and passkeys. Open on loopback, locked on anything else |
|
||||
|
||||
Likes, hidden companies and votes belong to a search agent, not to the account: two agents rank the same roles differently, and one may hide a company the other favours. An agent also learns from votes: its `taste` weight scores a role by how close it is to the roles the agent liked minus the ones it passed.
|
||||
|
||||
+3
-3
@@ -142,9 +142,9 @@ def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str
|
||||
return {"started": " > ".join(CASCADES["pull"])}
|
||||
|
||||
|
||||
#: What `app job <name>` and the Pulls page offer besides `pull`: one stage
|
||||
#: What `app job <name>` and `/api/jobs` start besides `pull`: one stage
|
||||
#: each, on its own. `pull` is not a task here, since it is a job: a name that
|
||||
#: is both would be two buttons on the Pulls page.
|
||||
#: is both would be listed twice by `/api/jobs/kinds`.
|
||||
for _name in ("extract", "embed", "score", "summarise"):
|
||||
TASKS[_name] = partial(task, _name)
|
||||
|
||||
@@ -152,7 +152,7 @@ for _name in ("extract", "embed", "score", "summarise"):
|
||||
# ---------------------------------------------------------------- schedules
|
||||
|
||||
# The default, next to the job it fires. A starting point rather than a
|
||||
# setting: a cadence edited or switched off on the Pulls page survives
|
||||
# setting: a cadence edited or switched off under Runs survives
|
||||
# every restart (see pipeline/schedule.py).
|
||||
declare("pull", "pull", "every 6h",
|
||||
label="Pull every board, then embed, score, extract, rescore and summarise")
|
||||
+23
-24
@@ -78,41 +78,40 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
|
||||
they list.
|
||||
|
||||
`targets` is keyword-only without a default, which is what makes it a
|
||||
required payload key to the queue: the Pulls page offers no button for a
|
||||
batch that could only fail without one.
|
||||
required payload key to the queue: `/api/jobs/kinds` lists no batch that
|
||||
could only fail without one.
|
||||
|
||||
Builds the plugin scoped to just these targets — `Source.targets()` reads
|
||||
`options[subscription_field]`, so handing it only this batch's entries is
|
||||
what keeps one job from re-fetching every other job's boards.
|
||||
One plugin per target, scoped to just that entry — `Source.targets()`
|
||||
reads `options[subscription_field]` — so one job never re-fetches another
|
||||
job's boards, and `context.check()` runs before each fetch: a Kill lands
|
||||
between two boards rather than after the tenth.
|
||||
"""
|
||||
from plugins.ats import ATSSource
|
||||
from plugins.search import SearchSource
|
||||
|
||||
# Fetchable now, not when the pull was queued: a board switched off on the
|
||||
# Searches or Companies page since then is skipped.
|
||||
# Job boards or Company boards since then is skipped.
|
||||
subs = {s.target: s for s in store.subscriptions(source=source, fetchable=True)
|
||||
if s.target in targets}
|
||||
options = dict(config.sources.get(source) or {})
|
||||
plugin: ATSSource | SearchSource
|
||||
if source == "search":
|
||||
options["searches"] = [{"url": t, "label": subs[t].label, "params": subs[t].params}
|
||||
for t in targets if t in subs]
|
||||
plugin = SearchSource(options)
|
||||
else:
|
||||
options["boards"] = [{"url": t, "label": subs[t].company or subs[t].label}
|
||||
for t in targets if t in subs]
|
||||
plugin = ATSSource(options)
|
||||
plugin.state = store.state
|
||||
|
||||
stored = 0
|
||||
for item in plugin.fetch():
|
||||
item.source = source
|
||||
for target in targets:
|
||||
sub = subs.get(target)
|
||||
if sub is None:
|
||||
continue
|
||||
context.check()
|
||||
options = dict(config.sources.get(source) or {})
|
||||
plugin: ATSSource | SearchSource
|
||||
if source == "search":
|
||||
target = str(item.raw.get("search", ""))
|
||||
options["searches"] = [{"url": target, "label": sub.label, "params": sub.params}]
|
||||
plugin = SearchSource(options)
|
||||
else:
|
||||
target = f"{item.raw.get('board', '')}:{item.raw.get('slug', '')}"
|
||||
store.upsert_role(item, subs.get(target))
|
||||
stored += 1
|
||||
options["boards"] = [{"url": target, "label": sub.company or sub.label}]
|
||||
plugin = ATSSource(options)
|
||||
plugin.state = store.state
|
||||
for item in plugin.fetch():
|
||||
item.source = source
|
||||
store.upsert_role(item, sub)
|
||||
stored += 1
|
||||
return {"roles": stored}
|
||||
|
||||
|
||||
|
||||
@@ -186,7 +186,7 @@ def run_batch(config: Config, store: Store, *, context: RunContext = NULL,
|
||||
|
||||
|
||||
def summarise_task() -> Task:
|
||||
"""Finds nothing while `summarise.enabled` is false, so the task, the
|
||||
button on the Pulls page and the cascade stage are all no-ops with the
|
||||
switch off."""
|
||||
"""Finds nothing while `summarise.enabled` is false, so the task, `app
|
||||
job summarise` and the cascade stage are all no-ops with the switch
|
||||
off."""
|
||||
return replace(modeltask.of(TASK), label=TASK.label)
|
||||
@@ -275,13 +275,6 @@ button.secondary {
|
||||
font-size: 0.8rem;
|
||||
}
|
||||
|
||||
/* -- pulls ------------------------------------------------------------------ */
|
||||
/* "Run one stage" on the Pulls page: extract, embed and score on their own,
|
||||
under the schedule rather than beside Pull now. */
|
||||
|
||||
details.stage summary { cursor: pointer; color: var(--dim); font-size: 0.95rem; font-weight: 600; }
|
||||
details.stage[open] summary { margin-bottom: 0.75rem; }
|
||||
|
||||
/* -- the notice log ------------------------------------------------------- */
|
||||
|
||||
.notice {
|
||||
|
||||
+45
-46
@@ -1,11 +1,12 @@
|
||||
/* The Runs section of Sources: Pull now on top, one button per remaining
|
||||
* kind under "Run one stage".
|
||||
/* The Runs section of Sources: the "Run the pull now" button.
|
||||
*
|
||||
* The list comes from `/api/jobs/kinds`, which is the registry, so a new
|
||||
* `@job` in `app/jobs.py` is a new button and this file is not edited --
|
||||
* except that `pull` itself is pulled out of the list and wired to the
|
||||
* dedicated button instead of appearing twice. What the queue is doing is
|
||||
* `<desk-jobs>`'s business, and it draws itself, idle line included.
|
||||
* What the queue is doing and when it runs next are `<desk-jobs>`' and
|
||||
* `<desk-schedules>`' business, and they draw themselves. The single stages
|
||||
* have no buttons: the cascade is the unit of work, and `app job <stage>`
|
||||
* keeps the per-stage entry points.
|
||||
*
|
||||
* Both read the `app:jobs` snapshot from live.js rather than asking the
|
||||
* elements, which keep to a shadow root and owe nobody an answer.
|
||||
*
|
||||
* Failures are the server's sentence through `api()`, which toasts it -- so
|
||||
* there is no error handling here beyond keeping the button usable.
|
||||
@@ -14,51 +15,49 @@
|
||||
(function () {
|
||||
"use strict";
|
||||
|
||||
const host = document.getElementById("kinds");
|
||||
const pullButton = document.getElementById("pull-now");
|
||||
|
||||
async function start(kind, button) {
|
||||
button.disabled = true;
|
||||
try {
|
||||
const job = await window.api("/api/jobs", { method: "POST", body: { kind } });
|
||||
window.toast.good(`${job.label || kind}: queued.`);
|
||||
} catch (e) {
|
||||
/* already toasted */
|
||||
} finally {
|
||||
button.disabled = false;
|
||||
}
|
||||
let pending = false; // the POST is in the air
|
||||
let pulling = false; // the latest snapshot has a live pull cascade
|
||||
|
||||
/* A pull queued by the button or a schedule is the `pull` job, which
|
||||
begins a cascade of its own named `pull on its own`, and the chain's
|
||||
tasks join it; `pull` is the name a cascade started as one would have.
|
||||
Live while any task has a job queued or running: the pipeline keeps a
|
||||
finished cascade in the snapshot for a few seconds, and that is not a
|
||||
pull still going. As newsdesk's Fetch: the whole cascade, not the `pull`
|
||||
task, which is over seconds in with five tasks to go. */
|
||||
const isPull = (c) => c.name === "pull" || c.name === "pull on its own";
|
||||
const live = (c) => (c.tasks || []).some((t) => t.running || t.queued);
|
||||
|
||||
function paint() {
|
||||
if (!pullButton) return;
|
||||
pullButton.disabled = pending || pulling;
|
||||
pullButton.title = pulling ? "A pull is running." : "";
|
||||
}
|
||||
|
||||
if (pullButton) {
|
||||
pullButton.addEventListener("click", () => start("pull", pullButton));
|
||||
}
|
||||
|
||||
async function draw() {
|
||||
if (!host) return;
|
||||
let kinds;
|
||||
try {
|
||||
kinds = await window.api("/api/jobs/kinds");
|
||||
} catch (e) {
|
||||
host.innerHTML = '<span class="muted">could not read the job kinds</span>';
|
||||
return;
|
||||
}
|
||||
kinds = kinds.filter((k) => k.kind !== "pull");
|
||||
if (!kinds.length) {
|
||||
host.innerHTML = '<span class="muted">nothing is registered</span>';
|
||||
return;
|
||||
}
|
||||
host.innerHTML = "";
|
||||
kinds.forEach((k) => {
|
||||
const button = document.createElement("button");
|
||||
button.type = "button";
|
||||
button.className = "secondary";
|
||||
button.textContent = k.label || k.kind;
|
||||
button.title = k.solo ? `${k.kind} -- one at a time` : k.kind;
|
||||
button.dataset.kind = k.kind;
|
||||
button.addEventListener("click", () => start(k.kind, button));
|
||||
host.appendChild(button);
|
||||
pullButton.addEventListener("click", async () => {
|
||||
pending = true;
|
||||
paint();
|
||||
try {
|
||||
const job = await window.api("/api/jobs", { method: "POST", body: { kind: "pull" } });
|
||||
window.toast.good(`${job.label || "pull"}: queued.`);
|
||||
} catch (e) {
|
||||
/* already toasted */
|
||||
} finally {
|
||||
pending = false;
|
||||
paint();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
document.addEventListener("DOMContentLoaded", draw);
|
||||
function update(state) {
|
||||
pulling = (state.cascades || []).some((c) => isPull(c) && live(c));
|
||||
paint();
|
||||
}
|
||||
|
||||
document.addEventListener("app:jobs", (e) => update(e.detail || {}));
|
||||
// A snapshot that arrived before this script loaded.
|
||||
if (window.live && window.live.jobs) update(window.live.jobs);
|
||||
})();
|
||||
+14
-9
@@ -206,17 +206,19 @@
|
||||
|
||||
// ------------------------------------------------------ one section shown
|
||||
|
||||
// Sources shows one section: the one the fragment names, or the first when
|
||||
// there is none or it names none, so `/settings/sources#runs` (a rail link,
|
||||
// an old path's 301, the Search agents page's "follow it under Runs") opens
|
||||
// Runs. The rest are hidden, not removed, so their scripts still find their
|
||||
// roots. It runs as the script loads rather than on DOMContentLoaded, so the
|
||||
// page never paints all four first.
|
||||
// Sources shows one section: the one the fragment names or contains, or the
|
||||
// first when there is none or it names none, so `/settings/sources#runs` (a
|
||||
// rail link, an old path's 301, the Search agents page's "follow it under
|
||||
// Runs") opens Runs, and so does `#messages`, the log at its end. The rest
|
||||
// are hidden, not removed, so their scripts still find their roots. It runs
|
||||
// as the script loads rather than on DOMContentLoaded, so the page never
|
||||
// paints all three first.
|
||||
function showSection() {
|
||||
const sections = [...document.querySelectorAll("section[data-section]")];
|
||||
if (!sections.length) return;
|
||||
const want = decodeURIComponent(location.hash.slice(1));
|
||||
const shown = sections.find((s) => s.id === want) || sections[0];
|
||||
const target = want && document.getElementById(want);
|
||||
const shown = sections.find((s) => target && s.contains(target)) || sections[0];
|
||||
for (const section of sections) section.hidden = section !== shown;
|
||||
for (const link of document.querySelectorAll(".settings-rail a[data-section]")) {
|
||||
if (link.dataset.section === shown.id) {
|
||||
@@ -229,10 +231,13 @@
|
||||
}
|
||||
|
||||
showSection();
|
||||
// A switch starts the section at the top of the page, as opening it would.
|
||||
// A switch starts the section at the top of the page, as opening it would;
|
||||
// a fragment inside one (`#messages`) scrolls to itself.
|
||||
window.addEventListener("hashchange", () => {
|
||||
showSection();
|
||||
window.scrollTo(0, 0);
|
||||
const target = document.getElementById(decodeURIComponent(location.hash.slice(1)));
|
||||
if (target && !target.matches("section[data-section]")) target.scrollIntoView();
|
||||
else window.scrollTo(0, 0);
|
||||
});
|
||||
|
||||
// -------------------------------------------------------------------- init
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
<p class="lede">The one password, the devices signed in with it, and the passkeys that sign in without it.</p>
|
||||
|
||||
<section class="card">
|
||||
{# As on the Pulls page: the element and its script belong to
|
||||
{# As in Sources' Runs: the element and its script belong to
|
||||
the module, so this page is two lines and `auth/` owns what they draw.
|
||||
|
||||
It says its own failures in place, beside the button that caused them,
|
||||
|
||||
@@ -1,14 +1,15 @@
|
||||
{% extends "settings/_layout.html" %}
|
||||
{% block title %}Sources — roledesk{% endblock %}
|
||||
|
||||
{# One page, four sections: the `search` plugin's job board queries, the
|
||||
`ats` plugin's company boards, the runs that read both, and the message
|
||||
log. One is shown at a time, the one the URL fragment names (`#runs`), or
|
||||
Job boards without one; sources.js hides the rest and the rail picks one.
|
||||
Hidden rather than separate pages, because each section keeps the element
|
||||
ids it had as a page of its own and its script (sources.js, companies.js,
|
||||
jobs.js, messages.js) is unchanged: it finds its root or does nothing. The
|
||||
ids are `SECTIONS` in app/web/pages.py, which the rail links to. #}
|
||||
{# One page, three sections: the `search` plugin's job board queries, the
|
||||
`ats` plugin's company boards, and the runs that read both, with the
|
||||
message log as their last block. One is shown at a time, the one the URL
|
||||
fragment names (`#runs`) or contains (`#messages`), or Job boards without
|
||||
one; sources.js hides the rest and the rail picks one. Hidden rather than
|
||||
separate pages, because each section keeps the element ids it had as a
|
||||
page of its own and its script (sources.js, companies.js, jobs.js,
|
||||
messages.js) is unchanged: it finds its root or does nothing. The ids are
|
||||
`SECTIONS` in app/web/pages.py, which the rail links to. #}
|
||||
{% block content %}
|
||||
<h1>Sources</h1>
|
||||
<p class="lede">Sources are where roles come from: queries on job boards that span companies, and the boards of the companies you follow. Each section is one plugin; the runs read them all.</p>
|
||||
@@ -109,9 +110,9 @@
|
||||
<section class="card" id="runs" data-section>
|
||||
<div class="row-between">
|
||||
<h2>Runs</h2>
|
||||
<button type="button" id="pull-now" class="secondary">Pull now</button>
|
||||
<button type="button" id="pull-now" class="secondary">Run the pull now</button>
|
||||
</div>
|
||||
<p class="lede">Every 6 hours the pull reads every board, embed and score rank every role, extract fills in what the best postings say, score runs again with what it filled, and summarise writes the summaries. Change the cadence below.</p>
|
||||
<p class="lede">On the schedule below the pull reads every board, embed and score rank every role, extract fills in what the best postings say, score runs again with what it filled, and summarise writes the summaries.</p>
|
||||
|
||||
<h3>What is running</h3>
|
||||
{# The element is the queue's own, served by its own router — see
|
||||
@@ -129,22 +130,16 @@
|
||||
<script type="module" src="/_jobs/schedules.js"></script>
|
||||
<desk-schedules add="off"></desk-schedules>
|
||||
|
||||
<details class="stage">
|
||||
<summary>Run one stage</summary>
|
||||
<div id="kinds" class="buttons">
|
||||
<span class="muted">loading…</span>
|
||||
{# The message log, the queue's own through `/api/notices`. `#messages` is
|
||||
where the old Messages paths and the toasts' "all messages" link land. #}
|
||||
<div id="messages">
|
||||
<div class="row-between">
|
||||
<h3>Problems</h3>
|
||||
<button type="button" id="clear-all" class="secondary">Clear all</button>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
</section>
|
||||
|
||||
<section class="card" id="messages" data-section>
|
||||
<div class="row-between">
|
||||
<h2>Messages</h2>
|
||||
<button type="button" id="clear-all" class="secondary">Clear all</button>
|
||||
<p class="lede">What the server said that nobody has cleared, newest first. A message goes when you clear it, or a fortnight after it was last said.</p>
|
||||
<div id="notices"><span class="muted">loading…</span></div>
|
||||
</div>
|
||||
<p class="lede">What the server said that nobody has cleared, newest first. A message goes when you clear it, or a fortnight after it was last said.</p>
|
||||
<div id="notices"><span class="muted">loading…</span></div>
|
||||
</section>
|
||||
{% endblock %}
|
||||
|
||||
|
||||
+10
-8
@@ -16,17 +16,19 @@ one page, so a link to a role and the role opened from the feed are the same
|
||||
screen. What the server renders is only what a select needs to offer, so the
|
||||
options cannot drift from what the store accepts.
|
||||
|
||||
Sources is one page in four sections: job boards (the `search` plugin's
|
||||
queries), company boards (`ats`), the runs that read them, and the message
|
||||
log. One is shown at a time, the one the URL fragment names; the others stay
|
||||
in the page, hidden, so each section's script (the ones they had as pages of
|
||||
their own, each guarding on its own root element) keeps working.
|
||||
Sources is one page in three sections: job boards (the `search` plugin's
|
||||
queries), company boards (`ats`), and the runs that read them, which end in
|
||||
the message log. One is shown at a time, the one the URL fragment names or
|
||||
contains; the others stay in the page, hidden, so each section's script (the
|
||||
ones they had as pages of their own, each guarding on its own root element)
|
||||
keeps working.
|
||||
|
||||
Every path a page has had before (`/agents`, `/companies`, `/sources`,
|
||||
`/jobs`, `/schedules`, `/messages`, `/account`, and the six sub-tabs the
|
||||
settings shell started with) answers 301 to its place now, query string
|
||||
kept, so a bookmark still lands. A merged page's old path lands on its
|
||||
section's anchor.
|
||||
section's anchor; the old Messages paths land on `#messages`, the log's
|
||||
block at the end of Runs.
|
||||
|
||||
`FUNCTIONS` is `plugins.ats`'s, not `app`'s: the nine job functions are the
|
||||
classifier's vocabulary, and this reaches `plugins/` the same way `app.cli`
|
||||
@@ -88,12 +90,12 @@ SECTIONS: dict[str, tuple[tuple[str, str], ...]] = {
|
||||
("job-boards", "Job boards"),
|
||||
("company-boards", "Company boards"),
|
||||
("runs", "Runs"),
|
||||
("messages", "Messages"),
|
||||
),
|
||||
}
|
||||
|
||||
#: Where each page that used to be a tab or a sub-tab lives now. The fragment
|
||||
#: is the section on the merged page; a browser keeps it across the 301.
|
||||
#: is the section on the merged page, or an element in one (`#messages`, in
|
||||
#: Runs); a browser keeps it across the 301.
|
||||
MOVED: dict[str, str] = {
|
||||
"/agents": "/settings/agents",
|
||||
"/settings/ranking": "/settings/agents",
|
||||
|
||||
@@ -112,6 +112,8 @@ What was decided for roledesk and the one reason for each. The longer reasoning
|
||||
|
||||
**Settings pages sit beside a rail, and Sources shows one section at a time, chosen by the URL fragment.** The company boards list runs to 50 or 60 companies, so Runs at the end of a stacked page was out of reach and jump links still meant scrolling back up. The rail (`settings/_rail.html`, from `SETTINGS` and `SECTIONS` in `app/web/pages.py`) is sticky beside the content, and a strip under the header below 60rem. The fragment rather than a path per section, because the old paths already 301 to `/settings/sources#runs` and the like, and a hidden section stays in the DOM, so each section's script runs unchanged.
|
||||
|
||||
**Runs is the queue's own elements plus a Problems log.** `<desk-jobs>` and `<desk-schedules>` draw what is running and when the pull runs next, the app adds only "Run the pull now", disabled while a pull cascade is live, and the message log is the section's last block rather than a section of its own; `#messages` still lands on it. "Run one stage" went because the cascade is the unit of work: a stage run alone is a partial pull whose result the next stage never sees. `app job <stage>` and `POST /api/jobs` keep the per-stage entry points, which is what "Save and score" uses.
|
||||
|
||||
**Search agents are cards, edited in place.** An agent is defined by what it looks for, so the intent is the card's body and the weights a small line under it; a list of names and weights hid the part that matters. Edit turns that card into the form and New puts a blank one on top, because the new agent sorts first by id and a form at the bottom of the page was out of sight of the card it changed.
|
||||
|
||||
**Affinity and votes are per agent.** A company hidden for one hunt may be a favourite for another, and a vote says what one agent should learn from, so both carry the agent id, and the Roles tab and the company list ask for one agent's. Without an agent id the API uses the default agent, the enabled one with the lowest id, so the CLI and older callers keep working.
|
||||
|
||||
+45
-45
@@ -532,50 +532,50 @@ class ATSSource(Source):
|
||||
)
|
||||
|
||||
def fetch(self) -> Iterator[Item]:
|
||||
client = httpx.Client()
|
||||
# Written by the pull job before any board is fetched (see
|
||||
# `app/jobs/pull.py`); a plugin run on its own — a test, a dry
|
||||
# run — has no run to stamp itself against, so it makes one up.
|
||||
run_marker = self.state.get("ats:run", "") or datetime.now(UTC).isoformat(
|
||||
timespec="seconds")
|
||||
with httpx.Client() as client:
|
||||
# Written by the pull job before any board is fetched (see
|
||||
# `app/jobs/pull.py`); a plugin run on its own — a test, a dry
|
||||
# run — has no run to stamp itself against, so it makes one up.
|
||||
run_marker = self.state.get("ats:run", "") or datetime.now(UTC).isoformat(
|
||||
timespec="seconds")
|
||||
|
||||
for target, label in self.targets():
|
||||
host, _, slug = target.partition(":")
|
||||
try:
|
||||
rows = _fetch_rows(client, target)
|
||||
except Exception as exc:
|
||||
log.warning("ats: %s failed (%s)", target, exc)
|
||||
continue
|
||||
for target, label in self.targets():
|
||||
host, _, slug = target.partition(":")
|
||||
try:
|
||||
rows = _fetch_rows(client, target)
|
||||
except Exception as exc:
|
||||
log.warning("ats: %s failed (%s)", target, exc)
|
||||
continue
|
||||
|
||||
seen_ids = []
|
||||
for row in rows:
|
||||
# The compound id, matching `roles.source_id` — what
|
||||
# `Store.mark_closed` compares against, not the board's own
|
||||
# bare job id.
|
||||
compound_id = f"{host}:{slug}:{row['source_id']}"
|
||||
seen_ids.append(compound_id)
|
||||
yield Item(
|
||||
source_id=compound_id,
|
||||
location=row["url"],
|
||||
title=row["title"],
|
||||
excerpt=(row["description"] or "")[:300],
|
||||
body=row["description"],
|
||||
published_at=row["posted"],
|
||||
subscription=label,
|
||||
kind="role",
|
||||
raw={
|
||||
"company": label,
|
||||
"place": row["location"],
|
||||
"department": row["department"],
|
||||
"function": classify(row["department"], row["title"]),
|
||||
"remote": row["remote"],
|
||||
"salary": row["salary"],
|
||||
"board": host,
|
||||
"slug": slug,
|
||||
},
|
||||
)
|
||||
# Stamped only once the board has parsed without raising, so a
|
||||
# board that blew up half way through is neither marked ok nor
|
||||
# has a partial id list treated as the whole of it.
|
||||
self.state[f"ats:ok:{target}"] = run_marker
|
||||
self.state[f"ats:ids:{target}"] = json.dumps(seen_ids)
|
||||
seen_ids = []
|
||||
for row in rows:
|
||||
# The compound id, matching `roles.source_id` — what
|
||||
# `Store.mark_closed` compares against, not the board's own
|
||||
# bare job id.
|
||||
compound_id = f"{host}:{slug}:{row['source_id']}"
|
||||
seen_ids.append(compound_id)
|
||||
yield Item(
|
||||
source_id=compound_id,
|
||||
location=row["url"],
|
||||
title=row["title"],
|
||||
excerpt=(row["description"] or "")[:300],
|
||||
body=row["description"],
|
||||
published_at=row["posted"],
|
||||
subscription=label,
|
||||
kind="role",
|
||||
raw={
|
||||
"company": label,
|
||||
"place": row["location"],
|
||||
"department": row["department"],
|
||||
"function": classify(row["department"], row["title"]),
|
||||
"remote": row["remote"],
|
||||
"salary": row["salary"],
|
||||
"board": host,
|
||||
"slug": slug,
|
||||
},
|
||||
)
|
||||
# Stamped only once the board has parsed without raising, so a
|
||||
# board that blew up half way through is neither marked ok nor
|
||||
# has a partial id list treated as the whole of it.
|
||||
self.state[f"ats:ok:{target}"] = run_marker
|
||||
self.state[f"ats:ids:{target}"] = json.dumps(seen_ids)
|
||||
+42
-42
@@ -308,46 +308,46 @@ class SearchSource(Source):
|
||||
)
|
||||
|
||||
def fetch(self) -> Iterator[Item]:
|
||||
client = httpx.Client()
|
||||
run_marker = self.state.get("search:run", "") or datetime.now(UTC).isoformat(
|
||||
timespec="seconds")
|
||||
for target, label in self.targets():
|
||||
provider = provider_of(target)
|
||||
if not provider:
|
||||
log.warning("search: %r is not provider:name; skipped", target)
|
||||
continue
|
||||
try:
|
||||
rows, complete = _SEARCHES[provider](
|
||||
client, self.params_for(target), self.options.get(provider) or {})
|
||||
except Exception as exc:
|
||||
log.warning("search: %s failed (%s)", target, exc)
|
||||
continue
|
||||
with httpx.Client() as client:
|
||||
run_marker = self.state.get("search:run", "") or datetime.now(UTC).isoformat(
|
||||
timespec="seconds")
|
||||
for target, label in self.targets():
|
||||
provider = provider_of(target)
|
||||
if not provider:
|
||||
log.warning("search: %r is not provider:name; skipped", target)
|
||||
continue
|
||||
try:
|
||||
rows, complete = _SEARCHES[provider](
|
||||
client, self.params_for(target), self.options.get(provider) or {})
|
||||
except Exception as exc:
|
||||
log.warning("search: %s failed (%s)", target, exc)
|
||||
continue
|
||||
|
||||
seen_ids = []
|
||||
for row in rows:
|
||||
source_id = f"{provider}:{row['id']}"
|
||||
seen_ids.append(source_id)
|
||||
yield Item(
|
||||
source_id=source_id,
|
||||
location=row["url"],
|
||||
title=row["title"],
|
||||
excerpt=(row["description"] or "")[:300],
|
||||
body=row["description"],
|
||||
published_at=row["posted"],
|
||||
subscription=label,
|
||||
kind="role",
|
||||
raw={
|
||||
"company": row["company"],
|
||||
"place": row["place"],
|
||||
"country": row["country"],
|
||||
"remote": row["remote"],
|
||||
"salary_min": row["salary_min"],
|
||||
"salary_max": row["salary_max"],
|
||||
"currency": row["currency"],
|
||||
"provider": provider,
|
||||
"search": target,
|
||||
},
|
||||
)
|
||||
self.state[f"search:ok:{target}"] = run_marker
|
||||
self.state[f"search:complete:{target}"] = "1" if complete else ""
|
||||
self.state[f"search:ids:{target}"] = json.dumps(seen_ids)
|
||||
seen_ids = []
|
||||
for row in rows:
|
||||
source_id = f"{provider}:{row['id']}"
|
||||
seen_ids.append(source_id)
|
||||
yield Item(
|
||||
source_id=source_id,
|
||||
location=row["url"],
|
||||
title=row["title"],
|
||||
excerpt=(row["description"] or "")[:300],
|
||||
body=row["description"],
|
||||
published_at=row["posted"],
|
||||
subscription=label,
|
||||
kind="role",
|
||||
raw={
|
||||
"company": row["company"],
|
||||
"place": row["place"],
|
||||
"country": row["country"],
|
||||
"remote": row["remote"],
|
||||
"salary_min": row["salary_min"],
|
||||
"salary_max": row["salary_max"],
|
||||
"currency": row["currency"],
|
||||
"provider": provider,
|
||||
"search": target,
|
||||
},
|
||||
)
|
||||
self.state[f"search:ok:{target}"] = run_marker
|
||||
self.state[f"search:complete:{target}"] = "1" if complete else ""
|
||||
self.state[f"search:ids:{target}"] = json.dumps(seen_ids)
|
||||
@@ -16,6 +16,7 @@ if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from desk.models import Item # noqa: E402
|
||||
from pipeline import Cancelled, RunContext # noqa: E402
|
||||
from plugins.ats import ATSSource # noqa: E402
|
||||
from app.jobs import pull # noqa: E402
|
||||
from app.storage import Store # noqa: E402
|
||||
@@ -81,6 +82,35 @@ def test_run_batch_upserts_every_item_the_plugin_yields(
|
||||
assert store.roles()[0].subscription_id is not None
|
||||
|
||||
|
||||
def test_a_kill_stops_the_batch_before_the_next_board(
|
||||
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""`check()` before each board: a Kill pressed while the first is read
|
||||
ends the job before the second is fetched, and keeps what the first
|
||||
stored."""
|
||||
for slug in ("acme", "beta"):
|
||||
store.add_subscription("ats", f"greenhouse:{slug}", label=slug.title(),
|
||||
company=slug.title(), board="greenhouse")
|
||||
fetched: list[str] = []
|
||||
killed = False
|
||||
|
||||
def fake_fetch(self: ATSSource) -> object:
|
||||
nonlocal killed
|
||||
[(target, _)] = self.targets()
|
||||
fetched.append(target)
|
||||
yield _item("greenhouse", target.partition(":")[2], "1")
|
||||
killed = True
|
||||
|
||||
monkeypatch.setattr(ATSSource, "fetch", fake_fetch)
|
||||
context = RunContext(is_cancelled=lambda: killed)
|
||||
|
||||
with pytest.raises(Cancelled):
|
||||
pull.run_batch(FakeConfig(), store, targets=["greenhouse:acme", "greenhouse:beta"],
|
||||
context=context)
|
||||
|
||||
assert fetched == ["greenhouse:acme"]
|
||||
assert {r.source_id for r in store.roles()} == {"greenhouse:acme:1"}
|
||||
|
||||
|
||||
def test_close_boards_closes_what_a_completed_board_no_longer_lists(
|
||||
store: Store) -> None:
|
||||
sub = store.add_subscription("ats", "greenhouse:acme", label="Acme",
|
||||
|
||||
+37
-16
@@ -549,18 +549,22 @@ def test_every_settings_page_has_the_rail_and_not_the_tab_row(client: TestClient
|
||||
|
||||
def test_a_moved_section_is_one_the_sources_page_shows(client: TestClient) -> None:
|
||||
"""Every old path that lands on a fragment of Sources names a section of
|
||||
`SECTIONS`, and each is a `<section data-section>` on the page, which is
|
||||
what sources.js shows one of."""
|
||||
`SECTIONS` or an element inside one, and each section is a `<section
|
||||
data-section>` on the page, which is what sources.js shows one of. The
|
||||
old Messages paths land on `#messages`, the log at the end of Runs."""
|
||||
anchors = [anchor for anchor, _ in SECTIONS["/settings/sources"]]
|
||||
assert anchors == ["job-boards", "company-boards", "runs", "messages"]
|
||||
for to in MOVED.values():
|
||||
path, _, anchor = to.partition("#")
|
||||
if anchor:
|
||||
assert path == "/settings/sources" and anchor in anchors, to
|
||||
assert anchors == ["job-boards", "company-boards", "runs"]
|
||||
page = client.get("/settings/sources").text
|
||||
assert page.count("data-section>") == len(anchors)
|
||||
for anchor in anchors:
|
||||
assert f'<section class="card" id="{anchor}" data-section>' in page
|
||||
runs = page[page.index('id="runs"'):]
|
||||
for to in MOVED.values():
|
||||
path, _, anchor = to.partition("#")
|
||||
if anchor:
|
||||
assert path == "/settings/sources", to
|
||||
assert anchor in anchors or f'id="{anchor}"' in runs, to
|
||||
assert MOVED["/messages"] == MOVED["/settings/messages"] == "/settings/sources#messages"
|
||||
|
||||
|
||||
def test_settings_is_search_agents(client: TestClient) -> None:
|
||||
@@ -586,8 +590,9 @@ def test_an_old_tab_moves_permanently_and_keeps_its_query(
|
||||
|
||||
|
||||
def test_each_page_loads_only_its_own_script(client: TestClient) -> None:
|
||||
"""Sources is four sections' scripts on one page, each guarding on its own
|
||||
root element; every other page is one."""
|
||||
"""Sources is four scripts on one page, one per section and one for the
|
||||
log in Runs, each guarding on its own root element; every other page is
|
||||
one."""
|
||||
scripts = {"roles.js", "applications.js", "agents.js", "companies.js", "sources.js",
|
||||
"jobs.js", "messages.js"}
|
||||
for path, container, own in [
|
||||
@@ -632,14 +637,15 @@ def test_the_agents_page_lists_and_edits_search_agents(client: TestClient) -> No
|
||||
|
||||
|
||||
def test_the_sources_page_has_a_section_per_plugin(client: TestClient) -> None:
|
||||
"""Job boards, company boards, the runs and the message log, in that
|
||||
order, each linked from the rail; no jump row any more."""
|
||||
"""Job boards, company boards and the runs, in that order, each linked
|
||||
from the rail; no jump row any more."""
|
||||
page = client.get("/settings/sources").text
|
||||
assert '<aside class="settings-rail"' in page and 'class="jump"' not in page
|
||||
for anchor in ("job-boards", "company-boards", "runs", "messages"):
|
||||
for anchor in ("job-boards", "company-boards", "runs"):
|
||||
assert f'href="/settings/sources#{anchor}"' in page and f'id="{anchor}"' in page
|
||||
assert page.index('id="job-boards"') < page.index('id="company-boards"') \
|
||||
< page.index('id="runs"') < page.index('id="messages"')
|
||||
< page.index('id="runs"')
|
||||
assert 'href="/settings/sources#messages"' not in page
|
||||
# Job boards: the Jobindex areas and the Adzuna countries, from plugins.search.
|
||||
assert 'id="searches-list"' in page and 'id="search-form"' in page
|
||||
assert 'value="1" selected>Region Hovedstaden' in page
|
||||
@@ -650,9 +656,16 @@ def test_the_sources_page_has_a_section_per_plugin(client: TestClient) -> None:
|
||||
assert '<option value="greenhouse">' in page and '<option value="personio">' in page
|
||||
boards = page[page.index('id="company-boards"'):page.index('id="runs"')].lower()
|
||||
assert "hidden" in boards and "mute" not in boards
|
||||
# Runs: Pull now, the queue's elements, one stage at a time, the log.
|
||||
assert 'id="pull-now"' in page and 'id="kinds"' in page
|
||||
assert 'id="notices"' in page and 'id="clear-all"' in page
|
||||
# Runs: the pull button, the queue's elements, and the log as its last
|
||||
# block. No button per stage, and no cadence in the lede.
|
||||
runs = page[page.index('id="runs"'):page.index("</section>", page.index('id="runs"'))]
|
||||
assert '<button type="button" id="pull-now" class="secondary">Run the pull now</button>' in runs
|
||||
assert 'id="kinds"' not in page and "Run one stage" not in page
|
||||
assert "On the schedule below the pull reads every board" in runs
|
||||
assert "Every 6 hours" not in runs
|
||||
assert '<desk-jobs idle="show">' in runs and 'id="idle"' not in runs
|
||||
assert runs.index('<desk-schedules add="off">') < runs.index('<div id="messages">') \
|
||||
< runs.index("<h3>Problems</h3>") < runs.index('id="clear-all"') < runs.index('id="notices"')
|
||||
|
||||
|
||||
def test_the_modules_bring_their_own_elements(client: TestClient) -> None:
|
||||
@@ -690,6 +703,14 @@ def test_the_jobs_tab_offers_pull_once_and_no_batch(client: TestClient) -> None:
|
||||
assert not {"pull_batch", "extract_batch", "advance"} & set(kinds)
|
||||
|
||||
|
||||
def test_save_and_score_queues_a_score(client: TestClient, queue: Any) -> None:
|
||||
"""The Search agents page's "Save and score" posts `{kind: "score"}`: a
|
||||
task on its own, now that no stage has a button of its own."""
|
||||
res = client.post("/api/jobs", json={"kind": "score", "payload": {}})
|
||||
assert res.status_code == 202
|
||||
assert queue.jobs.last("score", states=("queued", "running", "done")) is not None
|
||||
|
||||
|
||||
def test_an_error_is_one_sentence(client: TestClient) -> None:
|
||||
"""`api()` toasts `detail` as it is, so a 422 is a sentence, not a list."""
|
||||
res = client.post("/api/agents", json={"name": "Bad",
|
||||
|
||||
Reference in new issue
Block a user