Merge branch 'r2-web'

# Conflicts:
#	app/static/roles.js
#	app/web/api.py
#	tests/test_web.py
This commit is contained in:
Wolf committed 2026-09-29 15:31:35 +02:00
commit a629eda8de
13 files changed
+548 -203

No files matched your search

+2 -2
View File
@@ -28,9 +28,9 @@ uv run app companies set-board lunar greenhouse:lunar # attach a board to a com
uv run app companies detach lunar # stop reading its board; the company stays
```
The seed is a hand-picked list of European tech employers, not a list of boards: a company whose careers site runs on something the app cannot read yet (Workday, SuccessFactors, Teamtailor, its own pages) is still a row, with that host and its slug there where one was found, shown under Companies as "not readable yet" with a link to its careers page, so the gap is visible and one parser at a time closes it. A row where nothing was found reads "board not found"; the "Not readable yet" filter shows both. Each row has a Set board control (Change once it has one) to attach, replace or detach its board by hand. Replacing or detaching closes the old board's open roles; a board is read once before it is kept, and one that lists no roles yet is kept too. `companies discover` reads a company's site for a link into one of the boards it knows, probes the hosts with a cheap per-slug endpoint only when no page names any host, and with `--add` sets the board of a new or existing company, or records the host it found when nothing readable was. A probe hit is a guess by name: check the roles it lists before trusting it for a common name.
The seed is a hand-picked list of European tech employers, not a list of boards: a company whose careers site runs on something the app cannot read yet (Workday, SuccessFactors, Teamtailor, its own pages) is still a row, with that host and its slug there where one was found, shown under Companies as "not readable yet" with a link to its careers page, so the gap is visible and one parser at a time closes it. A row where nothing was found reads "board not found"; the "Not readable yet" filter shows both. Each row has a Set board control (Change once it has one) to attach, replace or detach its board by hand. Replacing or detaching closes the old board's open roles, and asks first when there are open roles to lose; a board is read once before it is kept, and one that lists no roles yet is kept too. Setting a board queues a pull right after, so its roles are read by the one already running or the very next one, rather than waiting for the schedule. `companies discover` reads a company's site for a link into one of the boards it knows, probes the hosts with a cheap per-slug endpoint only when no page names any host, and with `--add` sets the board of a new or existing company, or records the host it found when nothing readable was. A probe hit is a guess by name: check the roles it lists before trusting it for a common name.
A Jobindex query is its own syntax: words are OR, `"a b"` is a phrase, `+word` is required; `--area 1` is Region Hovedstaden. An Adzuna search needs a free key from developer.adzuna.com in `ADZUNA_APP_ID` and `ADZUNA_APP_KEY`. `searches add` runs the search once and saves it only if it finds something. Switching a board or search off closes its open roles; the next pull after switching it back on reopens the ones it still lists. The `embeddings` extra is what `fit` needs, `agent` is for `extract` on a model, `passkeys` for the Account tab's passkeys.
A Jobindex query is its own syntax: words are OR, `"a b"` is a phrase, `+word` is required; `--area 1` is Region Hovedstaden. An Adzuna search needs a free key from developer.adzuna.com in `ADZUNA_APP_ID` and `ADZUNA_APP_KEY`. `searches add` runs the search once and saves it only if it finds something. Switching a board or search off closes its open roles, and asks first when there are open roles to lose; switching it back on, or saving a search, queues a pull right after, so the ones it still lists reopen without waiting for the schedule. The `embeddings` extra is what `fit` needs, `agent` is for `extract` on a model, `passkeys` for the Account tab's passkeys.
## The tabs
+68 -2
View File
@@ -823,13 +823,79 @@ class ExtractResult(ModelTaskResult):
class ScoredRole(Role):
"""A role as `/api/roles` serves it: with one agent's score and gate
reason when an agent was asked for, and without otherwise."""
"""A role as `/api/roles/{uid}` serves it: with one agent's score and
gate reason when an agent was asked for, and without otherwise."""
score: float | None = None
gated_reason: str = ""
#: How many characters `RoleRow.excerpt` keeps at most: the feed row's own
#: cap, built once here rather than by the client, so a list response never
#: carries more of a role's text than a row draws.
EXCERPT_LIMIT = 200
def excerpt(text: str, limit: int = EXCERPT_LIMIT) -> str:
"""`text` (a role's summary, or the opening of its description) with its
whitespace collapsed, cut to `limit` characters at a word boundary with
an ellipsis. What `Store.roles_page` builds `RoleRow.excerpt` with, off a
`SUBSTR` of the source column rather than the whole of it, so the feed's
list never carries what only the reader needs."""
collapsed = " ".join((text or "").split())
if len(collapsed) <= limit:
return collapsed
cut = collapsed[:limit]
space = cut.rfind(" ")
trimmed = cut[:space] if space > limit * 0.6 else cut
return trimmed.rstrip(" ,.;:" + chr(0x2013) + "-") + "…"
class RoleRow(Model):
"""One row of the feed: what `GET /api/roles` serves per role, the
fields a row draws and nothing else — never `description`, `feed_text`,
`raw` or `article_text`, the four columns docs/decisions.md measures at
about 5.5 KB a role between them. The reader fetches the rest by uid when
it opens a role (`GET /api/roles/{uid}`, the full `Role`).
`excerpt` is built on the server (see `excerpt`, above): the summary when
there is one, else the description's opening, cut to `EXCERPT_LIMIT`
characters. The pay fields are what the row's badge reads: the three
normalised columns, the line the board stated, and `salary_state`,
derived as on `Role`. `original_url` is one short string, kept so the
`o` key opens a selected row's posting without opening the reader."""
uid: str
company_key: str = ""
company: str = ""
title: str = ""
place: str = ""
country: str = ""
remote: Remote = ""
function: str = ""
salary_min: int | None = None
salary_max: int | None = None
currency: str = ""
salary_text: str = ""
original_url: str = ""
posted_at: str = ""
score: float | None = None
gated_reason: str = ""
vote: Literal[-1, 0, 1] | None = None
voted_at: str | None = None
read_at: str | None = None
tags: list[str] = []
excerpt: str = ""
@computed_field # type: ignore[prop-decorator]
@property
def salary_state(self) -> str:
"""As `Role.salary_state`: 'normalised', 'stated' or ''."""
if self.salary_min is not None or self.salary_max is not None:
return "normalised"
return "stated" if self.salary_text else ""
class Company(Model):
"""A row of `companies`, with `affinity` as the agent it was read for
has it (see `agent_companies`); 0 when no agent was asked for."""
+16 -4
View File
@@ -14,9 +14,10 @@
* forgetting the header is the failure that looks like a server bug.
*
* Beside `api` on `window`, the few helpers more than one page script uses:
* `count`, `cell`, `debounce`, and the company list with its affinity control
* (`loadCompanies`, `companies`, `affinityControl`, `setAffinity`). One copy
* each, since two copies of a helper is one of them being fixed.
* `count`, `cell`, `debounce`, `safeUrl`, and the company list with its
* affinity control (`loadCompanies`, `companies`, `affinityControl`,
* `setAffinity`). One copy each, since two copies of a helper is one of them
* being fixed.
*
* Affinity is per search agent. The three company helpers take the agent's
* id as an optional last argument and send it along; a caller that passes
@@ -104,6 +105,17 @@
};
}
/** An `http(s)` URL, or "" for anything else — `javascript:`, a bare path,
empty. Every link whose address came from a feed, a board, discovery or
a person (a search result's `url`, a company's `careers_url`, a saved
application's `url`, a role's `original_url`) goes through this before
it lands in an `href`, so a board that wrote something other than a URL
cannot make this page navigate to it. Moved here from roles.js, which
first needed it for the reader's outbound links. */
function safeUrl(u) {
return /^https?:\/\//i.test(u || "") ? u : "";
}
// --------------------------------------------------------------- companies
/** @type {Map<string, object>} company_key -> row from /api/companies. One
@@ -170,6 +182,6 @@
}
Object.assign(window, {
api, count, cell, debounce, companies, loadCompanies, affinityControl, setAffinity,
api, count, cell, debounce, safeUrl, companies, loadCompanies, affinityControl, setAffinity,
});
})();
+4 -3
View File
@@ -9,7 +9,7 @@
(function () {
"use strict";
const { api, count, cell, companies, loadCompanies } = window;
const { api, count, cell, safeUrl, companies, loadCompanies } = window;
const $ = (sel, root = document) => root.querySelector(sel);
// `ROLEDESK_STATUSES` is set by applications.html from the same `STATUSES`
@@ -41,13 +41,14 @@
body.replaceChildren();
for (const application of applications) {
const tr = document.createElement("tr");
if (application.role_uid || application.url) {
const externalUrl = safeUrl(application.url);
if (application.role_uid || externalUrl) {
const link = document.createElement("a");
link.textContent = application.role_title || "(untitled)";
if (application.role_uid) {
link.href = `/roles/${encodeURIComponent(application.role_uid)}`;
} else {
link.href = application.url;
link.href = externalUrl;
link.target = "_blank";
link.rel = "noopener";
}
+45 -9
View File
@@ -21,7 +21,7 @@
(function () {
"use strict";
const { api, count, debounce, companies, loadCompanies, affinityControl } = window;
const { api, count, debounce, safeUrl, companies, loadCompanies, affinityControl } = window;
const $ = (sel, root = document) => root.querySelector(sel);
/** @type {object[]} the search agents, by id */
@@ -79,9 +79,10 @@
// where this company posts its roles.
sub.append("board not found");
}
if (company.careers_url && !company.board) {
const careersUrl = safeUrl(company.careers_url);
if (careersUrl && !company.board) {
const link = document.createElement("a");
link.href = company.careers_url;
link.href = careersUrl;
link.target = "_blank";
link.rel = "noopener";
link.textContent = "careers page";
@@ -150,13 +151,22 @@
form.addEventListener("submit", async (event) => {
event.preventDefault();
const board = `${host.value}:${slug.value.trim()}`;
// Replacing a different board closes its roles (see set_board in
// app/jobs/sources.py); ask first, naming both boards and the count,
// the same as Detach and a removed search already do.
if (company.board && company.board !== board
&& !window.confirm(`Replace ${company.name}'s board (${company.board}) `
+ `with ${board}? Its ${count(company.open_roles, "open role")} close.`)) {
return;
}
save.disabled = true;
try {
const result = await api(`/api/companies/${encodeURIComponent(company.key)}/board`,
{ method: "POST", body: { board: `${host.value}:${slug.value.trim()}` } });
{ method: "POST", body: { board } });
const closed = result.closed ? `, ${count(result.closed, "old role")} closed` : "";
window.toast.info(`${company.name}: reading ${result.source.target}, `
+ `${count(result.found, "role")} listed${closed}.`);
window.toast.info(`${company.name}: ${result.source.target} listed `
+ `${count(result.found, "role")}${closed}. Reading now, or with the pull right after.`);
await done();
} catch (err) {
save.disabled = false;
@@ -168,6 +178,11 @@
detach.className = "linky";
detach.textContent = "Detach";
detach.addEventListener("click", async () => {
if (company.open_roles
&& !window.confirm(`Detach ${company.name}'s board (${company.board})? `
+ `Its ${count(company.open_roles, "open role")} close.`)) {
return;
}
detach.disabled = true;
try {
const result = await api(`/api/companies/${encodeURIComponent(company.key)}/board`,
@@ -193,6 +208,14 @@
box.checked = Boolean(company.on);
box.setAttribute("aria-label", `Read ${company.name}'s board`);
box.addEventListener("change", async () => {
// Switching off closes every open role in one tap; ask first, naming
// the count, as removing a search and deleting an agent already do.
if (!box.checked && company.open_roles
&& !window.confirm(`Switch off ${company.name}'s board? `
+ `Its ${count(company.open_roles, "open role")} close.`)) {
box.checked = true;
return;
}
box.disabled = true;
try {
const result = await api(`/api/sources/${company.subscription_id}/switch`,
@@ -200,6 +223,8 @@
company.on = result.source.on;
if (!box.checked && result.closed) {
window.toast.info(`${company.name}: ${count(result.closed, "open role")} closed.`);
} else if (box.checked) {
window.toast.info(`${company.name}: reading now, or with the pull right after.`);
}
} catch (err) {
box.checked = !box.checked;
@@ -263,19 +288,30 @@
event.preventDefault();
const status = $("#board-status");
const button = $("#board-form button[type=submit]");
const name = $("#b-name").value.trim();
const board = `${$("#b-host").value}:${$("#b-slug").value.trim()}`;
// This name may already be a company with a different board: `set_board`
// would replace it and close its roles, the same as the row's own
// editor does, so it asks first too.
const existing = [...companies.values()]
.find((c) => c.name.toLowerCase() === name.toLowerCase());
if (existing && existing.board && existing.board !== board
&& !window.confirm(`Replace ${existing.name}'s board (${existing.board}) `
+ `with ${board}? Its ${count(existing.open_roles, "open role")} close.`)) {
return;
}
button.disabled = true;
status.textContent = "Reading the board once before following it…";
const board = `${$("#b-host").value}:${$("#b-slug").value.trim()}`;
try {
const result = await api("/api/boards", {
method: "POST",
body: { board, name: $("#b-name").value.trim() },
body: { board, name },
});
await loadCompanies(agentId());
render();
$("#board-form").reset();
status.textContent = `Following ${result.source.label}: `
+ `${count(result.found, "role")} listed. They arrive with the next pull.`;
+ `${count(result.found, "role")} listed. Reading now, or with the pull right after.`;
} catch (err) {
status.textContent = "";
}
+92 -42
View File
@@ -47,7 +47,7 @@
(function () {
"use strict";
const { api, esc, debounce, companies, loadCompanies, affinityControl, judgeButtons,
const { api, esc, debounce, safeUrl, companies, loadCompanies, affinityControl, judgeButtons,
loadTracked } = window;
const $ = (sel, root = document) => root.querySelector(sel);
@@ -107,10 +107,9 @@
pushedReader: false, // we own a history entry, so closing is `history.back()`
summarising: null, // the uid whose summary is being waited for
listY: 0, // where the page was scrolled when the reader replaced the feed
feedError: false, // the feed's own fetch failed; the empty state offers a retry
};
const safeUrl = (u) => (/^https?:\/\//i.test(u || "") ? u : "");
function ago(iso) {
const mins = (Date.now() - new Date(iso)) / 60000;
if (!Number.isFinite(mins)) return "";
@@ -202,11 +201,20 @@
try {
data = await api(`/api/roles?${params}`);
} catch (err) {
// An append failure leaves what is already on screen; a first load or
// a reload has nothing to show, so the empty state offers a retry
// rather than leaving the feed blank for good.
if (!append && request === feedRequest) {
state.feedError = true;
state.loaded = false;
renderFeed();
}
return;
} finally {
if (request === feedRequest) state.loading = false;
}
if (request !== feedRequest) return;
state.feedError = false;
const held = append ? state.items[state.selected]?.uid : state.readingUid;
state.total = data.total;
@@ -475,15 +483,23 @@
for (const el of had.values()) el.remove();
}
/* The summary when there is one, else the description's opening, cut at a
word. */
/* Built on the server (`app.models.excerpt`) and served as `role.excerpt`,
so the list response never carries the full description a row would
otherwise need to cut this from — see docs/decisions.md, "The pages". */
function excerpt(role) {
if (role.summary) return role.summary;
const text = (role.description || "").replace(/\s+/g, " ").trim();
if (text.length <= 160) return text;
const cut = text.slice(0, 160);
return role.excerpt || "";
}
/* The same cut, run here instead: for the one row `merge` refreshes after
a fetch or a summarise, off the full role that call answers with, which
is the only time this page ever sees a role's `summary` or
`description` directly. */
function excerptOf(role) {
const text = (role.summary || role.description || "").replace(/\s+/g, " ").trim();
if (text.length <= 200) return text;
const cut = text.slice(0, 200);
const space = cut.lastIndexOf(" ");
return `${(space > 100 ? cut.slice(0, space) : cut).replace(/[\s,.;:–-]+$/, "")}…`;
return `${(space > 120 ? cut.slice(0, space) : cut).replace(/[\s,.;:–-]+$/, "")}…`;
}
function tagChips(tags) {
@@ -584,21 +600,29 @@
sync($("#roles-list"), state.items.map((role, i) => [role.uid, { role, i }]), makeRow, fillRow);
const empty = $("#feed-empty");
empty.hidden = !(state.loaded && !state.items.length);
empty.hidden = !((state.loaded || state.feedError) && !state.items.length);
if (!empty.hidden) {
const f = state.filter;
const filtered = state.q || f.tags.size || f.period !== "all" || !f.showRead
|| f.company.trim() || f.place.trim() || f.function || f.remote || f.pay || f.judged;
empty.innerHTML = filtered
? `<h2>Nothing matches</h2>
<p>No open role fits these filters.
<button type="button" class="linky" data-clear>Clear them</button> and start again.</p>`
: `<h2>No roles yet</h2>
<p>Nothing has been pulled yet. <a href="/settings/sources#runs">Runs</a> starts one;
<a href="/settings/sources#job-boards">Job boards</a> and
<a href="/settings/sources#company-boards">Company boards</a> decide what it brings in.</p>`;
const clear = empty.querySelector("[data-clear]");
if (clear) clear.onclick = () => { state.q = ""; $("#search").value = ""; clearFilters(); };
if (state.feedError) {
empty.innerHTML = `<h2>Could not load the feed</h2>
<p>The server did not answer. <button type="button" class="linky" data-retry
>Try again</button>.</p>`;
const retry = empty.querySelector("[data-retry]");
if (retry) retry.onclick = () => loadFeed();
} else {
const f = state.filter;
const filtered = state.q || f.tags.size || f.period !== "all" || !f.showRead
|| f.company.trim() || f.place.trim() || f.function || f.remote || f.pay || f.judged;
empty.innerHTML = filtered
? `<h2>Nothing matches</h2>
<p>No open role fits these filters.
<button type="button" class="linky" data-clear>Clear them</button> and start again.</p>`
: `<h2>No roles yet</h2>
<p>Nothing has been pulled yet. <a href="/settings/sources#runs">Runs</a> starts one;
<a href="/settings/sources#job-boards">Job boards</a> and
<a href="/settings/sources#company-boards">Company boards</a> decide what it brings in.</p>`;
const clear = empty.querySelector("[data-clear]");
if (clear) clear.onclick = () => { state.q = ""; $("#search").value = ""; clearFilters(); };
}
}
// The end of the list says which end it is: a list that has run out and
@@ -755,9 +779,13 @@
}
if (!open) { el.replaceChildren(); return; }
// The list row is already in memory, and a list row is the whole role, so
// the pane paints from it at once and the fetch only brings it up to date.
// The list row is already in memory, and carries what a row draws —
// title, company, place, tags, pay — so the pane paints those at once;
// the body, the summary state and the original link wait on the fetch
// (`loaded`), since the list row no longer carries them (see
// `Store.roles_page`).
const stub = state.items.find((r) => r.uid === state.readingUid);
const loaded = Boolean(state.reading) && state.reading.uid === state.readingUid;
const r = state.reading || stub || {};
const same = el.dataset.uid === state.readingUid;
const scrolled = same ? el.querySelector(".reader-body")?.scrollTop || 0 : 0;
@@ -768,13 +796,13 @@
const text = (r.article_text || r.description || "").trim();
const body = error
? `<p class="reader-note">This role could not be loaded: ${esc(error)}</p>`
: !r.uid
: !loaded
? `<p class="reader-note">Loading…</p>`
: text ? paragraphs(text)
: `<p class="reader-note">The board gave no description for this one.</p>`;
const pay = payDisplay(r);
const waiting = state.summarising === state.readingUid;
const teaser = r.uid && !r.article_text && out && (r.description || "").length < TEASER;
const teaser = loaded && !r.article_text && out && (r.description || "").length < TEASER;
el.innerHTML = `
<div class="reader-bar">
@@ -801,7 +829,7 @@
${r.summary ? `<section class="reader-summary">
<h2 class="reader-label">Summary</h2><p>${esc(r.summary)}</p></section>` : ""}
<div class="reader-tools">
${r.uid && !r.summary && !waiting
${loaded && !r.summary && !waiting
? `<button type="button" class="secondary" data-tool="summarise">Summarise</button>` : ""}
${teaser ? `<button type="button" class="secondary" data-tool="fetch">Fetch the posting</button>` : ""}
<span class="muted small" role="status">${waiting
@@ -855,9 +883,9 @@
else state.reading = updated;
const item = state.items.find((x) => x.uid === updated.uid);
if (item) {
Object.assign(item, { summary: updated.summary, article_text: updated.article_text });
item.excerpt = excerptOf(updated);
const row = $(`#roles-list .row[data-uid="${CSS.escape(updated.uid)}"]`);
if (row) row.role = null; // drawn again from the new text
if (row) row.role = null; // drawn again from the new excerpt
renderFeed();
}
renderReader();
@@ -1116,19 +1144,41 @@
wire();
renderRail();
renderActive();
// The agent first: the companies, the feed and the counts are its.
try {
await Promise.all([loadTracked(), loadAgents()]);
chooseAgent();
await loadCompanies(state.filter.agent);
} catch (err) {
return;
}
renderRail();
renderActive();
loadFilters().catch(() => {});
// The feed is the page; agents and companies are enhancements that fill
// in when they arrive, not something it waits on — a slow or failed
// /api/agents or /api/companies must not leave it blank. Each one drawn
// here is greyed or missing until `enhance` lands, which is what "no
// agent" (renderAgent), an unjoined company (paintRow) and the tag row
// being empty already mean, so nothing here is a new state to handle.
const before = state.filter.agent;
const enhance = (async () => {
try {
await Promise.all([loadTracked(), loadAgents()]);
chooseAgent();
await loadCompanies(state.filter.agent);
} catch (err) {
return;
}
renderRail();
renderActive();
loadFilters().catch(() => {});
// The feed's first request went out with whatever agent `readUrl` had
// (often none): reload for the score and votes once the real one is
// known, and either way bring the reader's affinity control and judge
// buttons up to date, since `companies` was still empty when it first
// painted.
if (state.filter.agent !== before) reload();
else renderFeed();
if (state.readingUid) {
if (state.filter.agent !== before) refreshReader(state.readingUid);
else renderReader();
}
})();
route();
await loadFeed({ keep: was ? was.count : 0 });
if (was && !state.readingUid) window.scrollTo(0, was.y || 0);
await enhance;
});
})();
+26 -10
View File
@@ -14,7 +14,7 @@
(function () {
"use strict";
const { api, count, debounce } = window;
const { api, count, debounce, safeUrl } = window;
const $ = (sel, root = document) => root.querySelector(sel);
/** @type {{searches: object[], adzuna_key: boolean}} */
@@ -34,6 +34,14 @@
box.checked = source.on;
box.setAttribute("aria-label", `Run ${source.label}`);
box.addEventListener("change", async () => {
// Switching off closes every open role in one tap; ask first, naming
// the count, as removing a search and deleting an agent already do.
if (!box.checked && source.open_roles
&& !window.confirm(`Switch off ${source.label}? `
+ `Its ${count(source.open_roles, "open role")} close.`)) {
box.checked = true;
return;
}
box.disabled = true;
try {
const result = await api(`/api/sources/${source.id}/switch`,
@@ -41,6 +49,8 @@
Object.assign(source, result.source);
if (!box.checked && result.closed) {
window.toast.info(`${source.label}: ${count(result.closed, "open role")} closed.`);
} else if (box.checked) {
window.toast.info(`${source.label}: reading now, or with the pull right after.`);
}
} catch (err) {
box.checked = !box.checked;
@@ -158,12 +168,18 @@
const list = $("#search-sample");
for (const row of result.sample) {
const li = document.createElement("li");
const link = document.createElement("a");
link.href = row.url;
link.target = "_blank";
link.rel = "noopener";
link.textContent = row.title;
li.append(link, ` — ${[row.company, row.place].filter(Boolean).join(", ")}`);
const url = safeUrl(row.url);
if (url) {
const link = document.createElement("a");
link.href = url;
link.target = "_blank";
link.rel = "noopener";
link.textContent = row.title;
li.appendChild(link);
} else {
li.append(row.title);
}
li.append(` — ${[row.company, row.place].filter(Boolean).join(", ")}`);
list.appendChild(li);
}
} catch (err) {
@@ -184,7 +200,7 @@
renderSearches();
fillSearchForm(null);
status.textContent = `Saved ${result.source.label}. ${found(result)} `
+ "Its roles arrive with the next pull.";
+ "Reading now, or with the pull right after.";
} catch (err) {
status.textContent = "";
}
@@ -208,8 +224,8 @@
// Sources shows one section: the one the fragment names or contains, or the
// first when there is none or it names none, so `/settings/sources#runs` (a
// rail link, an old path's 301, the Search agents page's "follow it under
// Runs") opens Runs, and so does `#messages`, the log at its end. The rest
// rail link, or the Search agents page's "follow it under Runs") opens
// Runs, and so does `#messages`, the log at its end. The rest
// are hidden, not removed, so their scripts still find their roots. It runs
// as the script loads rather than on DOMContentLoaded, so the page never
// paints all three first.
+65 -2
View File
@@ -22,8 +22,8 @@ import numpy as np
from desk.models import Item
from app.models import (
AFFINITY, REMOTE, STATUSES, Agent, Application, ApplicationEvent, Candidate,
Company, Gate, Limits, Role, ScoredRole, Subscription, Weights, company_key,
country_of, place_key, posted_at, posted_host, remote_of, title_key,
Company, Gate, Limits, Role, RoleRow, ScoredRole, Subscription, Weights, company_key,
country_of, excerpt, place_key, posted_at, posted_host, remote_of, title_key,
)
from app.storage import schema
@@ -81,6 +81,17 @@ _VOTED_SCORE = ("MIN(1.0, s.score * CASE v.vote WHEN -1 THEN 0.5 "
#: nothing, so `vote` and `voted_at` are NULL: one query shape either way.
_WITH_VOTES = "LEFT JOIN role_votes v ON v.uid = roles.uid AND v.agent_id = ?"
#: What `roles_page` reads for `RoleRow.excerpt`: the summary when there is
#: one, else the description's opening. A `SUBSTR` of the source column, not
#: the column itself, so a big `description` never leaves SQLite whole for a
#: row that only ever shows `EXCERPT_LIMIT` characters of it.
_EXCERPT_SOURCE = "COALESCE(NULLIF(roles.summary, ''), roles.description)"
#: How many raw characters `roles_page` pulls before `app.models.excerpt`
#: cuts it to size: slack over `EXCERPT_LIMIT` so the word-boundary trim
#: still has somewhere to land.
_EXCERPT_RAW = 240
_COMPANY_FIELDS = ("name", "domain", "status", "careers_url", "notes", "ats", "country", "city",
"ats_slug")
@@ -539,6 +550,58 @@ class Store:
r.tags = by_uid.get(r.uid, [])
return found
def roles_page(self, limit: int = 100, company: str | None = None,
place: str | None = None, open_only: bool = True,
q: str | None = None, function: str | None = None,
remote: str | None = None, days: int | None = None,
hide_muted: bool = False, include_read: bool = True,
tags: Sequence[str] = (), tag_mode: str = "any",
agent: int | None = None, by_score: bool = True, gated: bool = False,
judged: str | None = None, pay: str | None = None,
offset: int = 0) -> list[RoleRow]:
"""What `GET /api/roles` serves: the same rows `roles` would, in the
same order, with the same filters and paging, but only the fields a
row draws (see `RoleRow`) — never `description`, `feed_text`, `raw`
or `article_text`, which a full `roles.*` select would carry whether
or not a page ever renders them. The reader still gets the whole role
by uid, see `role`.
`excerpt` is `app.models.excerpt` over a `SUBSTR` of `_EXCERPT_SOURCE`
(`_EXCERPT_RAW` characters, trimmed further in Python), so the source
column itself never crosses out of SQLite for a list."""
sql, params = self._roles_from(
company=company, place=place, open_only=open_only, q=q, function=function,
remote=remote, days=days, hide_muted=hide_muted, include_read=include_read,
tags=tags, tag_mode=tag_mode, agent=agent, by_score=by_score, gated=gated,
judged=judged, pay=pay)
select = (
"SELECT roles.uid, roles.company_key, roles.company, roles.title, roles.place, "
"roles.country, roles.remote, roles.function, roles.salary_min, "
"roles.salary_max, roles.currency, roles.salary_text, roles.original_url, "
"roles.posted_at, roles.read_at, "
"v.vote, v.voted_at, "
f"SUBSTR({_EXCERPT_SOURCE}, 1, {_EXCERPT_RAW}) AS excerpt")
order = "roles.posted_at DESC, roles.uid"
scored = agent is not None and by_score
if scored:
select += f", {_VOTED_SCORE} AS score, s.gated_reason"
order = "score DESC, " + order
elif open_only:
# Same reasoning as `roles`: the planner otherwise sorts every
# open role before applying LIMIT.
sql = sql.replace("FROM roles ", "FROM roles INDEXED BY idx_roles_posted ", 1)
rows = self._rows(f"{select} {sql} ORDER BY {order} LIMIT ? OFFSET ?",
[*params, limit, offset])
found = []
for r in rows:
row = dict(r)
row["excerpt"] = excerpt(row["excerpt"])
found.append(RoleRow.from_row(row))
by_uid = self._tags_by_uid([r.uid for r in found])
for r in found:
r.tags = by_uid.get(r.uid, [])
return found
def count_roles(self, company: str | None = None, place: str | None = None,
open_only: bool = True, q: str | None = None,
function: str | None = None, remote: str | None = None,
+2 -2
View File
@@ -7,8 +7,8 @@ from __future__ import annotations
from app.web.api import router as api_router
from app.web.pages import (
MOVED, SETTINGS, STATIC_DIR, TABS, router as pages_router, templates,
SETTINGS, STATIC_DIR, TABS, router as pages_router, templates,
)
__all__ = ["MOVED", "SETTINGS", "STATIC_DIR", "TABS", "api_router", "pages_router",
__all__ = ["SETTINGS", "STATIC_DIR", "TABS", "api_router", "pages_router",
"templates"]
+38 -13
View File
@@ -17,7 +17,7 @@ from pydantic import BaseModel, Field
from app import jobs
from app.models import (
AFFINITY, Agent, Application, Company, Gate, Limits, Role, ScoredRole, Status,
AFFINITY, Agent, Application, Company, Gate, Limits, Role, RoleRow, Status,
Subscription, Weights, company_key,
)
from app.storage import Store
@@ -112,9 +112,14 @@ def unread_role(uid: str, store: StoreDep) -> Role:
class RolesPage(BaseModel):
"""One page of roles, and how many the same filters match in all."""
"""One page of roles, and how many the same filters match in all. `items`
is `RoleRow`, not the full `Role`: the fields a feed row draws, with an
excerpt built on the server, never the big columns (`description`,
`feed_text`, `raw`, `article_text`) a full role carries — see
`Store.roles_page`. The reader fetches the rest by uid when it opens one,
`GET /api/roles/{uid}`."""
items: list[ScoredRole]
items: list[RoleRow]
total: int
@@ -165,7 +170,7 @@ def list_roles(
"hide_muted": not show_muted, "include_read": include_read, "tags": tags,
"tag_mode": tag_mode, "agent": agent_id, "by_score": by_score,
"gated": gated, "judged": judged or None, "pay": pay or None}
return RolesPage(items=store.roles(limit=limit, offset=offset, **filters),
return RolesPage(items=store.roles_page(limit=limit, offset=offset, **filters),
total=store.count_roles(**filters))
@@ -335,16 +340,21 @@ class CompanyBoardBody(BaseModel):
@router.post("/companies/{key}/board")
def set_company_board(key: str, body: CompanyBoardBody, store: StoreDep) -> dict[str, Any]:
def set_company_board(key: str, body: CompanyBoardBody, store: StoreDep,
queue: QueueDep) -> dict[str, Any]:
"""Attach a board to a company, or replace the one it has. The board is
fetched first and refused if it does not answer; replacing closes the
old board's open roles."""
old board's open roles. Queues a pull right after, so the board is read
by the one already running or the very next one rather than the
schedule, up to six hours later; a refusal because one is already
running is not an error, see `app.jobs.pull.claim_run`."""
if store.company(key) is None:
raise HTTPException(404, f"no such company: {key}")
try:
sub, found, closed = jobs.sources.set_board(store, key, body.board)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
queue.enqueue("pull")
return {"source": _one_source(store, sub.id), "found": found, "closed": closed}
@@ -492,12 +502,18 @@ class SwitchBody(BaseModel):
@router.post("/sources/{sub_id}/switch")
def switch_source(sub_id: int, body: SwitchBody, store: StoreDep) -> dict[str, Any]:
"""On or off for the next pull. Off closes the open roles at once; on
reopens the ones the next pull still finds."""
def switch_source(sub_id: int, body: SwitchBody, store: StoreDep,
queue: QueueDep) -> dict[str, Any]:
"""On or off. Off closes the open roles at once; on queues a pull right
after, so the ones it still lists reopen once the pull that is already
running reads it, or the very next one does, rather than waiting for the
schedule, up to six hours later. A refusal because one is already
running is not an error, see `app.jobs.pull.claim_run`."""
if store.subscription(sub_id) is None:
raise HTTPException(404, f"no such board or search: {sub_id}")
_, closed = jobs.sources.set_fetch(store, sub_id, body.on)
if body.on:
queue.enqueue("pull")
return {"source": _one_source(store, sub_id), "closed": closed}
@@ -534,14 +550,19 @@ def try_search(body: SearchBody, config: ConfigDep) -> dict[str, Any]:
@router.post("/searches")
def save_search(body: SearchBody, store: StoreDep, config: ConfigDep) -> dict[str, Any]:
def save_search(body: SearchBody, store: StoreDep, config: ConfigDep,
queue: QueueDep) -> dict[str, Any]:
"""Save a search, or update the one of that provider and name. It is run
once first and refused if it finds nothing."""
once first and refused if it finds nothing. Queues a pull right after,
so its roles are read by the one already running or the very next one
rather than the schedule, up to six hours later; a refusal because one is
already running is not an error, see `app.jobs.pull.claim_run`."""
try:
sub, found, complete = jobs.sources.save_search(config, store, body.provider,
body.name, _params(body))
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
queue.enqueue("pull")
return {"source": _one_source(store, sub.id), "found": found, "complete": complete}
@@ -560,14 +581,18 @@ class BoardBody(BaseModel):
@router.post("/boards", status_code=201)
def add_board(body: BoardBody, store: StoreDep) -> dict[str, Any]:
def add_board(body: BoardBody, store: StoreDep, queue: QueueDep) -> dict[str, Any]:
"""Follow a company by its ATS board: `set_board` keyed by the name, so
an existing company gets it attached or replaced. The board is fetched
first and refused if it does not answer."""
first and refused if it does not answer. Queues a pull right after, so
its roles are read by the one already running or the very next one
rather than the schedule, up to six hours later; a refusal because one is
already running is not an error, see `app.jobs.pull.claim_run`."""
try:
sub, found = jobs.sources.add_board(store, body.board, body.name)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
queue.enqueue("pull")
return {"source": _one_source(store, sub.id), "found": found}
-44
View File
@@ -23,13 +23,6 @@ contains; the others stay in the page, hidden, so each section's script (the
ones they had as pages of their own, each guarding on its own root element)
keeps working.
Every path a page has had before (`/agents`, `/companies`, `/sources`,
`/jobs`, `/schedules`, `/messages`, `/account`, and the six sub-tabs the
settings shell started with) answers 301 to its place now, query string
kept, so a bookmark still lands. A merged page's old path lands on its
section's anchor; the old Messages paths land on `#messages`, the log's
block at the end of Runs.
`FUNCTIONS` and `HOSTS` are `plugins.ats`'s, not `app`'s: the nine job
functions are the classifier's vocabulary and the hosts are what it parses,
and this reaches `plugins/` the same way `app.cli` does, since a plugin is
@@ -41,7 +34,6 @@ Jobindex areas and Adzuna countries on the Sources page come from
from __future__ import annotations
import sys
from collections.abc import Callable
from dataclasses import dataclass
from pathlib import Path
@@ -94,24 +86,6 @@ SECTIONS: dict[str, tuple[tuple[str, str], ...]] = {
),
}
#: Where each page that used to be a tab or a sub-tab lives now. The fragment
#: is the section on the merged page, or an element in one (`#messages`, in
#: Runs); a browser keeps it across the 301.
MOVED: dict[str, str] = {
"/agents": "/settings/agents",
"/settings/ranking": "/settings/agents",
"/companies": "/settings/sources#company-boards",
"/settings/companies": "/settings/sources#company-boards",
"/sources": "/settings/sources#job-boards",
"/settings/searches": "/settings/sources#job-boards",
"/jobs": "/settings/sources#runs",
"/schedules": "/settings/sources#runs",
"/settings/pulls": "/settings/sources#runs",
"/messages": "/settings/sources#messages",
"/settings/messages": "/settings/sources#messages",
"/account": "/settings/account",
}
#: `value, label` for each select option. `FUNCTIONS` is already the value a
#: role's `function` column holds; only the label is dressed up.
FUNCTION_OPTIONS = [(f, f.replace("/", " / ").capitalize()) for f in FUNCTIONS]
@@ -197,21 +171,3 @@ def sources(request: Request) -> HTMLResponse:
@router.get("/settings/account", response_class=HTMLResponse)
def account(request: Request) -> HTMLResponse:
return page(request, "settings/account.html")
def _moved(to: str) -> Callable[[Request], RedirectResponse]:
"""A 301 to `to`, carrying the query string, so `/agents?x=1` keeps it.
The query goes before `to`'s fragment, if it has one, or the browser
would read `#runs?x=1` as one fragment."""
path, _, fragment = to.partition("#")
def redirect(request: Request) -> RedirectResponse:
query = request.url.query
target = f"{path}?{query}" if query else path
return RedirectResponse(f"{target}#{fragment}" if fragment else target,
status_code=301)
return redirect
for _old, _new in MOVED.items():
router.add_api_route(_old, _moved(_new), methods=["GET"],
response_class=RedirectResponse)
+13 -3
View File
@@ -70,6 +70,8 @@ What was decided for roledesk and the one reason for each. The longer reasoning
**Switching a board or search off closes its open roles.** Only a pull that reads a target closes its roles, so a target no longer read would keep them open for good. Switched back on, the next pull reopens what it still lists. A batch already in flight when the switch happens reopens them again, so the pull's settle closes every switched-off target once more.
**A destructive switch asks first; a reopening one queues a pull instead of waiting for the schedule.** Switching a company's board or a job board search off, replacing a company's board (the Follow form and the row's Set board/Change editor, both `set_board`) and detaching one each close open roles in a tap; the page asks first when there are roles to lose, naming the count, a replace also naming the old and the new board, the same as removing a search and deleting an agent already do. Switching a target back on, following a board, setting a board and saving a search used to answer with nothing until the schedule's next pull, up to six hours later; each now queues `pull` right after, in the API route (`/api/sources/{id}/switch`, `/api/companies/{key}/board`, `/api/boards`, `/api/searches`) rather than in the page's script, so a script or `app job pull` calling the same endpoint gets what a click gets. `pull`'s own refusal while one is already running (`claim_run`) is not surfaced as an error: the change is read by the run already in flight, or by the very next one either way.
**The Sources tab and the CLI share `app.jobs.sources`.** Adding a board or a search and switching one are one function each, called by `app companies`, `app searches` and `/api/sources*`, so the two cannot disagree about what "off" does. A search or board is run once before it is saved, and refused if it does not answer. A search that finds nothing is refused too, since that is usually a typo in the query; a board that lists nothing is kept, since a company between openings is still one to follow.
**Every way a company gets a board is `app.jobs.sources.set_board`.** `app companies add|set-board|discover --add|seed`, `/api/boards` and `/api/companies/{key}/board` all call it, so attaching a board to an existing row, replacing one and creating a company are one code path; the seed importer moved out of `app/cli.py` for this. The company row and its subscription change in one transaction (`Store.set_company_board`), so a board another company already has leaves no company row behind. Its host and slug become the row's `ats` and `ats_slug`; other facts only fill empty columns, so a person's or the seed's values survive.
@@ -132,21 +134,27 @@ What was decided for roledesk and the one reason for each. The longer reasoning
**Each roledesk tab renders an empty shell; `static/roles.js` fills it and drives every action.** Roles, Applications, Agents and Companies extend the boilerplate's `base.html`; the server renders only select options (from `app.models`, `plugins.ats.FUNCTIONS` and `plugins.ats.HOSTS`, so they cannot drift from what the store and the plugin accept). One code path for "show this" beats a server-rendered first page plus a client re-render after every filter change.
**The feed loads whether or not its enhancements do.** `/api/agents` and `/api/companies` only dim hidden companies, draw the affinity control and pick the viewing agent's score order; the init block used to await both before ever calling `loadFeed`, so a slow or failed one left the feed blank for good behind a silent `catch (err) { return; }`. Now the feed's own request goes out alongside them rather than after, agents and companies fill in and redraw what they touch (the rail, the feed's dimming, the reader's affinity slot) whenever they land, and a feed already fetched with no agent known reloads once the real one is. A failed feed fetch of its own gets an empty state with a retry (`state.feedError`) instead of the same silent blank.
**`roles(company=, place=)` match a case-insensitive substring of what the board wrote.** The API's company and place fields are free text a person types, not a slug, so "Trade Republic" finds `trade-republic-bank` without knowing the key.
**A muted company is excluded in SQL, before `LIMIT`.** Filtering after `LIMIT` shipped first and showed nothing once Databricks' 884 freshly pulled roles filled the newest 200 rows and Databricks was muted.
**`GET /api/roles` serves a row, not a role.** It used to answer `roles.*`, the whole `Role` — `description`, `feed_text`, `raw` and `article_text` included, about 5.5 KB a role on average — for a feed that only ever shows a title, a few badges and an excerpt; a 1000-row restore was several megabytes of text a phone never renders. `Store.roles_page` selects only the fields a row draws (`RoleRow`), with `excerpt` built by `app.models.excerpt` off a `SUBSTR` of `summary` or `description`, never the column whole. Measured on a copy of the live database: a 60-row page went from 392 KiB to 39 KiB and a 1000-row restore from 6.6 MB to 631 KiB, both about a tenth. The reader is unaffected: it already fetched the whole role by uid when it opened one (`GET /api/roles/{uid}`, still the full `Role`); what changed is that the list row it painted from first, before that fetch lands, no longer carries a body to paint with, so the reader now waits on the fetch for the posting text and the original link the same way it always waited for the summary.
## The phone
**Two tabs and a cog.** `TABS` in `app/web/pages.py` becomes Roles and Applications; the other six pages (Agents, Companies, Sources, Jobs, Schedules, Messages, Account) move behind a settings cog at `/settings/<name>`, because nine tabs wrapped to three rows on a phone and six of them were setup or system pages. The old paths answer 301 to their new place, query string kept, so a bookmark or a `?agent=` link still lands.
**Two tabs and a cog.** `TABS` in `app/web/pages.py` becomes Roles and Applications; the other six pages (Agents, Companies, Sources, Jobs, Schedules, Messages, Account) move behind a settings cog at `/settings/<name>`, because nine tabs wrapped to three rows on a phone and six of them were setup or system pages.
**Settings are three sub-tabs: Search agents, Sources, Account.** Six sub-tabs were three setup pages and three system pages for two things, what the app ranks by and where roles come from, so each is one page now and Account stays its own. The old paths answer 301 to the section they became.
**Settings are three sub-tabs: Search agents, Sources, Account.** Six sub-tabs were three setup pages and three system pages for two things, what the app ranks by and where roles come from, so each is one page now and Account stays its own.
**The old paths' 301s are gone.** `MOVED` answered 301 for the twelve paths above, kept for the few days between the move and this cut, on the assumption that a bookmark should still land. A browser caches a 301 indefinitely, so a path a future page reused would be unreachable through no fault of that page; a one-person app with no outstanding bookmarks does not need to carry that risk.
**Sources is one page, one section per plugin.** Job boards are the `search` plugin's queries, company boards are `ats`'s, and the runs that read both end the page with the message log. A source's section says which plugin it is, and following a company, hiding it and switching its board off are one list. The UI says "job board" for what the API and CLI call a search, since "search agent" already means something else.
**Search agents are back in the UI as a list.** One person may run several: their own hunt, and one that monitors roles for someone else, each with its own intent and gates. The list has New and Delete; a delete takes the agent's votes, liked companies and scores with it.
**Settings pages sit beside a rail, and Sources shows one section at a time, chosen by the URL fragment.** The company boards list runs to 50 or 60 companies, so Runs at the end of a stacked page was out of reach and jump links still meant scrolling back up. The rail (`settings/_rail.html`, from `SETTINGS` and `SECTIONS` in `app/web/pages.py`) is sticky beside the content, and a strip under the header below 60rem. The fragment rather than a path per section, because the old paths already 301 to `/settings/sources#runs` and the like, and a hidden section stays in the DOM, so each section's script runs unchanged.
**Settings pages sit beside a rail, and Sources shows one section at a time, chosen by the URL fragment.** The company boards list runs to 50 or 60 companies, so Runs at the end of a stacked page was out of reach and jump links still meant scrolling back up. The rail (`settings/_rail.html`, from `SETTINGS` and `SECTIONS` in `app/web/pages.py`) is sticky beside the content, and a strip under the header below 60rem. The fragment rather than a path per section, because a hidden section stays in the DOM, so each section's script runs unchanged.
**Runs is the queue's own elements plus a Problems log.** `<desk-jobs>` and `<desk-schedules>` draw what is running and when the pull runs next, the app adds only "Run the pull now", disabled while a pull cascade is live, and the message log is the section's last block rather than a section of its own; `#messages` still lands on it. "Run one stage" went because the cascade is the unit of work: a stage run alone is a partial pull whose result the next stage never sees. `app job <stage>` and `POST /api/jobs` keep the per-stage entry points, which is what "Save and score" uses.
@@ -158,6 +166,8 @@ What was decided for roledesk and the one reason for each. The longer reasoning
**Roles are a feed with a reader, not links out.** `GET /roles/{uid}` renders the feed with its reader open on that role instead of sending the reader to the board, because 29,194 of 29,196 open board roles already have a `description` averaging 6,300 characters, and a search role's short teaser is worth reading in place before deciding whether to fetch the rest.
**Every link built from someone else's data goes through `safeUrl`.** A board, a search result, a discovered careers page and a saved application's `url` are all free text from outside the app; an `http(s)` check before it lands in an `href` is what keeps a stray `javascript:` link or a bare string typed into a field from being clickable. The reader needed this first, for `original_url`, so `safeUrl` moved to `app.js` beside the app's other shared helpers rather than staying roles.js's own; companies.js (`careers_url`), sources.js (a search sample's `url`) and applications.js (`application.url`) now use the same one.
**A vote moves the rank at read time and never hides.** `Store.roles` multiplies the served score by the vote (`0.5` passed, `1.2` liked, capped at 1) instead of triggering a rescore, so a vote's effect on Best match shows at once and needs no job. A vote never removes a role from the list either, because a stray tap must stay visible and a single further tap must be able to undo it.
**Read state is a column on `roles`, and a vote stamps it too.** `read_at` sits beside `vote`, the same shape as newsdesk's `items.read_at`, so the feed filters and counts by it directly rather than joining a side table on every page load. `set_vote` stamps it only when it is still NULL, since voting is reading in newsdesk as well; clearing a vote leaves it alone, because taking back an opinion is not un-reading the posting.
+177 -67
View File
@@ -17,8 +17,9 @@ from fastapi.testclient import TestClient
from app.config import Config, ServerConfig
from app.main import create_app
from app.models import excerpt as models_excerpt
from app.storage import Store
from app.web import MOVED, SETTINGS, TABS
from app.web import SETTINGS, TABS
from app.web.api import _score_after_change
from app.web.pages import SECTIONS
from auth import AuthConfig
@@ -78,16 +79,17 @@ def test_list_roles_filters_by_query_place_and_function(
store.upsert_role(posting("1", company="Acme", place="Berlin", function="engineering"))
store.upsert_role(posting("2", company="Acme", place="Munich", function="sales"))
store.upsert_role(posting("3", company="Other", place="Berlin", function="engineering"))
uid = {r.source_id: r.uid for r in store.roles()}
res = client.get("/api/roles", params={"q": "engineer", "place": "Berlin"})
assert res.status_code == 200
assert {r["source_id"] for r in res.json()["items"]} == {"1", "3"}
assert {r["uid"] for r in res.json()["items"]} == {uid["1"], uid["3"]}
res = client.get("/api/roles", params={"function": "sales"})
assert [r["source_id"] for r in res.json()["items"]] == ["2"]
assert [r["uid"] for r in res.json()["items"]] == [uid["2"]]
res = client.get("/api/roles", params={"company": "other"})
assert [r["source_id"] for r in res.json()["items"]] == ["3"]
assert [r["uid"] for r in res.json()["items"]] == [uid["3"]]
def test_list_roles_exposes_and_filters_by_pay(client: TestClient, store: Store) -> None:
@@ -98,19 +100,20 @@ def test_list_roles_exposes_and_filters_by_pay(client: TestClient, store: Store)
currency="EUR", salary_text="EUR 70,000 to 90,000 per year"))
store.upsert_role(posting("stated", salary_text="USD 45 to 60 per hour"))
store.upsert_role(posting("none"))
uid = {r.source_id: r.uid for r in store.roles()}
items = {r["source_id"]: r for r in client.get("/api/roles").json()["items"]}
assert items["normalised"]["salary_text"] == "EUR 70,000 to 90,000 per year"
assert items["normalised"]["salary_state"] == "normalised"
assert items["stated"]["salary_state"] == "stated"
assert items["none"]["salary_state"] == ""
items = {r["uid"]: r for r in client.get("/api/roles").json()["items"]}
assert items[uid["normalised"]]["salary_text"] == "EUR 70,000 to 90,000 per year"
assert items[uid["normalised"]]["salary_state"] == "normalised"
assert items[uid["stated"]]["salary_state"] == "stated"
assert items[uid["none"]]["salary_state"] == ""
res = client.get("/api/roles", params={"pay": "stated"})
assert {r["source_id"] for r in res.json()["items"]} == {"normalised", "stated"}
assert {r["uid"] for r in res.json()["items"]} == {uid["normalised"], uid["stated"]}
res = client.get("/api/roles", params={"pay": "normalised"})
assert [r["source_id"] for r in res.json()["items"]] == ["normalised"]
assert [r["uid"] for r in res.json()["items"]] == [uid["normalised"]]
res = client.get("/api/roles", params={"pay": "stated_only"})
assert [r["source_id"] for r in res.json()["items"]] == ["stated"]
assert [r["uid"] for r in res.json()["items"]] == [uid["stated"]]
assert client.get("/api/roles", params={"pay": "bogus"}).status_code == 422
@@ -128,20 +131,21 @@ def test_muted_roles_are_hidden_unless_show_muted(client: TestClient, store: Sto
store.upsert_role(posting("2", company="Other"))
agent = store.save_agent("A")
other = store.save_agent("B")
uid = {r.source_id: r.uid for r in store.roles()}
def shown(**params: Any) -> set[str]:
return {r["source_id"] for r in client.get("/api/roles", params=params).json()["items"]}
return {r["uid"] for r in client.get("/api/roles", params=params).json()["items"]}
assert shown() == {"1", "2"}
assert shown() == {uid["1"], uid["2"]}
# No agent in the body: the default agent's, which is also who a list
# without `agent` hides for.
assert client.post("/api/companies/acme/affinity", json={"affinity": -1}).status_code == 200
assert shown() == {"2"}
assert shown(show_muted="true") == {"1", "2"}
assert shown(agent=agent.id, order="newest") == {"2"}
assert shown() == {uid["2"]}
assert shown(show_muted="true") == {uid["1"], uid["2"]}
assert shown(agent=agent.id, order="newest") == {uid["2"]}
# Hidden is per agent: the other agent still sees Acme.
assert shown(agent=other.id, order="newest") == {"1", "2"}
assert shown(agent=other.id, order="newest") == {uid["1"], uid["2"]}
assert [c["affinity"] for c in client.get("/api/companies").json()] == [-1]
assert [c["affinity"] for c in
client.get("/api/companies", params={"agent": other.id}).json()] == [0]
@@ -162,9 +166,10 @@ def test_muted_is_excluded_before_the_limit_not_after(
for i in range(3):
store.upsert_role(posting(f"a{i}", company="Acme"))
clock.tick()
uid = {r.source_id: r.uid for r in store.roles()}
res = client.get("/api/roles", params={"limit": 1})
assert [r["source_id"] for r in res.json()["items"]] == ["other"]
assert [r["uid"] for r in res.json()["items"]] == [uid["other"]]
assert res.json()["total"] == 1
@@ -175,12 +180,13 @@ def test_list_roles_pages_by_offset_and_counts_the_total(
for i in range(5):
store.upsert_role(posting(str(i)))
clock.tick()
uid = {r.source_id: r.uid for r in store.roles()}
first = client.get("/api/roles", params={"limit": 2}).json()
assert [r["source_id"] for r in first["items"]] == ["4", "3"]
assert [r["uid"] for r in first["items"]] == [uid["4"], uid["3"]]
assert first["total"] == 5
rest = client.get("/api/roles", params={"limit": 2, "offset": 4}).json()
assert [r["source_id"] for r in rest["items"]] == ["0"] and rest["total"] == 5
assert [r["uid"] for r in rest["items"]] == [uid["0"]] and rest["total"] == 5
assert client.get("/api/roles", params={"offset": 9}).json() == {"items": [], "total": 5}
assert client.get("/api/roles", params={"offset": -1}).status_code == 422
assert len(client.get("/api/roles").json()["items"]) == 5 # default 60
@@ -202,13 +208,13 @@ def test_a_vote_is_set_cleared_and_filtered_on(client: TestClient, store: Store)
def judged(value: str, **params: Any) -> set[str]:
page = client.get("/api/roles", params={"judged": value, **params}).json()
return {r["source_id"] for r in page["items"]}
return {r["uid"] for r in page["items"]}
assert judged("liked") == {"liked"}
assert judged("passed") == {"passed"}
assert judged("unjudged") == {"open"}
assert judged("") == {"liked", "passed", "fine", "open"} # a pass is not hidden
assert judged("liked", agent=agent.id, order="newest") == {"liked"}
assert judged("liked") == {uid["liked"]}
assert judged("passed") == {uid["passed"]}
assert judged("unjudged") == {uid["open"]}
assert judged("") == {uid["liked"], uid["passed"], uid["fine"], uid["open"]} # a pass is not hidden
assert judged("liked", agent=agent.id, order="newest") == {uid["liked"]}
assert client.get("/api/roles", params={"judged": "maybe"}).status_code == 422
assert client.get(f"/api/roles/{uid['liked']}").json()["vote"] == 1
assert client.get(f"/api/roles/{uid['liked']}", params={"agent": agent.id}).json()["vote"] == 1
@@ -216,7 +222,7 @@ def test_a_vote_is_set_cleared_and_filtered_on(client: TestClient, store: Store)
cleared = client.delete(f"/api/roles/{uid['liked']}/vote")
assert cleared.status_code == 200
assert cleared.json()["vote"] is None and cleared.json()["voted_at"] is None
assert judged("unjudged") == {"open", "liked"}
assert judged("unjudged") == {uid["open"], uid["liked"]}
cleared = client.delete(f"/api/roles/{uid['passed']}/vote", params={"agent": agent.id})
assert cleared.status_code == 200 and cleared.json()["vote"] is None
@@ -280,12 +286,12 @@ def test_roles_with_an_agent_can_be_ordered_newest_and_keep_the_votes(
store.set_vote(agent.id, uid["1"], 1)
page = client.get("/api/roles", params={"agent": agent.id, "order": "newest"}).json()
assert [r["source_id"] for r in page["items"]] == ["3", "2", "1"]
assert [r["uid"] for r in page["items"]] == [uid["3"], uid["2"], uid["1"]]
assert [r["score"] for r in page["items"]] == [None, None, None]
assert [r["vote"] for r in page["items"]] == [None, None, 1]
assert page["total"] == 3
scored = client.get("/api/roles", params={"agent": agent.id, "order": "score"}).json()
assert [r["source_id"] for r in scored["items"]] == ["1", "2"]
assert [r["uid"] for r in scored["items"]] == [uid["1"], uid["2"]]
assert client.get("/api/roles", params={"agent": agent.id,
"order": "best"}).status_code == 422
@@ -295,13 +301,14 @@ def test_a_vote_moves_a_role_in_the_ranking_at_once(
for sid in ("a", "b", "c"):
store.upsert_role(posting(sid))
uid = {r.source_id: r.uid for r in store.roles()}
label = {v: k for k, v in uid.items()}
agent = store.save_agent("A")
store.replace_scores(agent.id, [(uid["a"], 0.9, ""), (uid["b"], 0.6, ""),
(uid["c"], 0.55, "")])
def ranked() -> list[tuple[str, float]]:
page = client.get("/api/roles", params={"agent": agent.id}).json()
return [(r["source_id"], round(r["score"], 3)) for r in page["items"]]
return [(label[r["uid"]], round(r["score"], 3)) for r in page["items"]]
assert ranked() == [("a", 0.9), ("b", 0.6), ("c", 0.55)]
client.post(f"/api/roles/{uid['a']}/vote", json={"value": -1})
@@ -331,14 +338,15 @@ def test_read_state_is_set_and_cleared_over_the_api(client: TestClient, store: S
def test_list_roles_filters_by_include_read(client: TestClient, store: Store) -> None:
store.upsert_role(posting("1"))
store.upsert_role(posting("2"))
ids = {r.source_id: r.uid for r in store.roles()}
uid = posting("1").uid
client.post(f"/api/roles/{uid}/read")
all_ = client.get("/api/roles").json()
assert {r["source_id"] for r in all_["items"]} == {"1", "2"}
assert {r["uid"] for r in all_["items"]} == {ids["1"], ids["2"]}
unread = client.get("/api/roles", params={"include_read": "false"}).json()
assert {r["source_id"] for r in unread["items"]} == {"2"}
assert {r["uid"] for r in unread["items"]} == {ids["2"]}
assert unread["total"] == 1
@@ -350,11 +358,11 @@ def test_list_roles_filters_by_tags_any_and_all(client: TestClient, store: Store
store.set_tags(uid["2"], ["python", "frontend"])
res = client.get("/api/roles", params={"tags": ["python"]})
assert {r["source_id"] for r in res.json()["items"]} == {"1", "2"}
assert {r["uid"] for r in res.json()["items"]} == {uid["1"], uid["2"]}
both = client.get("/api/roles", params={"tags": ["python", "backend"],
"tag_mode": "all"})
assert {r["source_id"] for r in both.json()["items"]} == {"1"}
assert {r["uid"] for r in both.json()["items"]} == {uid["1"]}
[item] = both.json()["items"]
assert item["tags"] == ["backend", "python"]
@@ -678,11 +686,10 @@ def test_every_settings_page_has_the_rail_and_not_the_tab_row(client: TestClient
< rail.index('href="/settings/account"')
def test_a_moved_section_is_one_the_sources_page_shows(client: TestClient) -> None:
"""Every old path that lands on a fragment of Sources names a section of
`SECTIONS` or an element inside one, and each section is a `<section
data-section>` on the page, which is what sources.js shows one of. The
old Messages paths land on `#messages`, the log at the end of Runs."""
def test_every_section_of_the_sources_page_exists(client: TestClient) -> None:
"""Each entry of `SECTIONS["/settings/sources"]` is a `<section
data-section>` on the page, which is what sources.js shows one of, and
the log at the end of Runs still answers `#messages`."""
anchors = [anchor for anchor, _ in SECTIONS["/settings/sources"]]
assert anchors == ["job-boards", "company-boards", "runs"]
page = client.get("/settings/sources").text
@@ -690,12 +697,7 @@ def test_a_moved_section_is_one_the_sources_page_shows(client: TestClient) -> No
for anchor in anchors:
assert f'<section class="card" id="{anchor}" data-section>' in page
runs = page[page.index('id="runs"'):]
for to in MOVED.values():
path, _, anchor = to.partition("#")
if anchor:
assert path == "/settings/sources", to
assert anchor in anchors or f'id="{anchor}"' in runs, to
assert MOVED["/messages"] == MOVED["/settings/messages"] == "/settings/sources#messages"
assert 'id="messages"' in runs
def test_settings_is_search_agents(client: TestClient) -> None:
@@ -703,21 +705,15 @@ def test_settings_is_search_agents(client: TestClient) -> None:
assert res.status_code == 302 and res.headers["location"] == "/settings/agents"
@pytest.mark.parametrize(("old", "new"), sorted(MOVED.items()))
def test_an_old_tab_moves_permanently_and_keeps_its_query(
client: TestClient, old: str, new: str) -> None:
"""A merged page's old path lands on its section: the query goes before
the anchor, or the browser would take `#runs?q=` as one fragment."""
assert {"/agents", "/settings/ranking", "/companies", "/sources", "/jobs",
"/schedules", "/messages", "/settings/searches", "/settings/companies",
"/settings/pulls", "/settings/messages", "/account"} <= set(MOVED)
path, _, anchor = new.partition("#")
res = client.get(old, follow_redirects=False)
assert res.status_code == 301 and res.headers["location"] == new
res = client.get(f"{old}?agent=3&q=a+b", follow_redirects=False)
assert res.status_code == 301
assert res.headers["location"] == f"{path}?agent=3&q=a+b" + (f"#{anchor}" if anchor else "")
assert client.get(old).status_code == 200
@pytest.mark.parametrize("old", ["/agents", "/settings/ranking", "/companies", "/sources",
"/jobs", "/schedules", "/messages", "/settings/searches",
"/settings/companies", "/settings/pulls",
"/settings/messages", "/account"])
def test_an_old_tab_path_is_gone(client: TestClient, old: str) -> None:
"""The paths the settings cog moved off of answer 404 now, not a cached
301: a browser caches a 301 indefinitely, so a path reused later would be
unreachable were the redirect still there."""
assert client.get(old).status_code == 404
def test_each_page_loads_only_its_own_script(client: TestClient) -> None:
@@ -946,7 +942,7 @@ def test_roles_by_agent_are_ordered_by_score_and_hide_the_gated(
(uids["gated"], 0.0, "place unknown")])
shown = client.get(f"/api/roles?agent={agent.id}").json()["items"]
assert [r["source_id"] for r in shown] == ["high", "low"]
assert [r["uid"] for r in shown] == [uids["high"], uids["low"]]
assert shown[0]["score"] == 0.8
everything = client.get(f"/api/roles?agent={agent.id}&gated=1").json()["items"]
assert everything[-1]["gated_reason"] == "place unknown"
@@ -954,6 +950,67 @@ def test_roles_by_agent_are_ordered_by_score_and_hide_the_gated(
assert client.get("/api/roles?agent=999").status_code == 404
def test_list_roles_serves_the_row_projection_and_not_the_big_columns(
client: TestClient, store: Store) -> None:
"""`GET /api/roles` is `RoleRow`: the fields a feed row draws, an
`excerpt` built on the server, and never `description`, `feed_text`,
`raw` or `article_text` — those are what the reader's own fetch,
`GET /api/roles/{uid}`, still answers in full."""
long_description = "Build things. " * 50
item = posting("1", company="Acme", place="Berlin, Germany", remote="hybrid",
function="engineering")
item.body = long_description
store.upsert_role(item)
uid = item.uid
row = client.get("/api/roles").json()["items"][0]
assert set(row) == {
"uid", "company_key", "company", "title", "place", "country", "remote", "function",
"salary_min", "salary_max", "currency", "salary_text", "salary_state",
"original_url", "posted_at", "score", "gated_reason", "vote",
"voted_at", "read_at", "tags", "excerpt",
}
assert row["uid"] == uid
assert row["company"] == "Acme" and row["place"] == "Berlin, Germany"
assert row["remote"] == "hybrid" and row["function"] == "engineering"
assert len(row["excerpt"]) <= 200
assert row["excerpt"] == models_excerpt(long_description)
full = client.get(f"/api/roles/{uid}").json()
assert full["description"] == long_description
def test_list_roles_page_stays_small_even_with_long_postings(
client: TestClient, store: Store) -> None:
"""The size bound the projection exists for: a page of rows with a real
posting's worth of text each (description, feed_text, raw and
article_text set, as a board's upsert would leave them) stays small,
because none of the four ever reach the response. Measured against a
copy of the live database in the report; this is the regression guard."""
big = "Requirement. " * 500 # ~6.5 KB, decisions.md's own average
for n in range(60):
item = posting(str(n), company="Acme")
item.body = big
store.upsert_role(item)
with store.transaction() as conn:
conn.execute("UPDATE roles SET feed_text = ?, raw = ?, article_text = ?",
(big, json.dumps({"blob": big}), big))
page = client.get("/api/roles", params={"limit": 60})
assert len(page.json()["items"]) == 60
# Well under one tenth of what 60 full roles (about 5.5 KB each,
# decisions.md's own figure) would answer with.
assert len(page.content) < 100_000
def test_list_roles_excerpt_prefers_the_summary(client: TestClient, store: Store) -> None:
store.upsert_role(posting("1"))
uid = posting("1").uid
store.set_summary(uid, "A short, hand-written summary.")
row = client.get("/api/roles").json()["items"][0]
assert row["excerpt"] == "A short, hand-written summary."
# ----------------------------------------------------------------- sources
def _fake_search(monkeypatch: pytest.MonkeyPatch, rows: int) -> None:
@@ -985,8 +1042,13 @@ def test_sources_lists_searches_and_boards_with_their_countries(
assert res["searches"][0]["params"] == {"q": "backend", "area": 1}
def test_switching_a_board_off_closes_its_roles_and_on_keeps_them_closed_until_a_pull(
client: TestClient, store: Store) -> None:
#: Every non-final state a job may be in, for a "was one queued at all" check
#: that does not race a background worker draining it before the assertion.
_ANY_STATE = ("queued", "running", "done", "failed", "cancelled")
def test_switching_a_board_off_closes_its_roles_and_on_queues_a_pull(
client: TestClient, store: Store, queue: Any) -> None:
board = store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
board="greenhouse")
store.upsert_role(posting("1", company="Acme"), board)
@@ -995,14 +1057,18 @@ def test_switching_a_board_off_closes_its_roles_and_on_keeps_them_closed_until_a
assert off["closed"] == 1 and off["source"]["on"] is False
assert store.subscriptions(source="ats", fetchable=True) == []
assert store.roles() == []
assert queue.jobs.last("pull", states=_ANY_STATE) is None # off reads nothing again
on = client.post(f"/api/sources/{board.id}/switch", json={"on": True}).json()
assert on["closed"] == 0 and on["source"]["on"] is True
# On queues a pull right after, so the reopened board is read by the one
# running or the next one rather than the schedule, up to six hours away.
assert queue.jobs.last("pull", states=_ANY_STATE) is not None
assert client.post("/api/sources/999/switch", json={"on": True}).status_code == 404
def test_a_search_is_tried_saved_updated_and_removed(
client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
client: TestClient, store: Store, queue: Any, monkeypatch: pytest.MonkeyPatch) -> None:
_fake_search(monkeypatch, 3)
body = {"provider": "jobindex", "name": "Copenhagen backend", "q": "backend", "area": 1}
@@ -1011,10 +1077,14 @@ def test_a_search_is_tried_saved_updated_and_removed(
assert tried["sample"][0] == {"title": "Engineer 0", "company": "Acme",
"place": "København", "url": "https://x.example/0"}
assert store.subscriptions(source="search") == []
assert queue.jobs.last("pull", states=_ANY_STATE) is None # a mere try reads nothing
saved = client.post("/api/searches", json=body).json()
assert saved["source"]["target"] == "jobindex:copenhagen-backend"
assert saved["source"]["params"] == {"q": "backend", "area": 1}
# Saving queues a pull right after, so its roles are read by the one
# running or the next one rather than the schedule, up to six hours away.
assert queue.jobs.last("pull", states=_ANY_STATE) is not None
updated = client.post("/api/searches", json={**body, "q": "python", "pages": 3}).json()
assert updated["source"]["id"] == saved["source"]["id"]
@@ -1040,7 +1110,7 @@ def test_a_search_that_finds_nothing_or_is_malformed_is_refused(
def test_a_board_is_followed_only_if_it_answers(
client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
client: TestClient, store: Store, queue: Any, monkeypatch: pytest.MonkeyPatch) -> None:
import plugins.ats as ats
monkeypatch.setattr(ats, "fetch_board", lambda target: ([{"source_id": "1"}], True))
@@ -1048,6 +1118,10 @@ def test_a_board_is_followed_only_if_it_answers(
assert res.status_code == 201
assert res.json()["found"] == 1 and res.json()["source"]["target"] == "ashby:acme"
assert store.company("acme") is not None
# Following a board queues a pull right after, so its roles are read by
# the one running or the next one rather than the schedule, up to six
# hours away.
assert queue.jobs.last("pull", states=_ANY_STATE) is not None
# Following a company that exists goes through `set_board`: the same
# board again is the same subscription, not a refusal or a second one.
@@ -1067,7 +1141,7 @@ def test_a_board_is_followed_only_if_it_answers(
def test_a_companys_board_is_set_replaced_and_detached(
client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
client: TestClient, store: Store, queue: Any, monkeypatch: pytest.MonkeyPatch) -> None:
import plugins.ats as ats
monkeypatch.setattr(ats, "fetch_board", lambda target: ([], True)) # between openings
@@ -1082,16 +1156,24 @@ def test_a_companys_board_is_set_replaced_and_detached(
[row] = client.get("/api/companies").json()
assert (row["board"], row["subscription_id"], row["ats"], row["ats_slug"]) == (
"greenhouse", sub_id, "greenhouse", "acme")
# Setting a board queues a pull right after, so it is read by the one
# running or the next one rather than the schedule, up to six hours away.
assert queue.jobs.last("pull", states=_ANY_STATE) is not None
res = client.post("/api/companies/acme/board", json={"board": "lever:acme"})
assert res.json()["closed"] == 1 and res.json()["source"]["id"] == sub_id
before_detach = queue.jobs.last("pull", states=_ANY_STATE)
res = client.delete("/api/companies/acme/board")
assert res.status_code == 200 and res.json() == {"closed": 0}
[row] = client.get("/api/companies").json()
assert (row["board"], row["subscription_id"], row["ats"]) == ("", None, "lever")
assert client.get("/api/sources").json()["boards"] == []
assert client.delete("/api/companies/acme/board").status_code == 409
# Detach reads nothing, so it queues no new pull.
after_detach = queue.jobs.last("pull", states=_ANY_STATE)
assert (before_detach.id if before_detach else None) == (
after_detach.id if after_detach else None)
assert client.post("/api/companies/nope/board",
json={"board": "lever:acme"}).status_code == 404
@@ -1100,6 +1182,34 @@ def test_a_companys_board_is_set_replaced_and_detached(
assert bad.status_code == 422 and "host:slug" in bad.json()["detail"]
def test_the_pull_a_source_switch_queues_is_refused_quietly_while_one_runs(
app: FastAPI, store: Store) -> None:
"""The four actions that queue a `pull` (switching a source on, following
or setting a board, saving a search) all go through the same job, whose
own refusal while another pull is in flight is not an error — see
`app.jobs.pull.claim_run` and `app.jobs.cascade.run_pull`. Exercised here
through the switch route as a stand-in for the other three, which queue
the identical `pull` job. No lifespan on this client, so no background
worker races the assertion: the queued job waits for the explicit
`drain`, and a refused pull starts no cascade — no fetch, no embed,
nothing else here needs mocking."""
from app.jobs import pull
client = TestClient(app)
board = store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
board="greenhouse")
running = pull.claim_run(store)
assert running
res = client.post(f"/api/sources/{board.id}/switch", json={"on": True})
assert res.status_code == 200
app.state.queue.drain(timeout=30)
job = app.state.queue.jobs.last("pull", states=("done",))
assert job is not None
assert job.result == {"skipped": pull.ALREADY_RUNNING, "running": running}
# ------------------------------------------------- roles: text and summary
def test_the_posting_text_is_fetched_once(