Sources tab: searches, company boards and the prune by region, in the app

What a pull reads can now be changed on a page instead of only the command
line: try a search and see its first ten results before saving it, edit or
remove one, follow a company's ATS board, switch any board or search on or
off, and preview and apply a prune by region. Each row shows its open roles
and the countries they are in.

The page and `app companies`/`app searches` call the same functions in
app/jobs/sources.py. Switching a target off closes its open roles, and the
pull's settle closes them again if a batch in flight reopened them. Roles
pulled before `country` existed are backfilled at startup.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-24 15:15:04 +02:00
1 parent 047f7cd8ad
commit 6b40702a9a
16 files changed
+1068 -92

No files matched your search

+34 -78
View File
@@ -83,7 +83,7 @@ def build_parser() -> argparse.ArgumentParser:
searches = sub.add_parser("searches", help="saved searches on cross-company job boards")
searches_sub = searches.add_subparsers(dest="action", required=True)
search_add = searches_sub.add_parser(
"add", help="run a search once, then save it for every pull")
"add", help="run a search once, then save it for every pull (or update it)")
search_add.add_argument("provider", choices=["jobindex", "adzuna"])
search_add.add_argument("name", help='what to call it, e.g. "Copenhagen backend"')
search_add.add_argument("--q", required=True,
@@ -97,7 +97,7 @@ def build_parser() -> argparse.ArgumentParser:
help="result pages per pull (default 5, at most 10)")
searches_sub.add_parser("list", help="every saved search and its open roles")
search_remove = searches_sub.add_parser(
"remove", help="stop running a search; its roles stay until they go stale")
"remove", help="stop running a search and close its open roles")
search_remove.add_argument("name")
db = sub.add_parser("db", help="database housekeeping")
@@ -275,34 +275,15 @@ def add_company(store: Store, board: str, name: str) -> int:
"""`app companies add greenhouse:acme "Acme"`: fetch the board first
and refuse if it does not answer, is not a known host, or the company
already exists. Port of seedgen's `-add`."""
from plugins.ats import careers_url as careers_url_of, fetch_board, is_known_target
from app import jobs
from app.models import company_key
if not is_known_target(board):
print(f"companies add: {board!r} is not host:slug for a known ATS host",
file=sys.stderr)
return 1
key = company_key(name)
if store.company(key) is not None:
print(f"companies add: {key!r} already exists", file=sys.stderr)
return 1
try:
rows = fetch_board(board)
except Exception as exc:
print(f"companies add: {board} did not answer: {exc}", file=sys.stderr)
return 1
if not rows:
print(f"companies add: {board} answered but lists no roles; nothing added",
file=sys.stderr)
return 1
host, _, _slug = board.partition(":")
store.add_company(name, key=key, careers_url=careers_url_of(board))
try:
store.add_subscription("ats", board, label=name, company=name, board=host)
sub, found = jobs.sources.add_board(store, board, name)
except ValueError as exc:
print(f"companies add: {exc}", file=sys.stderr)
return 1
print(json.dumps({"key": key, "board": board, "roles": len(rows)}))
print(json.dumps({"key": company_key(sub.company), "board": sub.target, "roles": found}))
return 0
@@ -325,38 +306,29 @@ def check_companies(store: Store) -> int:
def prune_companies(store: Store, region: str, *, apply: bool) -> int:
"""`app companies prune --in DK,EU`: every subscribed board none of whose
open roles is in the region, by `country`, which this fills from `place`
first where it is empty. `europe`, the default, is `EUROPE`. `--apply` unsubscribes those boards and closes their open roles,
since a board no longer pulled would otherwise keep them open for good.
`app companies seed` never brings one back: it skips a company that exists.
"""`app companies prune --in DK,EU`: every fetched board none of whose open
roles is in the region, which is `europe` unless named. `--apply`
switches them off, which closes their open roles. `app companies seed`
never brings one back: it skips a company that exists. See
`app.jobs.sources.prune_plan` for which boards count."""
from app import jobs
A board whose open roles all name no country is left alone: it says
nothing about where that company hires."""
from app.models import EUROPE
filled = store.backfill_country()
countries = store.open_countries_by_subscription()
wanted = {c.strip().upper() for c in region.split(",") if c.strip()}
if "EUROPE" in wanted:
wanted = (wanted - {"EUROPE"}) | set(EUROPE)
pruned = [(sub, sum(countries[sub.id].values()))
for sub in store.subscriptions(source="ats", fetchable=True)
if set(countries.get(sub.id, {})) - {""}
and not wanted.intersection(countries[sub.id])]
closed = 0
for sub, count in sorted(pruned, key=lambda p: -p[1]):
try:
filled, plan = jobs.sources.prune_plan(store, region)
except ValueError as exc:
print(f"companies prune: {exc}", file=sys.stderr)
return 1
for sub, count in plan:
print(f"{sub.target:40} {sub.label:30} {count:6} open, none in {region}")
if apply:
store.unsubscribe(sub.id)
closed += store.close_subscription(sub.id)
kept = len(store.subscriptions(source="ats", fetchable=True))
print(json.dumps({"countries_filled": filled, "pruned": len(pruned), "applied": apply,
"roles_closed": closed, "boards_left": kept}))
result: dict[str, Any] = {"countries_filled": filled, "pruned": len(plan), "applied": apply}
if apply:
result |= jobs.sources.prune(store, region)
print(json.dumps(result))
return 0
def run_searches(args: argparse.Namespace) -> int:
from app import jobs
from app.storage import Store
config = config_mod.load(args.config)
@@ -368,12 +340,12 @@ def run_searches(args: argparse.Namespace) -> int:
if sub is None:
print(f"searches remove: no search named {args.name!r}", file=sys.stderr)
return 1
store.unsubscribe(sub.id)
print(json.dumps({"removed": sub.label}))
_, closed = jobs.sources.remove_search(store, sub.id)
print(json.dumps({"removed": sub.label, "roles_closed": closed}))
return 0
countries = store.open_countries_by_subscription()
for sub in store.subscriptions(source="search"):
removed = "" if sub.subscribed and sub.fetch else " (removed)"
removed = "" if sub.subscribed and sub.fetch else " (off)"
open_roles = sum(countries.get(sub.id, {}).values())
print(f"{sub.label:30} {sub.target:40} {open_roles:5} open "
f"{json.dumps(sub.params, ensure_ascii=False)}{removed}")
@@ -382,36 +354,20 @@ def run_searches(args: argparse.Namespace) -> int:
def add_search(config: config_mod.Config, store: Store, args: argparse.Namespace) -> int:
"""`app searches add jobindex "Copenhagen backend" --q '"backend developer"'
--area 1`: run the search once and refuse if it fails or finds nothing, so
a typo in a query is found now rather than by an empty tab later."""
from app.jobs import search_target
from plugins.search import run_search
--area 1`: run the search once and save it only if it finds something.
A search of the same name is updated in place."""
from app import jobs
params: dict[str, Any] = {"q": args.q}
if args.provider == "jobindex" and args.area:
params["area"] = args.area
if args.provider == "adzuna":
params["country"] = args.country.lower()
if args.where:
params["where"] = args.where
if args.pages:
params["pages"] = args.pages
target = search_target(args.provider, args.name)
try:
rows, complete = run_search(target, params, dict(config.sources.get("search") or {}))
except Exception as exc:
print(f"searches add: {args.provider} did not answer: {exc}", file=sys.stderr)
return 1
if not rows:
print("searches add: the search found nothing; not saved", file=sys.stderr)
return 1
try:
store.add_subscription("search", target, label=args.name, board=args.provider,
params=params)
params = jobs.sources.search_params(args.provider, args.q, area=args.area,
country=args.country, where=args.where,
pages=args.pages)
sub, found, complete = jobs.sources.save_search(config, store, args.provider, args.name,
params)
except ValueError as exc:
print(f"searches add: {exc}", file=sys.stderr)
return 1
print(json.dumps({"search": args.name, "target": target, "roles": len(rows),
print(json.dumps({"search": sub.label, "target": sub.target, "roles": found,
"complete": complete}, ensure_ascii=False))
return 0
+4 -2
View File
@@ -7,13 +7,15 @@ task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`.
score.py every open role scored for every enabled search agent
cascade.py pull → extract → embed → score, what `TASKS` offers, the
`pull` job a schedule fires, and that schedule
sources.py what a pull reads, changed by hand: boards and searches
added, switched on or off, pruned by region
"""
from __future__ import annotations
from typing import Any
from app.jobs import cascade, embed, extract, pull, score
from app.jobs import cascade, embed, extract, pull, score, sources
from app.jobs.pull import search_target, seed_searches
from app.jobs.score import seed_agents
@@ -33,4 +35,4 @@ def bootstrap(config_path: str = "") -> tuple[Any, Any]:
__all__ = ["BOOTSTRAP", "bootstrap", "cascade", "embed", "extract", "pull", "score",
"search_target", "seed_agents", "seed_searches"]
"search_target", "seed_agents", "seed_searches", "sources"]
+11 -1
View File
@@ -88,7 +88,10 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
from plugins.ats import ATSSource
from plugins.search import SearchSource
subs = {s.target: s for s in store.subscriptions(source=source) if s.target in targets}
# Fetchable now, not when the pull was queued: a board switched off on the
# Sources tab since then is skipped.
subs = {s.target: s for s in store.subscriptions(source=source, fetchable=True)
if s.target in targets}
options = dict(config.sources.get(source) or {})
plugin: ATSSource | SearchSource
if source == "search":
@@ -124,9 +127,16 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
`ats:ok:<target>` was not written during *this* run — it failed, or its
whole batch job never got to it — is never closed. A search closes by its
list only when that list was complete, and otherwise by `STALE_DAYS`.
A board or search switched off while its batch was in flight had its
roles reopened by that batch, after the switch closed them, so every
switched-off one is closed again here.
"""
answered = searched = failed = closed = 0
for source in SOURCES:
for sub in store.subscriptions(source=source):
if not (sub.subscribed and sub.fetch):
closed += store.close_subscription(sub.id)
run_marker = store.state.get(f"{source}:run", "")
for sub in store.subscriptions(source=source, fetchable=True):
stamp = store.state.get(f"{source}:ok:{sub.target}", "")
+171
View File
@@ -0,0 +1,171 @@
"""What a pull reads, changed by hand: add a board or a search, switch one on
or off, prune boards by region. `app companies`, `app searches` and the
Sources tab all call these, so the command line and the page cannot disagree
about what "off" means.
A board or search switched off has its open roles closed at once. Nothing
else would ever close them, since only a pull that reads a target closes its
roles; switched back on, the next pull reopens every role it still lists.
"""
from __future__ import annotations
import os
from typing import Any
from app.config import Config
from app.models import EUROPE, Subscription, company_key
from app.storage import Store
from app.jobs.pull import search_target
#: The providers `plugins.search` runs, and which of them needs a key.
PROVIDERS = ("jobindex", "adzuna")
def region_codes(region: str) -> set[str]:
"""`europe`, or comma-separated ISO codes such as `DK,SE,EU`, as a set of
codes. `europe` may be one entry among codes: `europe,US`."""
codes = {c.strip().upper() for c in (region or "").split(",") if c.strip()}
if "EUROPE" in codes:
codes = (codes - {"EUROPE"}) | set(EUROPE)
if not codes:
raise ValueError("name a region: europe, or country codes such as DK,SE,EU")
return codes
def prune_plan(store: Store, region: str) -> tuple[int, list[tuple[Subscription, int]]]:
"""Every fetched board none of whose open roles is in the region, with its
open role count, largest first; and how many roles had `country` filled
from `place` on the way. A board whose roles all name no country is kept:
it says nothing about where that company hires."""
wanted = region_codes(region)
filled = store.backfill_country()
countries = store.open_countries_by_subscription()
plan = [(sub, sum(countries[sub.id].values()))
for sub in store.subscriptions(source="ats", fetchable=True)
if set(countries.get(sub.id, {})) - {""}
and not wanted.intersection(countries[sub.id])]
return filled, sorted(plan, key=lambda p: -p[1])
def prune(store: Store, region: str) -> dict[str, Any]:
"""Switch off every board `prune_plan` names."""
filled, plan = prune_plan(store, region)
closed = sum(set_fetch(store, sub.id, False)[1] for sub, _ in plan)
return {"countries_filled": filled, "pruned": len(plan), "roles_closed": closed,
"boards_left": len(store.subscriptions(source="ats", fetchable=True))}
def set_fetch(store: Store, sub_id: int, fetch: bool) -> tuple[Subscription, int]:
"""Switch one board or search on or off. Off closes its open roles and
returns how many; see the module docstring."""
sub = store.update_subscription(sub_id, subscribed=fetch, fetch=fetch, show=True)
closed = 0 if fetch else store.close_subscription(sub_id)
return sub, closed
def add_board(store: Store, board: str, name: str) -> tuple[Subscription, int]:
"""Follow one company's ATS board: fetch it first, and refuse if the host
is unknown, the company exists, the board does not answer or lists
nothing. Returns the subscription and how many roles the board lists."""
from plugins.ats import careers_url, fetch_board, is_known_target
board, name = board.strip(), name.strip()
if not name:
raise ValueError("a company needs a name")
if not is_known_target(board):
raise ValueError(f"{board!r} is not host:slug for a known ATS host, "
"e.g. greenhouse:acme or ashby:acme")
key = company_key(name)
if store.company(key) is not None:
raise ValueError(f"{name} is already followed")
try:
rows = fetch_board(board)
except Exception as exc:
raise ValueError(f"{board} did not answer: {exc}") from exc
if not rows:
raise ValueError(f"{board} answered but lists no roles; nothing added")
host = board.partition(":")[0]
store.add_company(name, key=key, careers_url=careers_url(board))
sub = store.add_subscription("ats", board, label=name, company=name, board=host)
return sub, len(rows)
def search_params(provider: str, q: str, *, area: int = 0, country: str = "",
where: str = "", pages: int = 0) -> dict[str, Any]:
"""The `params` a search subscription stores, with only the keys its
provider reads."""
if provider not in PROVIDERS:
raise ValueError(f"provider is one of {', '.join(PROVIDERS)}")
if not q.strip():
raise ValueError("a search needs a query")
params: dict[str, Any] = {"q": q.strip()}
if provider == "jobindex" and area:
params["area"] = int(area)
if provider == "adzuna":
params["country"] = country.strip().lower()
if where.strip():
params["where"] = where.strip()
if pages:
params["pages"] = int(pages)
return params
def has_key(config: Config, provider: str) -> bool:
"""Whether `provider` can run: jobindex always, adzuna with a key in the
config or the environment."""
if provider != "adzuna":
return True
keys = (config.sources.get("search") or {}).get("adzuna") or {}
return bool((keys.get("app_id") or os.environ.get("ADZUNA_APP_ID"))
and (keys.get("app_key") or os.environ.get("ADZUNA_APP_KEY")))
def try_search(config: Config, provider: str, name: str,
params: dict[str, Any]) -> tuple[list[dict[str, Any]], bool]:
"""Run a search once, outside any pull. Raises ValueError with the
provider's own reason when it does not answer."""
from plugins.search import run_search
try:
return run_search(search_target(provider, name or "preview"), params,
dict(config.sources.get("search") or {}))
except ValueError:
raise
except Exception as exc:
raise ValueError(f"{provider} did not answer: {exc}") from exc
def save_search(config: Config, store: Store, provider: str, name: str,
params: dict[str, Any]) -> tuple[Subscription, int, bool]:
"""Run a search once and save it only if it finds something, so a typo in a
query shows now and not as an empty tab later. A search of the same
provider and name is updated in place, removed or not: its roles keep
their subscription. Returns the subscription, how many roles it found, and
whether that was every match."""
name = name.strip()
if not name:
raise ValueError("a search needs a name")
rows, complete = try_search(config, provider, name, params)
if not rows:
raise ValueError("the search found nothing; not saved")
target = search_target(provider, name)
existing = next((s for s in store.subscriptions(source="search") if s.target == target),
None)
if existing is None:
sub = store.add_subscription("search", target, label=name, board=provider,
params=params)
else:
sub = store.update_subscription(existing.id, label=name, params=params,
subscribed=True, fetch=True, show=True)
return sub, len(rows), complete
def remove_search(store: Store, sub_id: int) -> tuple[Subscription, int]:
"""Stop running a search and hide it. Its open roles close; adding a search
of the same name again brings the row back."""
sub = store.subscription(sub_id)
if sub is None or sub.source != "search":
raise KeyError(sub_id)
store.unsubscribe(sub_id)
return sub, store.close_subscription(sub_id)
+3
View File
@@ -46,6 +46,9 @@ def create_app(config: Config | None = None) -> FastAPI:
store = Store(config.database)
jobs.seed_agents(config, store)
jobs.seed_searches(config, store)
# Roles pulled before `country` was read from `place` have none; after the
# first start only the places the lookup cannot place are left to read.
store.backfill_country()
app = FastAPI(title="roledesk", docs_url="/api/docs",
summary="ATS boards and job searches in, roles scored per search agent, "
+20
View File
@@ -352,6 +352,26 @@ textarea:focus-visible {
.weights label { flex: 1 1 6rem; }
.agent-actions { display: flex; gap: .5rem; flex-wrap: wrap; }
/* sources */
.section { margin: 1.5rem 0 2rem; }
.section > h2 { margin-bottom: .25rem; }
.source-form { margin-top: 1rem; }
.form-row { display: flex; gap: .75rem; flex-wrap: wrap; }
.form-row label { flex: 0 1 auto; }
.form-row label.grow { flex: 1 1 14rem; }
.source-form select {
background: var(--bg); border: 1px solid var(--line);
border-radius: .3rem; padding: .35rem .5rem;
}
.source-form input[type="number"] { max-width: 6rem; }
.source-form [hidden] { display: none; }
.sample { margin: 0; padding-left: 1.25rem; font-size: .9rem; }
.sample li { margin: .15rem 0; }
.mono { font-family: var(--mono); overflow-wrap: anywhere; }
.row-actions { display: inline-flex; gap: .35rem; }
.row-actions button { padding: .25rem .6rem; }
/* applications and companies */
table.list { width: 100%; border-collapse: collapse; background: var(--panel); }
+366
View File
@@ -0,0 +1,366 @@
/* The Sources tab: saved searches, company boards, and the prune by region.
*
* The template is a shell; the lists come from /api/sources and every action
* is a fetch through `api()` from app.js, so a refusal (a search that finds
* nothing, a board that does not answer) is one toast with the server's own
* sentence. The catch blocks here only put a control back.
*/
(function () {
"use strict";
const api = window.api;
const $ = (sel, root = document) => root.querySelector(sel);
/** @type {{searches: object[], boards: object[], adzuna_key: boolean}} */
let data = { searches: [], boards: [], adzuna_key: false };
/** @type {object[]} the boards the last prune preview named */
let pruneBoards = [];
function count(n, one, many) {
return `${n} ${n === 1 ? one : many || `${one}s`}`;
}
function debounce(fn, ms) {
let timer;
return (...args) => {
clearTimeout(timer);
timer = setTimeout(() => fn(...args), ms);
};
}
function cell(tr, label, content) {
const td = document.createElement("td");
td.dataset.label = label;
if (typeof content === "string") td.textContent = content;
else td.appendChild(content);
tr.appendChild(td);
return td;
}
/** "DK 120 · EU 4 · unknown 2", the three biggest. */
function countries(source) {
const shown = source.countries.slice(0, 3)
.map(([code, n]) => `${code || "unknown"} ${n}`);
if (source.countries.length > 3) shown.push("…");
return shown.join(" · ");
}
function query(source) {
const p = source.params || {};
const where = source.provider === "adzuna"
? [p.country && p.country.toUpperCase(), p.where].filter(Boolean).join(", ")
: (p.area ? $(`#s-area option[value="${p.area}"]`)?.textContent : "All of Denmark");
return `${p.q || ""}${where ? ` — ${where}` : ""}`;
}
function onSwitch(source, rerender) {
const box = document.createElement("input");
box.type = "checkbox";
box.checked = source.on;
box.setAttribute("aria-label", `Pull ${source.label}`);
box.addEventListener("change", async () => {
box.disabled = true;
try {
const result = await api(`/api/sources/${source.id}/switch`,
{ method: "POST", body: { on: box.checked } });
Object.assign(source, result.source);
if (!box.checked && result.closed) {
window.toast.info(`${source.label}: ${count(result.closed, "open role")} closed.`);
}
} catch (err) {
box.checked = !box.checked;
}
box.disabled = false;
rerender();
});
return box;
}
async function load() {
data = await api("/api/sources");
}
// ---------------------------------------------------------------- searches
function renderSearches() {
const body = $("#searches-list");
body.replaceChildren();
for (const source of data.searches) {
const tr = document.createElement("tr");
if (!source.on) tr.classList.add("muted");
const name = document.createElement("span");
name.textContent = source.label;
const provider = document.createElement("span");
provider.className = "badge";
provider.textContent = source.provider;
const title = document.createElement("span");
title.append(name, " ", provider);
cell(tr, "Search", title);
cell(tr, "Query", query(source)).className = "mono small";
cell(tr, "Open roles", String(source.open_roles)).className = "num";
cell(tr, "Countries", countries(source)).className = "small";
cell(tr, "On", onSwitch(source, renderSearches));
const actions = document.createElement("span");
actions.className = "row-actions";
const edit = document.createElement("button");
edit.type = "button";
edit.className = "secondary";
edit.textContent = "Edit";
edit.addEventListener("click", () => fillSearchForm(source));
const remove = document.createElement("button");
remove.type = "button";
remove.className = "secondary";
remove.textContent = "Remove";
remove.addEventListener("click", () => removeSearch(source));
actions.append(edit, remove);
cell(tr, "", actions);
body.appendChild(tr);
}
const open = data.searches.reduce((sum, s) => sum + s.open_roles, 0);
$("#searches-count").textContent = data.searches.length
? `${count(data.searches.length, "search", "searches")}, ${count(open, "open role")}`
: "No searches yet. Add one below; the starter searches in config.yaml import on the first pull.";
}
function showProvider() {
const provider = $("#s-provider").value;
for (const node of document.querySelectorAll("#search-form [data-provider]")) {
node.hidden = node.dataset.provider !== provider;
}
$("#adzuna-key").hidden = provider !== "adzuna" || data.adzuna_key;
}
function fillSearchForm(source) {
const p = (source && source.params) || {};
$("#search-form-title").textContent = source ? `Edit ${source.label}` : "New search";
$("#s-provider").value = source ? source.provider : "jobindex";
$("#s-name").value = source ? source.label : "";
$("#s-name").readOnly = Boolean(source);
$("#s-provider").disabled = Boolean(source);
$("#s-q").value = p.q || "";
$("#s-area").value = String(p.area || (source ? 0 : 1));
$("#s-country").value = p.country || "de";
$("#s-where").value = p.where || "";
$("#s-pages").value = String(p.pages || 10);
$("#search-status").textContent = "";
$("#search-sample").replaceChildren();
showProvider();
if (source) $("#search-form").scrollIntoView({ behavior: "smooth", block: "start" });
}
function searchBody() {
return {
provider: $("#s-provider").value,
name: $("#s-name").value.trim(),
q: $("#s-q").value.trim(),
area: Number($("#s-area").value) || 0,
country: $("#s-country").value,
where: $("#s-where").value.trim(),
pages: Number($("#s-pages").value) || 0,
};
}
function found(result) {
const all = result.complete ? "every match" : "the page limit cut it short";
return `Found ${count(result.found, "role")} (${all}).`;
}
async function trySearch() {
const status = $("#search-status");
const button = $("#s-try");
button.disabled = true;
status.textContent = "Searching…";
$("#search-sample").replaceChildren();
try {
const result = await api("/api/searches/try", { method: "POST", body: searchBody() });
status.textContent = `${found(result)} The newest ${Math.min(result.found, 10)}:`;
const list = $("#search-sample");
for (const row of result.sample) {
const li = document.createElement("li");
const link = document.createElement("a");
link.href = row.url;
link.target = "_blank";
link.rel = "noopener";
link.textContent = row.title;
li.append(link, ` — ${[row.company, row.place].filter(Boolean).join(", ")}`);
list.appendChild(li);
}
} catch (err) {
status.textContent = "";
}
button.disabled = false;
}
async function saveSearch(event) {
event.preventDefault();
const status = $("#search-status");
const button = $("#search-form button[type=submit]");
button.disabled = true;
status.textContent = "Running it once before saving…";
try {
const result = await api("/api/searches", { method: "POST", body: searchBody() });
await load();
renderSearches();
fillSearchForm(null);
status.textContent = `Saved ${result.source.label}. ${found(result)} `
+ "Its roles arrive with the next pull.";
} catch (err) {
status.textContent = "";
}
button.disabled = false;
}
async function removeSearch(source) {
if (!window.confirm(`Remove ${source.label}? Its ${count(source.open_roles, "open role")} close.`)) {
return;
}
try {
await api(`/api/searches/${source.id}`, { method: "DELETE" });
data.searches = data.searches.filter((s) => s.id !== source.id);
renderSearches();
} catch (err) {
/* already toasted */
}
}
// ------------------------------------------------------------------ boards
function renderBoards() {
const q = $("#b-q").value.trim().toLowerCase();
const on = $("#b-on").value;
const rows = data.boards
.filter((b) => !q || b.label.toLowerCase().includes(q) || b.target.includes(q))
.filter((b) => !on || (on === "on") === b.on)
.sort((a, b) => Number(b.on) - Number(a.on) || b.open_roles - a.open_roles
|| a.label.localeCompare(b.label));
const body = $("#boards-list");
body.replaceChildren();
for (const board of rows) {
const tr = document.createElement("tr");
if (!board.on) tr.classList.add("muted");
const link = document.createElement("a");
link.href = `/?company=${encodeURIComponent(board.label)}`;
link.title = "This company's open roles";
link.textContent = board.label;
cell(tr, "Company", link);
cell(tr, "Board", board.target).className = "mono small";
cell(tr, "Open roles", String(board.open_roles)).className = "num";
cell(tr, "Countries", countries(board)).className = "small";
cell(tr, "On", onSwitch(board, renderBoards));
body.appendChild(tr);
}
const onCount = data.boards.filter((b) => b.on).length;
$("#boards-count").textContent =
`${count(data.boards.length, "board")}, ${onCount} on. Showing ${rows.length}.`;
}
async function addBoard(event) {
event.preventDefault();
const status = $("#board-status");
const button = $("#board-form button[type=submit]");
button.disabled = true;
status.textContent = "Reading the board once before following it…";
try {
const result = await api("/api/boards", {
method: "POST",
body: { board: $("#b-board").value.trim(), name: $("#b-name").value.trim() },
});
data.boards.push(result.source);
renderBoards();
$("#board-form").reset();
status.textContent = `Following ${result.source.label}: `
+ `${count(result.found, "role")} listed. They arrive with the next pull.`;
} catch (err) {
status.textContent = "";
}
button.disabled = false;
}
// ------------------------------------------------------------------- prune
async function previewPrune(event) {
event.preventDefault();
const status = $("#prune-status");
const list = $("#prune-list");
const region = $("#p-region").value.trim();
list.replaceChildren();
$("#p-apply").disabled = true;
status.textContent = "Checking every board…";
try {
const result = await api(`/api/prune?region=${encodeURIComponent(region)}`);
pruneBoards = result.boards;
const roles = pruneBoards.reduce((sum, b) => sum + b.open_roles, 0);
status.textContent = pruneBoards.length
? `${count(pruneBoards.length, "board")} with no open role in ${region}, `
+ `${count(roles, "open role")} between them:`
: `Every board that is on has a role in ${region}.`;
for (const board of pruneBoards) {
const li = document.createElement("li");
li.textContent = `${board.label} (${board.target}), ${count(board.open_roles, "open role")}`;
list.appendChild(li);
}
$("#p-apply").disabled = pruneBoards.length === 0;
} catch (err) {
status.textContent = "";
}
}
async function applyPrune() {
const region = $("#p-region").value.trim();
if (!window.confirm(`Switch off ${count(pruneBoards.length, "board")} and close their roles? `
+ "Each can be switched back on in the list above.")) return;
$("#p-apply").disabled = true;
try {
const result = await api("/api/prune", { method: "POST", body: { region } });
$("#prune-status").textContent = `Switched off ${count(result.pruned, "board")}, `
+ `closed ${count(result.roles_closed, "role")}. ${result.boards_left} boards are on.`;
$("#prune-list").replaceChildren();
await load();
renderBoards();
} catch (err) {
$("#p-apply").disabled = false;
}
}
// -------------------------------------------------------------------- pull
async function pullNow() {
const button = $("#pull-now");
button.disabled = true;
try {
await api("/api/jobs", { method: "POST", body: { kind: "pull", payload: {} } });
$("#pull-status").replaceChildren("Pull started. ");
const link = document.createElement("a");
link.href = "/jobs";
link.textContent = "Follow it on the Jobs tab";
$("#pull-status").appendChild(link);
} catch (err) {
/* already toasted */
}
button.disabled = false;
}
// -------------------------------------------------------------------- init
document.addEventListener("DOMContentLoaded", async () => {
if (!$("#searches-list")) return;
try {
await load();
} catch (err) {
return;
}
renderSearches();
renderBoards();
fillSearchForm(null);
$("#s-provider").addEventListener("change", showProvider);
$("#s-try").addEventListener("click", trySearch);
$("#search-form").addEventListener("submit", saveSearch);
$("#s-new").addEventListener("click", () => fillSearchForm(null));
$("#b-q").addEventListener("input", debounce(renderBoards, 150));
$("#b-on").addEventListener("change", renderBoards);
$("#board-form").addEventListener("submit", addBoard);
$("#prune-form").addEventListener("submit", previewPrune);
$("#p-apply").addEventListener("click", applyPrune);
$("#pull-now").addEventListener("click", pullNow);
});
})();
+107
View File
@@ -0,0 +1,107 @@
{% extends "base.html" %}
{% block title %}Sources — roledesk{% endblock %}
{% block content %}
<div class="row-between">
<h1>Sources</h1>
<button type="button" id="pull-now" class="secondary">Pull now</button>
</div>
<p class="lede">What a pull reads: saved searches, which find roles at any company, and company boards, which read every role one company lists. A pull runs on the Schedules tab's cadence. Switching one off closes its open roles at once; switched back on, the next pull reopens the ones it still finds.</p>
<p id="pull-status" class="count" role="status"></p>
<section class="section">
<h2>Searches</h2>
<p id="searches-count" class="count" role="status"></p>
<table class="list">
<thead>
<tr><th>Search</th><th>Query</th><th class="num">Open roles</th><th>Countries</th><th>On</th><th></th></tr>
</thead>
<tbody id="searches-list"></tbody>
</table>
<form id="search-form" class="agent-form source-form" autocomplete="off">
<h2 id="search-form-title">New search</h2>
<div class="form-row">
<label>Board
<select id="s-provider">
<option value="jobindex">Jobindex (Denmark)</option>
<option value="adzuna">Adzuna (UK and EU)</option>
</select></label>
<label class="grow">Name <input type="text" id="s-name" required placeholder="Copenhagen backend"></label>
</div>
<label>Query
<span class="hint" data-provider="jobindex">Jobindex syntax: words are OR, <code>"a b"</code> is a phrase, <code>+word</code> is required.</span>
<span class="hint" data-provider="adzuna" hidden>Every word must appear.</span>
<input type="text" id="s-q" required spellcheck="false"></label>
<div class="form-row">
<label data-provider="jobindex">Area
<select id="s-area">
{% for value, label in areas %}<option value="{{ value }}"{% if value == 1 %} selected{% endif %}>{{ label }}</option>{% endfor %}
</select></label>
<label data-provider="adzuna" hidden>Country
<select id="s-country">
{% for value, label in countries %}<option value="{{ value }}">{{ label }}</option>{% endfor %}
</select></label>
<label data-provider="adzuna" hidden class="grow">Place <input type="text" id="s-where" placeholder="Berlin"></label>
<label>Pages <span class="hint">per pull, up to 10</span>
<input type="number" id="s-pages" min="1" max="10" value="10"></label>
</div>
<p id="adzuna-key" class="note" hidden>No Adzuna key is set. Get a free one at developer.adzuna.com and set <code>ADZUNA_APP_ID</code> and <code>ADZUNA_APP_KEY</code> before starting the app.</p>
<div class="agent-actions">
<button type="button" id="s-try" class="secondary">Try it</button>
<button type="submit">Save</button>
<button type="button" id="s-new" class="secondary">New</button>
</div>
<p id="search-status" class="count" role="status"></p>
<ol id="search-sample" class="sample"></ol>
</form>
</section>
<section class="section">
<h2>Company boards</h2>
<form class="filters" autocomplete="off" onsubmit="return false">
<input type="search" id="b-q" placeholder="Filter by company or board" aria-label="Filter boards">
<select id="b-on" aria-label="On or off">
<option value="">On and off</option>
<option value="on">On</option>
<option value="off">Off</option>
</select>
</form>
<p id="boards-count" class="count" role="status"></p>
<table class="list">
<thead>
<tr><th>Company</th><th>Board</th><th class="num">Open roles</th><th>Countries</th><th>On</th></tr>
</thead>
<tbody id="boards-list"></tbody>
</table>
<form id="board-form" class="agent-form source-form" autocomplete="off">
<h2>Follow a company</h2>
<div class="form-row">
<label class="grow">Board <span class="hint">host:slug, from its careers page URL: greenhouse, lever, ashby, workable, smartrecruiters, recruitee, personio</span>
<input type="text" id="b-board" required placeholder="ashby:acme" spellcheck="false"></label>
<label class="grow">Company <input type="text" id="b-name" required placeholder="Acme"></label>
</div>
<div class="agent-actions"><button type="submit">Follow</button></div>
<p id="board-status" class="count" role="status"></p>
</form>
</section>
<section class="section">
<h2>Prune boards by region</h2>
<form id="prune-form" class="agent-form source-form" autocomplete="off">
<label>Region <span class="hint"><code>europe</code>, or country codes such as <code>DK,SE,NO,EU</code>. EU is a role open anywhere in Europe.</span>
<input type="text" id="p-region" value="europe" spellcheck="false"></label>
<div class="agent-actions">
<button type="submit" class="secondary">Preview</button>
<button type="button" id="p-apply" disabled>Switch them off</button>
</div>
<p id="prune-status" class="count" role="status"></p>
<ul id="prune-list" class="sample"></ul>
</form>
</section>
{% endblock %}
{% block scripts %}
<script src="{{ asset('sources.js') }}"></script>
{% endblock %}
+166 -4
View File
@@ -1,4 +1,5 @@
"""The JSON routes the tabs use: roles, companies, applications, agents.
"""The JSON routes the tabs use: roles, companies, sources, applications,
agents.
`/api/jobs*`, `/api/schedules*`, `/api/notices` and `/ws/jobs` are
`pipeline.web`'s and `/api/account*` is `auth`'s, both wired in `app/main.py`.
@@ -9,15 +10,17 @@ This is roledesk's own API, over the same store. Every refusal is
from __future__ import annotations
from datetime import timedelta
from typing import Annotated
from typing import Annotated, Any
from fastapi import APIRouter, HTTPException, Query, Response
from pydantic import BaseModel, Field
from app import jobs
from app.models import (
AFFINITY, Agent, Application, Company, Gate, ScoredRole, Status, Weights,
AFFINITY, Agent, Application, Company, Gate, ScoredRole, Status, Subscription, Weights,
)
from app.web.deps import StoreDep
from app.storage import Store
from app.web.deps import ConfigDep, StoreDep
router = APIRouter(prefix="/api", tags=["roledesk"])
@@ -87,6 +90,165 @@ def set_affinity(key: str, body: AffinityBody, store: StoreDep) -> Company:
raise HTTPException(404, f"no such company: {key}") from None
# ----------------------------------------------------------------- sources
class SourceOut(BaseModel):
"""A board or a saved search, with what it brings in: its open roles, and
the countries they are in, most first."""
id: int
source: str
target: str
label: str
provider: str
params: dict[str, Any]
on: bool
open_roles: int
countries: list[tuple[str, int]]
class SourcesOut(BaseModel):
searches: list[SourceOut]
boards: list[SourceOut]
#: Whether an adzuna search can run: a key in the config or the environment.
adzuna_key: bool
def _source_out(sub: Subscription, countries: dict[str, int]) -> SourceOut:
return SourceOut(
id=sub.id, source=sub.source, target=sub.target, label=sub.label,
provider=sub.board, params=sub.params, on=bool(sub.subscribed and sub.fetch),
open_roles=sum(countries.values()),
countries=sorted(countries.items(), key=lambda kv: -kv[1]))
def _one_source(store: Store, sub_id: int) -> SourceOut:
sub = store.subscription(sub_id)
if sub is None:
raise HTTPException(404, f"no such board or search: {sub_id}")
return _source_out(sub, store.open_countries_by_subscription().get(sub_id, {}))
@router.get("/sources")
def list_sources(store: StoreDep, config: ConfigDep) -> SourcesOut:
"""Every board, and every search not removed. A board switched off stays
listed, so it can be switched back on."""
counts = store.open_countries_by_subscription()
searches = [_source_out(s, counts.get(s.id, {}))
for s in store.subscriptions(source="search") if s.show]
boards = [_source_out(s, counts.get(s.id, {})) for s in store.subscriptions(source="ats")]
return SourcesOut(searches=searches, boards=boards,
adzuna_key=jobs.sources.has_key(config, "adzuna"))
class SwitchBody(BaseModel):
on: bool
@router.post("/sources/{sub_id}/switch")
def switch_source(sub_id: int, body: SwitchBody, store: StoreDep) -> dict[str, Any]:
"""On or off for the next pull. Off closes the open roles at once; on
reopens the ones the next pull still finds."""
if store.subscription(sub_id) is None:
raise HTTPException(404, f"no such board or search: {sub_id}")
_, closed = jobs.sources.set_fetch(store, sub_id, body.on)
return {"source": _one_source(store, sub_id), "closed": closed}
class SearchBody(BaseModel):
provider: str
name: str = ""
q: str = ""
area: int = 0
country: str = ""
where: str = ""
pages: Annotated[int, Field(ge=0, le=10)] = 0
def _params(body: SearchBody) -> dict[str, Any]:
try:
return jobs.sources.search_params(body.provider, body.q, area=body.area,
country=body.country, where=body.where,
pages=body.pages)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
@router.post("/searches/try")
def try_search(body: SearchBody, config: ConfigDep) -> dict[str, Any]:
"""Run a search once without saving it: how many it finds, whether that is
every match, and the first ten, so a query can be tuned before it is kept."""
try:
rows, complete = jobs.sources.try_search(config, body.provider, body.name,
_params(body))
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
sample = [{k: r.get(k) for k in ("title", "company", "place", "url")} for r in rows[:10]]
return {"found": len(rows), "complete": complete, "sample": sample}
@router.post("/searches")
def save_search(body: SearchBody, store: StoreDep, config: ConfigDep) -> dict[str, Any]:
"""Save a search, or update the one of that provider and name. It is run
once first and refused if it finds nothing."""
try:
sub, found, complete = jobs.sources.save_search(config, store, body.provider,
body.name, _params(body))
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
return {"source": _one_source(store, sub.id), "found": found, "complete": complete}
@router.delete("/searches/{sub_id}")
def remove_search(sub_id: int, store: StoreDep) -> dict[str, Any]:
try:
_, closed = jobs.sources.remove_search(store, sub_id)
except KeyError:
raise HTTPException(404, f"no such search: {sub_id}") from None
return {"closed": closed}
class BoardBody(BaseModel):
board: str
name: str
@router.post("/boards", status_code=201)
def add_board(body: BoardBody, store: StoreDep) -> dict[str, Any]:
"""Follow a company's ATS board. It is fetched first, and refused if it
does not answer or lists nothing."""
try:
sub, found = jobs.sources.add_board(store, body.board, body.name)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
return {"source": _one_source(store, sub.id), "found": found}
@router.get("/prune")
def prune_preview(store: StoreDep, region: str = "europe") -> dict[str, Any]:
"""The boards a prune would switch off: every one on, none of whose open
roles is in the region."""
try:
_, plan = jobs.sources.prune_plan(store, region)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
return {"region": region, "boards": [{"id": sub.id, "label": sub.label,
"target": sub.target, "open_roles": n}
for sub, n in plan]}
class PruneBody(BaseModel):
region: str = "europe"
@router.post("/prune")
def prune(body: PruneBody, store: StoreDep) -> dict[str, Any]:
try:
return jobs.sources.prune(store, body.region)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
# ----------------------------------------------------------- applications
class SaveBody(BaseModel):
+8
View File
@@ -17,6 +17,7 @@ from typing import Annotated
from fastapi import Depends
from starlette.requests import HTTPConnection
from app.config import Config
from app.storage import Store
@@ -25,3 +26,10 @@ def get_store(request: HTTPConnection) -> Store:
StoreDep = Annotated[Store, Depends(get_store)]
def get_config(request: HTTPConnection) -> Config:
return request.app.state.config
ConfigDep = Annotated[Config, Depends(get_config)]
+15 -3
View File
@@ -1,18 +1,21 @@
"""The tabs: eight pages, declared once.
"""The tabs: nine pages, declared once.
`TABS` is the whole navigation. `base.html` renders it, every page marks
itself active by its own `path`, and adding a tab is one entry here plus one
template.
Every page is a shell. Roles, Applications, Agents and Companies are filled
and driven by `static/roles.js` from the JSON routes in `web/api.py`; Jobs,
and driven by `static/roles.js` from the JSON routes in `web/api.py`, Sources
by `static/sources.js`; Jobs,
Schedules and Account are shells around the modules' own elements. What the
server renders is only what a select needs to offer, so the options cannot
drift from what the store accepts.
`FUNCTIONS` is `plugins.ats`'s, not `app`'s: the nine job functions are the
classifier's vocabulary, and this reaches `plugins/` the same way `app.cli`
does, since a plugin is app-agnostic and never reaches back into `app`.
does, since a plugin is app-agnostic and never reaches back into `app`. The
Jobindex areas and Adzuna countries on the Sources tab come from
`plugins.search` for the same reason.
"""
from __future__ import annotations
@@ -32,6 +35,7 @@ if str(_ROOT) not in sys.path:
sys.path.insert(0, str(_ROOT))
from plugins.ats import FUNCTIONS # noqa: E402
from plugins.search import ADZUNA_COUNTRIES, JOBINDEX_AREAS # noqa: E402
TEMPLATE_DIR = Path(__file__).resolve().parent.parent / "templates"
STATIC_DIR = Path(__file__).resolve().parent.parent / "static"
@@ -50,6 +54,7 @@ TABS: tuple[Tab, ...] = (
Tab("/applications", "Applications", "applications.html"),
Tab("/agents", "Agents", "agents.html"),
Tab("/companies", "Companies", "companies.html"),
Tab("/sources", "Sources", "sources.html"),
Tab("/jobs", "Jobs", "jobs.html"),
Tab("/schedules", "Schedules", "schedules.html"),
Tab("/messages", "Messages", "messages.html"),
@@ -61,6 +66,8 @@ TABS: tuple[Tab, ...] = (
FUNCTION_OPTIONS = [(f, f.replace("/", " / ").capitalize()) for f in FUNCTIONS]
REMOTE_OPTIONS = [(r, r.capitalize()) for r in REMOTE if r]
STATUS_OPTIONS = [(s, s.capitalize()) for s in STATUSES]
AREA_OPTIONS = list(JOBINDEX_AREAS.items())
COUNTRY_OPTIONS = [(c, c.upper()) for c in sorted(ADZUNA_COUNTRIES)]
def asset(path: str) -> str:
@@ -107,6 +114,11 @@ def companies(request: Request) -> HTMLResponse:
return page(request, "companies.html")
@router.get("/sources", response_class=HTMLResponse)
def sources(request: Request) -> HTMLResponse:
return page(request, "sources.html", areas=AREA_OPTIONS, countries=COUNTRY_OPTIONS)
@router.get("/jobs", response_class=HTMLResponse)
def jobs(request: Request) -> HTMLResponse:
return page(request, "jobs.html")