Sources tab: searches, company boards and the prune by region, in the app

What a pull reads can now be changed on a page instead of only the command
line: try a search and see its first ten results before saving it, edit or
remove one, follow a company's ATS board, switch any board or search on or
off, and preview and apply a prune by region. Each row shows its open roles
and the countries they are in.

The page and `app companies`/`app searches` call the same functions in
app/jobs/sources.py. Switching a target off closes its open roles, and the
pull's settle closes them again if a batch in flight reopened them. Roles
pulled before `country` existed are backfilled at startup.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-24 15:15:04 +02:00
1 parent 047f7cd8ad
commit 6b40702a9a
16 files changed
+1068 -92

No files matched your search

+4 -1
View File
@@ -17,6 +17,8 @@ uv run app serve # http://127.0.0.1:8790
## Searches and companies
The Sources tab does all of this; the same actions exist on the command line:
```bash
uv run app searches add jobindex "Copenhagen golang" --q "golang go-udvikler" --area 1
uv run app searches add adzuna "Berlin backend" --q backend --country de --where Berlin
@@ -24,7 +26,7 @@ uv run app searches list
uv run app companies prune --in DK,SE,NO,EU # lists; add --apply to unsubscribe
```
A Jobindex query is its own syntax: words are OR, `"a b"` is a phrase, `+word` is required; `--area 1` is Region Hovedstaden. An Adzuna search needs a free key from developer.adzuna.com in `ADZUNA_APP_ID` and `ADZUNA_APP_KEY`. `searches add` runs the search once and saves it only if it finds something. The `embeddings` extra is what `fit` needs, `agent` is for `extract` on a model, `passkeys` for the Account tab's passkeys.
A Jobindex query is its own syntax: words are OR, `"a b"` is a phrase, `+word` is required; `--area 1` is Region Hovedstaden. An Adzuna search needs a free key from developer.adzuna.com in `ADZUNA_APP_ID` and `ADZUNA_APP_KEY`. `searches add` runs the search once and saves it only if it finds something. Switching a board or search off closes its open roles; the next pull after switching it back on reopens the ones it still lists. The `embeddings` extra is what `fit` needs, `agent` is for `extract` on a model, `passkeys` for the Account tab's passkeys.
## The tabs
@@ -34,6 +36,7 @@ A Jobindex query is its own syntax: words are OR, `"a b"` is a phrase, `+word` i
| **Applications** | the saved roles and their status, each change recorded as an event |
| **Agents** | the search agents: intent, require and prefer gates, weights. Saving one rescores it |
| **Companies** | every company whose board is read, its open roles, and its affinity: mute, neutral, like, favourite |
| **Sources** | what a pull reads: saved searches (try, save, edit, remove), company boards (follow one, switch any on or off), and a prune of boards by region, with the countries each one's roles are in |
| **Jobs** | a button per job (`pull`, `extract`, `embed`, `score`) and what is running |
| **Schedules** | `pull` every 6h by default, editable in place |
| **Messages** | everything the server said that nobody has cleared |
+34 -78
View File
@@ -83,7 +83,7 @@ def build_parser() -> argparse.ArgumentParser:
searches = sub.add_parser("searches", help="saved searches on cross-company job boards")
searches_sub = searches.add_subparsers(dest="action", required=True)
search_add = searches_sub.add_parser(
"add", help="run a search once, then save it for every pull")
"add", help="run a search once, then save it for every pull (or update it)")
search_add.add_argument("provider", choices=["jobindex", "adzuna"])
search_add.add_argument("name", help='what to call it, e.g. "Copenhagen backend"')
search_add.add_argument("--q", required=True,
@@ -97,7 +97,7 @@ def build_parser() -> argparse.ArgumentParser:
help="result pages per pull (default 5, at most 10)")
searches_sub.add_parser("list", help="every saved search and its open roles")
search_remove = searches_sub.add_parser(
"remove", help="stop running a search; its roles stay until they go stale")
"remove", help="stop running a search and close its open roles")
search_remove.add_argument("name")
db = sub.add_parser("db", help="database housekeeping")
@@ -275,34 +275,15 @@ def add_company(store: Store, board: str, name: str) -> int:
"""`app companies add greenhouse:acme "Acme"`: fetch the board first
and refuse if it does not answer, is not a known host, or the company
already exists. Port of seedgen's `-add`."""
from plugins.ats import careers_url as careers_url_of, fetch_board, is_known_target
from app import jobs
from app.models import company_key
if not is_known_target(board):
print(f"companies add: {board!r} is not host:slug for a known ATS host",
file=sys.stderr)
return 1
key = company_key(name)
if store.company(key) is not None:
print(f"companies add: {key!r} already exists", file=sys.stderr)
return 1
try:
rows = fetch_board(board)
except Exception as exc:
print(f"companies add: {board} did not answer: {exc}", file=sys.stderr)
return 1
if not rows:
print(f"companies add: {board} answered but lists no roles; nothing added",
file=sys.stderr)
return 1
host, _, _slug = board.partition(":")
store.add_company(name, key=key, careers_url=careers_url_of(board))
try:
store.add_subscription("ats", board, label=name, company=name, board=host)
sub, found = jobs.sources.add_board(store, board, name)
except ValueError as exc:
print(f"companies add: {exc}", file=sys.stderr)
return 1
print(json.dumps({"key": key, "board": board, "roles": len(rows)}))
print(json.dumps({"key": company_key(sub.company), "board": sub.target, "roles": found}))
return 0
@@ -325,38 +306,29 @@ def check_companies(store: Store) -> int:
def prune_companies(store: Store, region: str, *, apply: bool) -> int:
"""`app companies prune --in DK,EU`: every subscribed board none of whose
open roles is in the region, by `country`, which this fills from `place`
first where it is empty. `europe`, the default, is `EUROPE`. `--apply` unsubscribes those boards and closes their open roles,
since a board no longer pulled would otherwise keep them open for good.
`app companies seed` never brings one back: it skips a company that exists.
"""`app companies prune --in DK,EU`: every fetched board none of whose open
roles is in the region, which is `europe` unless named. `--apply`
switches them off, which closes their open roles. `app companies seed`
never brings one back: it skips a company that exists. See
`app.jobs.sources.prune_plan` for which boards count."""
from app import jobs
A board whose open roles all name no country is left alone: it says
nothing about where that company hires."""
from app.models import EUROPE
filled = store.backfill_country()
countries = store.open_countries_by_subscription()
wanted = {c.strip().upper() for c in region.split(",") if c.strip()}
if "EUROPE" in wanted:
wanted = (wanted - {"EUROPE"}) | set(EUROPE)
pruned = [(sub, sum(countries[sub.id].values()))
for sub in store.subscriptions(source="ats", fetchable=True)
if set(countries.get(sub.id, {})) - {""}
and not wanted.intersection(countries[sub.id])]
closed = 0
for sub, count in sorted(pruned, key=lambda p: -p[1]):
try:
filled, plan = jobs.sources.prune_plan(store, region)
except ValueError as exc:
print(f"companies prune: {exc}", file=sys.stderr)
return 1
for sub, count in plan:
print(f"{sub.target:40} {sub.label:30} {count:6} open, none in {region}")
if apply:
store.unsubscribe(sub.id)
closed += store.close_subscription(sub.id)
kept = len(store.subscriptions(source="ats", fetchable=True))
print(json.dumps({"countries_filled": filled, "pruned": len(pruned), "applied": apply,
"roles_closed": closed, "boards_left": kept}))
result: dict[str, Any] = {"countries_filled": filled, "pruned": len(plan), "applied": apply}
if apply:
result |= jobs.sources.prune(store, region)
print(json.dumps(result))
return 0
def run_searches(args: argparse.Namespace) -> int:
from app import jobs
from app.storage import Store
config = config_mod.load(args.config)
@@ -368,12 +340,12 @@ def run_searches(args: argparse.Namespace) -> int:
if sub is None:
print(f"searches remove: no search named {args.name!r}", file=sys.stderr)
return 1
store.unsubscribe(sub.id)
print(json.dumps({"removed": sub.label}))
_, closed = jobs.sources.remove_search(store, sub.id)
print(json.dumps({"removed": sub.label, "roles_closed": closed}))
return 0
countries = store.open_countries_by_subscription()
for sub in store.subscriptions(source="search"):
removed = "" if sub.subscribed and sub.fetch else " (removed)"
removed = "" if sub.subscribed and sub.fetch else " (off)"
open_roles = sum(countries.get(sub.id, {}).values())
print(f"{sub.label:30} {sub.target:40} {open_roles:5} open "
f"{json.dumps(sub.params, ensure_ascii=False)}{removed}")
@@ -382,36 +354,20 @@ def run_searches(args: argparse.Namespace) -> int:
def add_search(config: config_mod.Config, store: Store, args: argparse.Namespace) -> int:
"""`app searches add jobindex "Copenhagen backend" --q '"backend developer"'
--area 1`: run the search once and refuse if it fails or finds nothing, so
a typo in a query is found now rather than by an empty tab later."""
from app.jobs import search_target
from plugins.search import run_search
--area 1`: run the search once and save it only if it finds something.
A search of the same name is updated in place."""
from app import jobs
params: dict[str, Any] = {"q": args.q}
if args.provider == "jobindex" and args.area:
params["area"] = args.area
if args.provider == "adzuna":
params["country"] = args.country.lower()
if args.where:
params["where"] = args.where
if args.pages:
params["pages"] = args.pages
target = search_target(args.provider, args.name)
try:
rows, complete = run_search(target, params, dict(config.sources.get("search") or {}))
except Exception as exc:
print(f"searches add: {args.provider} did not answer: {exc}", file=sys.stderr)
return 1
if not rows:
print("searches add: the search found nothing; not saved", file=sys.stderr)
return 1
try:
store.add_subscription("search", target, label=args.name, board=args.provider,
params=params)
params = jobs.sources.search_params(args.provider, args.q, area=args.area,
country=args.country, where=args.where,
pages=args.pages)
sub, found, complete = jobs.sources.save_search(config, store, args.provider, args.name,
params)
except ValueError as exc:
print(f"searches add: {exc}", file=sys.stderr)
return 1
print(json.dumps({"search": args.name, "target": target, "roles": len(rows),
print(json.dumps({"search": sub.label, "target": sub.target, "roles": found,
"complete": complete}, ensure_ascii=False))
return 0
+4 -2
View File
@@ -7,13 +7,15 @@ task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`.
score.py every open role scored for every enabled search agent
cascade.py pull → extract → embed → score, what `TASKS` offers, the
`pull` job a schedule fires, and that schedule
sources.py what a pull reads, changed by hand: boards and searches
added, switched on or off, pruned by region
"""
from __future__ import annotations
from typing import Any
from app.jobs import cascade, embed, extract, pull, score
from app.jobs import cascade, embed, extract, pull, score, sources
from app.jobs.pull import search_target, seed_searches
from app.jobs.score import seed_agents
@@ -33,4 +35,4 @@ def bootstrap(config_path: str = "") -> tuple[Any, Any]:
__all__ = ["BOOTSTRAP", "bootstrap", "cascade", "embed", "extract", "pull", "score",
"search_target", "seed_agents", "seed_searches"]
"search_target", "seed_agents", "seed_searches", "sources"]
+11 -1
View File
@@ -88,7 +88,10 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
from plugins.ats import ATSSource
from plugins.search import SearchSource
subs = {s.target: s for s in store.subscriptions(source=source) if s.target in targets}
# Fetchable now, not when the pull was queued: a board switched off on the
# Sources tab since then is skipped.
subs = {s.target: s for s in store.subscriptions(source=source, fetchable=True)
if s.target in targets}
options = dict(config.sources.get(source) or {})
plugin: ATSSource | SearchSource
if source == "search":
@@ -124,9 +127,16 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
`ats:ok:<target>` was not written during *this* run — it failed, or its
whole batch job never got to it — is never closed. A search closes by its
list only when that list was complete, and otherwise by `STALE_DAYS`.
A board or search switched off while its batch was in flight had its
roles reopened by that batch, after the switch closed them, so every
switched-off one is closed again here.
"""
answered = searched = failed = closed = 0
for source in SOURCES:
for sub in store.subscriptions(source=source):
if not (sub.subscribed and sub.fetch):
closed += store.close_subscription(sub.id)
run_marker = store.state.get(f"{source}:run", "")
for sub in store.subscriptions(source=source, fetchable=True):
stamp = store.state.get(f"{source}:ok:{sub.target}", "")
+171
View File
@@ -0,0 +1,171 @@
"""What a pull reads, changed by hand: add a board or a search, switch one on
or off, prune boards by region. `app companies`, `app searches` and the
Sources tab all call these, so the command line and the page cannot disagree
about what "off" means.
A board or search switched off has its open roles closed at once. Nothing
else would ever close them, since only a pull that reads a target closes its
roles; switched back on, the next pull reopens every role it still lists.
"""
from __future__ import annotations
import os
from typing import Any
from app.config import Config
from app.models import EUROPE, Subscription, company_key
from app.storage import Store
from app.jobs.pull import search_target
#: The providers `plugins.search` runs, and which of them needs a key.
PROVIDERS = ("jobindex", "adzuna")
def region_codes(region: str) -> set[str]:
"""`europe`, or comma-separated ISO codes such as `DK,SE,EU`, as a set of
codes. `europe` may be one entry among codes: `europe,US`."""
codes = {c.strip().upper() for c in (region or "").split(",") if c.strip()}
if "EUROPE" in codes:
codes = (codes - {"EUROPE"}) | set(EUROPE)
if not codes:
raise ValueError("name a region: europe, or country codes such as DK,SE,EU")
return codes
def prune_plan(store: Store, region: str) -> tuple[int, list[tuple[Subscription, int]]]:
"""Every fetched board none of whose open roles is in the region, with its
open role count, largest first; and how many roles had `country` filled
from `place` on the way. A board whose roles all name no country is kept:
it says nothing about where that company hires."""
wanted = region_codes(region)
filled = store.backfill_country()
countries = store.open_countries_by_subscription()
plan = [(sub, sum(countries[sub.id].values()))
for sub in store.subscriptions(source="ats", fetchable=True)
if set(countries.get(sub.id, {})) - {""}
and not wanted.intersection(countries[sub.id])]
return filled, sorted(plan, key=lambda p: -p[1])
def prune(store: Store, region: str) -> dict[str, Any]:
"""Switch off every board `prune_plan` names."""
filled, plan = prune_plan(store, region)
closed = sum(set_fetch(store, sub.id, False)[1] for sub, _ in plan)
return {"countries_filled": filled, "pruned": len(plan), "roles_closed": closed,
"boards_left": len(store.subscriptions(source="ats", fetchable=True))}
def set_fetch(store: Store, sub_id: int, fetch: bool) -> tuple[Subscription, int]:
"""Switch one board or search on or off. Off closes its open roles and
returns how many; see the module docstring."""
sub = store.update_subscription(sub_id, subscribed=fetch, fetch=fetch, show=True)
closed = 0 if fetch else store.close_subscription(sub_id)
return sub, closed
def add_board(store: Store, board: str, name: str) -> tuple[Subscription, int]:
"""Follow one company's ATS board: fetch it first, and refuse if the host
is unknown, the company exists, the board does not answer or lists
nothing. Returns the subscription and how many roles the board lists."""
from plugins.ats import careers_url, fetch_board, is_known_target
board, name = board.strip(), name.strip()
if not name:
raise ValueError("a company needs a name")
if not is_known_target(board):
raise ValueError(f"{board!r} is not host:slug for a known ATS host, "
"e.g. greenhouse:acme or ashby:acme")
key = company_key(name)
if store.company(key) is not None:
raise ValueError(f"{name} is already followed")
try:
rows = fetch_board(board)
except Exception as exc:
raise ValueError(f"{board} did not answer: {exc}") from exc
if not rows:
raise ValueError(f"{board} answered but lists no roles; nothing added")
host = board.partition(":")[0]
store.add_company(name, key=key, careers_url=careers_url(board))
sub = store.add_subscription("ats", board, label=name, company=name, board=host)
return sub, len(rows)
def search_params(provider: str, q: str, *, area: int = 0, country: str = "",
where: str = "", pages: int = 0) -> dict[str, Any]:
"""The `params` a search subscription stores, with only the keys its
provider reads."""
if provider not in PROVIDERS:
raise ValueError(f"provider is one of {', '.join(PROVIDERS)}")
if not q.strip():
raise ValueError("a search needs a query")
params: dict[str, Any] = {"q": q.strip()}
if provider == "jobindex" and area:
params["area"] = int(area)
if provider == "adzuna":
params["country"] = country.strip().lower()
if where.strip():
params["where"] = where.strip()
if pages:
params["pages"] = int(pages)
return params
def has_key(config: Config, provider: str) -> bool:
"""Whether `provider` can run: jobindex always, adzuna with a key in the
config or the environment."""
if provider != "adzuna":
return True
keys = (config.sources.get("search") or {}).get("adzuna") or {}
return bool((keys.get("app_id") or os.environ.get("ADZUNA_APP_ID"))
and (keys.get("app_key") or os.environ.get("ADZUNA_APP_KEY")))
def try_search(config: Config, provider: str, name: str,
params: dict[str, Any]) -> tuple[list[dict[str, Any]], bool]:
"""Run a search once, outside any pull. Raises ValueError with the
provider's own reason when it does not answer."""
from plugins.search import run_search
try:
return run_search(search_target(provider, name or "preview"), params,
dict(config.sources.get("search") or {}))
except ValueError:
raise
except Exception as exc:
raise ValueError(f"{provider} did not answer: {exc}") from exc
def save_search(config: Config, store: Store, provider: str, name: str,
params: dict[str, Any]) -> tuple[Subscription, int, bool]:
"""Run a search once and save it only if it finds something, so a typo in a
query shows now and not as an empty tab later. A search of the same
provider and name is updated in place, removed or not: its roles keep
their subscription. Returns the subscription, how many roles it found, and
whether that was every match."""
name = name.strip()
if not name:
raise ValueError("a search needs a name")
rows, complete = try_search(config, provider, name, params)
if not rows:
raise ValueError("the search found nothing; not saved")
target = search_target(provider, name)
existing = next((s for s in store.subscriptions(source="search") if s.target == target),
None)
if existing is None:
sub = store.add_subscription("search", target, label=name, board=provider,
params=params)
else:
sub = store.update_subscription(existing.id, label=name, params=params,
subscribed=True, fetch=True, show=True)
return sub, len(rows), complete
def remove_search(store: Store, sub_id: int) -> tuple[Subscription, int]:
"""Stop running a search and hide it. Its open roles close; adding a search
of the same name again brings the row back."""
sub = store.subscription(sub_id)
if sub is None or sub.source != "search":
raise KeyError(sub_id)
store.unsubscribe(sub_id)
return sub, store.close_subscription(sub_id)
+3
View File
@@ -46,6 +46,9 @@ def create_app(config: Config | None = None) -> FastAPI:
store = Store(config.database)
jobs.seed_agents(config, store)
jobs.seed_searches(config, store)
# Roles pulled before `country` was read from `place` have none; after the
# first start only the places the lookup cannot place are left to read.
store.backfill_country()
app = FastAPI(title="roledesk", docs_url="/api/docs",
summary="ATS boards and job searches in, roles scored per search agent, "
+20
View File
@@ -352,6 +352,26 @@ textarea:focus-visible {
.weights label { flex: 1 1 6rem; }
.agent-actions { display: flex; gap: .5rem; flex-wrap: wrap; }
/* sources */
.section { margin: 1.5rem 0 2rem; }
.section > h2 { margin-bottom: .25rem; }
.source-form { margin-top: 1rem; }
.form-row { display: flex; gap: .75rem; flex-wrap: wrap; }
.form-row label { flex: 0 1 auto; }
.form-row label.grow { flex: 1 1 14rem; }
.source-form select {
background: var(--bg); border: 1px solid var(--line);
border-radius: .3rem; padding: .35rem .5rem;
}
.source-form input[type="number"] { max-width: 6rem; }
.source-form [hidden] { display: none; }
.sample { margin: 0; padding-left: 1.25rem; font-size: .9rem; }
.sample li { margin: .15rem 0; }
.mono { font-family: var(--mono); overflow-wrap: anywhere; }
.row-actions { display: inline-flex; gap: .35rem; }
.row-actions button { padding: .25rem .6rem; }
/* applications and companies */
table.list { width: 100%; border-collapse: collapse; background: var(--panel); }
+366
View File
@@ -0,0 +1,366 @@
/* The Sources tab: saved searches, company boards, and the prune by region.
*
* The template is a shell; the lists come from /api/sources and every action
* is a fetch through `api()` from app.js, so a refusal (a search that finds
* nothing, a board that does not answer) is one toast with the server's own
* sentence. The catch blocks here only put a control back.
*/
(function () {
"use strict";
const api = window.api;
const $ = (sel, root = document) => root.querySelector(sel);
/** @type {{searches: object[], boards: object[], adzuna_key: boolean}} */
let data = { searches: [], boards: [], adzuna_key: false };
/** @type {object[]} the boards the last prune preview named */
let pruneBoards = [];
function count(n, one, many) {
return `${n} ${n === 1 ? one : many || `${one}s`}`;
}
function debounce(fn, ms) {
let timer;
return (...args) => {
clearTimeout(timer);
timer = setTimeout(() => fn(...args), ms);
};
}
function cell(tr, label, content) {
const td = document.createElement("td");
td.dataset.label = label;
if (typeof content === "string") td.textContent = content;
else td.appendChild(content);
tr.appendChild(td);
return td;
}
/** "DK 120 · EU 4 · unknown 2", the three biggest. */
function countries(source) {
const shown = source.countries.slice(0, 3)
.map(([code, n]) => `${code || "unknown"} ${n}`);
if (source.countries.length > 3) shown.push("…");
return shown.join(" · ");
}
function query(source) {
const p = source.params || {};
const where = source.provider === "adzuna"
? [p.country && p.country.toUpperCase(), p.where].filter(Boolean).join(", ")
: (p.area ? $(`#s-area option[value="${p.area}"]`)?.textContent : "All of Denmark");
return `${p.q || ""}${where ? ` — ${where}` : ""}`;
}
function onSwitch(source, rerender) {
const box = document.createElement("input");
box.type = "checkbox";
box.checked = source.on;
box.setAttribute("aria-label", `Pull ${source.label}`);
box.addEventListener("change", async () => {
box.disabled = true;
try {
const result = await api(`/api/sources/${source.id}/switch`,
{ method: "POST", body: { on: box.checked } });
Object.assign(source, result.source);
if (!box.checked && result.closed) {
window.toast.info(`${source.label}: ${count(result.closed, "open role")} closed.`);
}
} catch (err) {
box.checked = !box.checked;
}
box.disabled = false;
rerender();
});
return box;
}
async function load() {
data = await api("/api/sources");
}
// ---------------------------------------------------------------- searches
function renderSearches() {
const body = $("#searches-list");
body.replaceChildren();
for (const source of data.searches) {
const tr = document.createElement("tr");
if (!source.on) tr.classList.add("muted");
const name = document.createElement("span");
name.textContent = source.label;
const provider = document.createElement("span");
provider.className = "badge";
provider.textContent = source.provider;
const title = document.createElement("span");
title.append(name, " ", provider);
cell(tr, "Search", title);
cell(tr, "Query", query(source)).className = "mono small";
cell(tr, "Open roles", String(source.open_roles)).className = "num";
cell(tr, "Countries", countries(source)).className = "small";
cell(tr, "On", onSwitch(source, renderSearches));
const actions = document.createElement("span");
actions.className = "row-actions";
const edit = document.createElement("button");
edit.type = "button";
edit.className = "secondary";
edit.textContent = "Edit";
edit.addEventListener("click", () => fillSearchForm(source));
const remove = document.createElement("button");
remove.type = "button";
remove.className = "secondary";
remove.textContent = "Remove";
remove.addEventListener("click", () => removeSearch(source));
actions.append(edit, remove);
cell(tr, "", actions);
body.appendChild(tr);
}
const open = data.searches.reduce((sum, s) => sum + s.open_roles, 0);
$("#searches-count").textContent = data.searches.length
? `${count(data.searches.length, "search", "searches")}, ${count(open, "open role")}`
: "No searches yet. Add one below; the starter searches in config.yaml import on the first pull.";
}
function showProvider() {
const provider = $("#s-provider").value;
for (const node of document.querySelectorAll("#search-form [data-provider]")) {
node.hidden = node.dataset.provider !== provider;
}
$("#adzuna-key").hidden = provider !== "adzuna" || data.adzuna_key;
}
function fillSearchForm(source) {
const p = (source && source.params) || {};
$("#search-form-title").textContent = source ? `Edit ${source.label}` : "New search";
$("#s-provider").value = source ? source.provider : "jobindex";
$("#s-name").value = source ? source.label : "";
$("#s-name").readOnly = Boolean(source);
$("#s-provider").disabled = Boolean(source);
$("#s-q").value = p.q || "";
$("#s-area").value = String(p.area || (source ? 0 : 1));
$("#s-country").value = p.country || "de";
$("#s-where").value = p.where || "";
$("#s-pages").value = String(p.pages || 10);
$("#search-status").textContent = "";
$("#search-sample").replaceChildren();
showProvider();
if (source) $("#search-form").scrollIntoView({ behavior: "smooth", block: "start" });
}
function searchBody() {
return {
provider: $("#s-provider").value,
name: $("#s-name").value.trim(),
q: $("#s-q").value.trim(),
area: Number($("#s-area").value) || 0,
country: $("#s-country").value,
where: $("#s-where").value.trim(),
pages: Number($("#s-pages").value) || 0,
};
}
function found(result) {
const all = result.complete ? "every match" : "the page limit cut it short";
return `Found ${count(result.found, "role")} (${all}).`;
}
async function trySearch() {
const status = $("#search-status");
const button = $("#s-try");
button.disabled = true;
status.textContent = "Searching…";
$("#search-sample").replaceChildren();
try {
const result = await api("/api/searches/try", { method: "POST", body: searchBody() });
status.textContent = `${found(result)} The newest ${Math.min(result.found, 10)}:`;
const list = $("#search-sample");
for (const row of result.sample) {
const li = document.createElement("li");
const link = document.createElement("a");
link.href = row.url;
link.target = "_blank";
link.rel = "noopener";
link.textContent = row.title;
li.append(link, ` — ${[row.company, row.place].filter(Boolean).join(", ")}`);
list.appendChild(li);
}
} catch (err) {
status.textContent = "";
}
button.disabled = false;
}
async function saveSearch(event) {
event.preventDefault();
const status = $("#search-status");
const button = $("#search-form button[type=submit]");
button.disabled = true;
status.textContent = "Running it once before saving…";
try {
const result = await api("/api/searches", { method: "POST", body: searchBody() });
await load();
renderSearches();
fillSearchForm(null);
status.textContent = `Saved ${result.source.label}. ${found(result)} `
+ "Its roles arrive with the next pull.";
} catch (err) {
status.textContent = "";
}
button.disabled = false;
}
async function removeSearch(source) {
if (!window.confirm(`Remove ${source.label}? Its ${count(source.open_roles, "open role")} close.`)) {
return;
}
try {
await api(`/api/searches/${source.id}`, { method: "DELETE" });
data.searches = data.searches.filter((s) => s.id !== source.id);
renderSearches();
} catch (err) {
/* already toasted */
}
}
// ------------------------------------------------------------------ boards
function renderBoards() {
const q = $("#b-q").value.trim().toLowerCase();
const on = $("#b-on").value;
const rows = data.boards
.filter((b) => !q || b.label.toLowerCase().includes(q) || b.target.includes(q))
.filter((b) => !on || (on === "on") === b.on)
.sort((a, b) => Number(b.on) - Number(a.on) || b.open_roles - a.open_roles
|| a.label.localeCompare(b.label));
const body = $("#boards-list");
body.replaceChildren();
for (const board of rows) {
const tr = document.createElement("tr");
if (!board.on) tr.classList.add("muted");
const link = document.createElement("a");
link.href = `/?company=${encodeURIComponent(board.label)}`;
link.title = "This company's open roles";
link.textContent = board.label;
cell(tr, "Company", link);
cell(tr, "Board", board.target).className = "mono small";
cell(tr, "Open roles", String(board.open_roles)).className = "num";
cell(tr, "Countries", countries(board)).className = "small";
cell(tr, "On", onSwitch(board, renderBoards));
body.appendChild(tr);
}
const onCount = data.boards.filter((b) => b.on).length;
$("#boards-count").textContent =
`${count(data.boards.length, "board")}, ${onCount} on. Showing ${rows.length}.`;
}
async function addBoard(event) {
event.preventDefault();
const status = $("#board-status");
const button = $("#board-form button[type=submit]");
button.disabled = true;
status.textContent = "Reading the board once before following it…";
try {
const result = await api("/api/boards", {
method: "POST",
body: { board: $("#b-board").value.trim(), name: $("#b-name").value.trim() },
});
data.boards.push(result.source);
renderBoards();
$("#board-form").reset();
status.textContent = `Following ${result.source.label}: `
+ `${count(result.found, "role")} listed. They arrive with the next pull.`;
} catch (err) {
status.textContent = "";
}
button.disabled = false;
}
// ------------------------------------------------------------------- prune
async function previewPrune(event) {
event.preventDefault();
const status = $("#prune-status");
const list = $("#prune-list");
const region = $("#p-region").value.trim();
list.replaceChildren();
$("#p-apply").disabled = true;
status.textContent = "Checking every board…";
try {
const result = await api(`/api/prune?region=${encodeURIComponent(region)}`);
pruneBoards = result.boards;
const roles = pruneBoards.reduce((sum, b) => sum + b.open_roles, 0);
status.textContent = pruneBoards.length
? `${count(pruneBoards.length, "board")} with no open role in ${region}, `
+ `${count(roles, "open role")} between them:`
: `Every board that is on has a role in ${region}.`;
for (const board of pruneBoards) {
const li = document.createElement("li");
li.textContent = `${board.label} (${board.target}), ${count(board.open_roles, "open role")}`;
list.appendChild(li);
}
$("#p-apply").disabled = pruneBoards.length === 0;
} catch (err) {
status.textContent = "";
}
}
async function applyPrune() {
const region = $("#p-region").value.trim();
if (!window.confirm(`Switch off ${count(pruneBoards.length, "board")} and close their roles? `
+ "Each can be switched back on in the list above.")) return;
$("#p-apply").disabled = true;
try {
const result = await api("/api/prune", { method: "POST", body: { region } });
$("#prune-status").textContent = `Switched off ${count(result.pruned, "board")}, `
+ `closed ${count(result.roles_closed, "role")}. ${result.boards_left} boards are on.`;
$("#prune-list").replaceChildren();
await load();
renderBoards();
} catch (err) {
$("#p-apply").disabled = false;
}
}
// -------------------------------------------------------------------- pull
async function pullNow() {
const button = $("#pull-now");
button.disabled = true;
try {
await api("/api/jobs", { method: "POST", body: { kind: "pull", payload: {} } });
$("#pull-status").replaceChildren("Pull started. ");
const link = document.createElement("a");
link.href = "/jobs";
link.textContent = "Follow it on the Jobs tab";
$("#pull-status").appendChild(link);
} catch (err) {
/* already toasted */
}
button.disabled = false;
}
// -------------------------------------------------------------------- init
document.addEventListener("DOMContentLoaded", async () => {
if (!$("#searches-list")) return;
try {
await load();
} catch (err) {
return;
}
renderSearches();
renderBoards();
fillSearchForm(null);
$("#s-provider").addEventListener("change", showProvider);
$("#s-try").addEventListener("click", trySearch);
$("#search-form").addEventListener("submit", saveSearch);
$("#s-new").addEventListener("click", () => fillSearchForm(null));
$("#b-q").addEventListener("input", debounce(renderBoards, 150));
$("#b-on").addEventListener("change", renderBoards);
$("#board-form").addEventListener("submit", addBoard);
$("#prune-form").addEventListener("submit", previewPrune);
$("#p-apply").addEventListener("click", applyPrune);
$("#pull-now").addEventListener("click", pullNow);
});
})();
+107
View File
@@ -0,0 +1,107 @@
{% extends "base.html" %}
{% block title %}Sources — roledesk{% endblock %}
{% block content %}
<div class="row-between">
<h1>Sources</h1>
<button type="button" id="pull-now" class="secondary">Pull now</button>
</div>
<p class="lede">What a pull reads: saved searches, which find roles at any company, and company boards, which read every role one company lists. A pull runs on the Schedules tab's cadence. Switching one off closes its open roles at once; switched back on, the next pull reopens the ones it still finds.</p>
<p id="pull-status" class="count" role="status"></p>
<section class="section">
<h2>Searches</h2>
<p id="searches-count" class="count" role="status"></p>
<table class="list">
<thead>
<tr><th>Search</th><th>Query</th><th class="num">Open roles</th><th>Countries</th><th>On</th><th></th></tr>
</thead>
<tbody id="searches-list"></tbody>
</table>
<form id="search-form" class="agent-form source-form" autocomplete="off">
<h2 id="search-form-title">New search</h2>
<div class="form-row">
<label>Board
<select id="s-provider">
<option value="jobindex">Jobindex (Denmark)</option>
<option value="adzuna">Adzuna (UK and EU)</option>
</select></label>
<label class="grow">Name <input type="text" id="s-name" required placeholder="Copenhagen backend"></label>
</div>
<label>Query
<span class="hint" data-provider="jobindex">Jobindex syntax: words are OR, <code>"a b"</code> is a phrase, <code>+word</code> is required.</span>
<span class="hint" data-provider="adzuna" hidden>Every word must appear.</span>
<input type="text" id="s-q" required spellcheck="false"></label>
<div class="form-row">
<label data-provider="jobindex">Area
<select id="s-area">
{% for value, label in areas %}<option value="{{ value }}"{% if value == 1 %} selected{% endif %}>{{ label }}</option>{% endfor %}
</select></label>
<label data-provider="adzuna" hidden>Country
<select id="s-country">
{% for value, label in countries %}<option value="{{ value }}">{{ label }}</option>{% endfor %}
</select></label>
<label data-provider="adzuna" hidden class="grow">Place <input type="text" id="s-where" placeholder="Berlin"></label>
<label>Pages <span class="hint">per pull, up to 10</span>
<input type="number" id="s-pages" min="1" max="10" value="10"></label>
</div>
<p id="adzuna-key" class="note" hidden>No Adzuna key is set. Get a free one at developer.adzuna.com and set <code>ADZUNA_APP_ID</code> and <code>ADZUNA_APP_KEY</code> before starting the app.</p>
<div class="agent-actions">
<button type="button" id="s-try" class="secondary">Try it</button>
<button type="submit">Save</button>
<button type="button" id="s-new" class="secondary">New</button>
</div>
<p id="search-status" class="count" role="status"></p>
<ol id="search-sample" class="sample"></ol>
</form>
</section>
<section class="section">
<h2>Company boards</h2>
<form class="filters" autocomplete="off" onsubmit="return false">
<input type="search" id="b-q" placeholder="Filter by company or board" aria-label="Filter boards">
<select id="b-on" aria-label="On or off">
<option value="">On and off</option>
<option value="on">On</option>
<option value="off">Off</option>
</select>
</form>
<p id="boards-count" class="count" role="status"></p>
<table class="list">
<thead>
<tr><th>Company</th><th>Board</th><th class="num">Open roles</th><th>Countries</th><th>On</th></tr>
</thead>
<tbody id="boards-list"></tbody>
</table>
<form id="board-form" class="agent-form source-form" autocomplete="off">
<h2>Follow a company</h2>
<div class="form-row">
<label class="grow">Board <span class="hint">host:slug, from its careers page URL: greenhouse, lever, ashby, workable, smartrecruiters, recruitee, personio</span>
<input type="text" id="b-board" required placeholder="ashby:acme" spellcheck="false"></label>
<label class="grow">Company <input type="text" id="b-name" required placeholder="Acme"></label>
</div>
<div class="agent-actions"><button type="submit">Follow</button></div>
<p id="board-status" class="count" role="status"></p>
</form>
</section>
<section class="section">
<h2>Prune boards by region</h2>
<form id="prune-form" class="agent-form source-form" autocomplete="off">
<label>Region <span class="hint"><code>europe</code>, or country codes such as <code>DK,SE,NO,EU</code>. EU is a role open anywhere in Europe.</span>
<input type="text" id="p-region" value="europe" spellcheck="false"></label>
<div class="agent-actions">
<button type="submit" class="secondary">Preview</button>
<button type="button" id="p-apply" disabled>Switch them off</button>
</div>
<p id="prune-status" class="count" role="status"></p>
<ul id="prune-list" class="sample"></ul>
</form>
</section>
{% endblock %}
{% block scripts %}
<script src="{{ asset('sources.js') }}"></script>
{% endblock %}
+166 -4
View File
@@ -1,4 +1,5 @@
"""The JSON routes the tabs use: roles, companies, applications, agents.
"""The JSON routes the tabs use: roles, companies, sources, applications,
agents.
`/api/jobs*`, `/api/schedules*`, `/api/notices` and `/ws/jobs` are
`pipeline.web`'s and `/api/account*` is `auth`'s, both wired in `app/main.py`.
@@ -9,15 +10,17 @@ This is roledesk's own API, over the same store. Every refusal is
from __future__ import annotations
from datetime import timedelta
from typing import Annotated
from typing import Annotated, Any
from fastapi import APIRouter, HTTPException, Query, Response
from pydantic import BaseModel, Field
from app import jobs
from app.models import (
AFFINITY, Agent, Application, Company, Gate, ScoredRole, Status, Weights,
AFFINITY, Agent, Application, Company, Gate, ScoredRole, Status, Subscription, Weights,
)
from app.web.deps import StoreDep
from app.storage import Store
from app.web.deps import ConfigDep, StoreDep
router = APIRouter(prefix="/api", tags=["roledesk"])
@@ -87,6 +90,165 @@ def set_affinity(key: str, body: AffinityBody, store: StoreDep) -> Company:
raise HTTPException(404, f"no such company: {key}") from None
# ----------------------------------------------------------------- sources
class SourceOut(BaseModel):
"""A board or a saved search, with what it brings in: its open roles, and
the countries they are in, most first."""
id: int
source: str
target: str
label: str
provider: str
params: dict[str, Any]
on: bool
open_roles: int
countries: list[tuple[str, int]]
class SourcesOut(BaseModel):
searches: list[SourceOut]
boards: list[SourceOut]
#: Whether an adzuna search can run: a key in the config or the environment.
adzuna_key: bool
def _source_out(sub: Subscription, countries: dict[str, int]) -> SourceOut:
return SourceOut(
id=sub.id, source=sub.source, target=sub.target, label=sub.label,
provider=sub.board, params=sub.params, on=bool(sub.subscribed and sub.fetch),
open_roles=sum(countries.values()),
countries=sorted(countries.items(), key=lambda kv: -kv[1]))
def _one_source(store: Store, sub_id: int) -> SourceOut:
sub = store.subscription(sub_id)
if sub is None:
raise HTTPException(404, f"no such board or search: {sub_id}")
return _source_out(sub, store.open_countries_by_subscription().get(sub_id, {}))
@router.get("/sources")
def list_sources(store: StoreDep, config: ConfigDep) -> SourcesOut:
"""Every board, and every search not removed. A board switched off stays
listed, so it can be switched back on."""
counts = store.open_countries_by_subscription()
searches = [_source_out(s, counts.get(s.id, {}))
for s in store.subscriptions(source="search") if s.show]
boards = [_source_out(s, counts.get(s.id, {})) for s in store.subscriptions(source="ats")]
return SourcesOut(searches=searches, boards=boards,
adzuna_key=jobs.sources.has_key(config, "adzuna"))
class SwitchBody(BaseModel):
on: bool
@router.post("/sources/{sub_id}/switch")
def switch_source(sub_id: int, body: SwitchBody, store: StoreDep) -> dict[str, Any]:
"""On or off for the next pull. Off closes the open roles at once; on
reopens the ones the next pull still finds."""
if store.subscription(sub_id) is None:
raise HTTPException(404, f"no such board or search: {sub_id}")
_, closed = jobs.sources.set_fetch(store, sub_id, body.on)
return {"source": _one_source(store, sub_id), "closed": closed}
class SearchBody(BaseModel):
provider: str
name: str = ""
q: str = ""
area: int = 0
country: str = ""
where: str = ""
pages: Annotated[int, Field(ge=0, le=10)] = 0
def _params(body: SearchBody) -> dict[str, Any]:
try:
return jobs.sources.search_params(body.provider, body.q, area=body.area,
country=body.country, where=body.where,
pages=body.pages)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
@router.post("/searches/try")
def try_search(body: SearchBody, config: ConfigDep) -> dict[str, Any]:
"""Run a search once without saving it: how many it finds, whether that is
every match, and the first ten, so a query can be tuned before it is kept."""
try:
rows, complete = jobs.sources.try_search(config, body.provider, body.name,
_params(body))
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
sample = [{k: r.get(k) for k in ("title", "company", "place", "url")} for r in rows[:10]]
return {"found": len(rows), "complete": complete, "sample": sample}
@router.post("/searches")
def save_search(body: SearchBody, store: StoreDep, config: ConfigDep) -> dict[str, Any]:
"""Save a search, or update the one of that provider and name. It is run
once first and refused if it finds nothing."""
try:
sub, found, complete = jobs.sources.save_search(config, store, body.provider,
body.name, _params(body))
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
return {"source": _one_source(store, sub.id), "found": found, "complete": complete}
@router.delete("/searches/{sub_id}")
def remove_search(sub_id: int, store: StoreDep) -> dict[str, Any]:
try:
_, closed = jobs.sources.remove_search(store, sub_id)
except KeyError:
raise HTTPException(404, f"no such search: {sub_id}") from None
return {"closed": closed}
class BoardBody(BaseModel):
board: str
name: str
@router.post("/boards", status_code=201)
def add_board(body: BoardBody, store: StoreDep) -> dict[str, Any]:
"""Follow a company's ATS board. It is fetched first, and refused if it
does not answer or lists nothing."""
try:
sub, found = jobs.sources.add_board(store, body.board, body.name)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
return {"source": _one_source(store, sub.id), "found": found}
@router.get("/prune")
def prune_preview(store: StoreDep, region: str = "europe") -> dict[str, Any]:
"""The boards a prune would switch off: every one on, none of whose open
roles is in the region."""
try:
_, plan = jobs.sources.prune_plan(store, region)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
return {"region": region, "boards": [{"id": sub.id, "label": sub.label,
"target": sub.target, "open_roles": n}
for sub, n in plan]}
class PruneBody(BaseModel):
region: str = "europe"
@router.post("/prune")
def prune(body: PruneBody, store: StoreDep) -> dict[str, Any]:
try:
return jobs.sources.prune(store, body.region)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
# ----------------------------------------------------------- applications
class SaveBody(BaseModel):
+8
View File
@@ -17,6 +17,7 @@ from typing import Annotated
from fastapi import Depends
from starlette.requests import HTTPConnection
from app.config import Config
from app.storage import Store
@@ -25,3 +26,10 @@ def get_store(request: HTTPConnection) -> Store:
StoreDep = Annotated[Store, Depends(get_store)]
def get_config(request: HTTPConnection) -> Config:
return request.app.state.config
ConfigDep = Annotated[Config, Depends(get_config)]
+15 -3
View File
@@ -1,18 +1,21 @@
"""The tabs: eight pages, declared once.
"""The tabs: nine pages, declared once.
`TABS` is the whole navigation. `base.html` renders it, every page marks
itself active by its own `path`, and adding a tab is one entry here plus one
template.
Every page is a shell. Roles, Applications, Agents and Companies are filled
and driven by `static/roles.js` from the JSON routes in `web/api.py`; Jobs,
and driven by `static/roles.js` from the JSON routes in `web/api.py`, Sources
by `static/sources.js`; Jobs,
Schedules and Account are shells around the modules' own elements. What the
server renders is only what a select needs to offer, so the options cannot
drift from what the store accepts.
`FUNCTIONS` is `plugins.ats`'s, not `app`'s: the nine job functions are the
classifier's vocabulary, and this reaches `plugins/` the same way `app.cli`
does, since a plugin is app-agnostic and never reaches back into `app`.
does, since a plugin is app-agnostic and never reaches back into `app`. The
Jobindex areas and Adzuna countries on the Sources tab come from
`plugins.search` for the same reason.
"""
from __future__ import annotations
@@ -32,6 +35,7 @@ if str(_ROOT) not in sys.path:
sys.path.insert(0, str(_ROOT))
from plugins.ats import FUNCTIONS # noqa: E402
from plugins.search import ADZUNA_COUNTRIES, JOBINDEX_AREAS # noqa: E402
TEMPLATE_DIR = Path(__file__).resolve().parent.parent / "templates"
STATIC_DIR = Path(__file__).resolve().parent.parent / "static"
@@ -50,6 +54,7 @@ TABS: tuple[Tab, ...] = (
Tab("/applications", "Applications", "applications.html"),
Tab("/agents", "Agents", "agents.html"),
Tab("/companies", "Companies", "companies.html"),
Tab("/sources", "Sources", "sources.html"),
Tab("/jobs", "Jobs", "jobs.html"),
Tab("/schedules", "Schedules", "schedules.html"),
Tab("/messages", "Messages", "messages.html"),
@@ -61,6 +66,8 @@ TABS: tuple[Tab, ...] = (
FUNCTION_OPTIONS = [(f, f.replace("/", " / ").capitalize()) for f in FUNCTIONS]
REMOTE_OPTIONS = [(r, r.capitalize()) for r in REMOTE if r]
STATUS_OPTIONS = [(s, s.capitalize()) for s in STATUSES]
AREA_OPTIONS = list(JOBINDEX_AREAS.items())
COUNTRY_OPTIONS = [(c, c.upper()) for c in sorted(ADZUNA_COUNTRIES)]
def asset(path: str) -> str:
@@ -107,6 +114,11 @@ def companies(request: Request) -> HTMLResponse:
return page(request, "companies.html")
@router.get("/sources", response_class=HTMLResponse)
def sources(request: Request) -> HTMLResponse:
return page(request, "sources.html", areas=AREA_OPTIONS, countries=COUNTRY_OPTIONS)
@router.get("/jobs", response_class=HTMLResponse)
def jobs(request: Request) -> HTMLResponse:
return page(request, "jobs.html")
+5 -1
View File
@@ -48,7 +48,11 @@ What was decided for roledesk and the one reason for each. The longer reasoning
**Jobindex ads are read from the feed, not followed to the employer.** Each item links to Jobindex's own page, and its `/c?t=` redirect leads to the employer's site or ATS. Following it per role would count as a click in Jobindex's statistics and costs a request per role, so discovering a company's ATS board from a search result is left for later.
**`app companies prune` cuts the list by where it hires.** A board none of whose open roles is in the region (`--in`, Europe by default, or codes such as `DK,SE,EU`) is unsubscribed and its open roles closed, since a board no longer pulled would keep them open for good. A board whose roles name no country is kept, because it says nothing about where the company hires.
**`app companies prune` cuts the list by where it hires.** A board none of whose open roles is in the region (`--in`, Europe by default, or codes such as `DK,SE,EU`) is switched off. A board whose roles name no country is kept, because it says nothing about where the company hires.
**Switching a board or search off closes its open roles.** Only a pull that reads a target closes its roles, so a target no longer read would keep them open for good. Switched back on, the next pull reopens what it still lists. A batch already in flight when the switch happens reopens them again, so the pull's settle closes every switched-off target once more.
**The Sources tab and the CLI share `app.jobs.sources`.** Adding a board or a search, switching one, and pruning are one function each, called by `app companies`, `app searches` and `/api/sources*`, so the two cannot disagree about what "off" does. A search or board is run once before it is saved, and refused if it does not answer or finds nothing.
**SmartRecruiters' board list has no description.** Its `/postings` endpoint gives title, location and department; the job text is only on the per-posting endpoint, and one extra request per role would multiply a board's request count by its role count. `description` stays empty for this host.
+9 -1
View File
@@ -89,6 +89,14 @@ def _get(client: httpx.Client, url: str, params: dict[str, Any]) -> bytes:
JOBINDEX_RSS = "https://www.jobindex.dk/jobsoegning.rss"
JOBINDEX_PAGE = 20
#: Jobindex's `geoareaid` for the regions and the biggest towns, as checked
#: against the titles its feeds answer with. 0 is all of Denmark.
JOBINDEX_AREAS: dict[int, str] = {
0: "All of Denmark", 1: "Region Hovedstaden", 4: "Region Sjælland",
2: "Region Midtjylland", 5: "Region Syddanmark", 3: "Region Nordjylland",
8: "Aarhus", 7: "Aalborg",
}
_ALT = re.compile(r'\balt="([^"]+)"', re.IGNORECASE)
_H4 = re.compile(r"<h4>(.*?)</h4>", re.IGNORECASE | re.DOTALL)
_AREA = re.compile(r'class="jix_robotjob--area">(.*?)</span>', re.IGNORECASE | re.DOTALL)
@@ -112,7 +120,7 @@ def _remote(text: str) -> str | None:
def _text(fragment: str) -> str:
"""HTML to plain text. Jobindex breaks titles after a slash with a
zero-width space, which would otherwise end up in `title_key`."""
return html.unescape(strip_html(fragment)).replace("​", "").strip()
return html.unescape(strip_html(fragment)).replace("\u200b", "").strip()
def parse_jobindex(body: bytes) -> list[Row]:
+18
View File
@@ -260,3 +260,21 @@ def test_search_source_stamps_complete_and_ids(monkeypatch: pytest.MonkeyPatch)
assert items[0].raw["search"] == "jobindex:cph"
assert source.state["search:complete:jobindex:cph"] == "1"
assert source.state["search:ids:jobindex:cph"] == '["jobindex:h1"]'
def test_a_search_switched_off_mid_pull_ends_the_pull_closed(
store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
sub = store.add_subscription("search", "jobindex:cph", label="Copenhagen",
board="jobindex", params={"q": "backend"})
_stub_search(monkeypatch, ["h1"], complete=False)
batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type]
pull.run_batch(FakeConfig(), store, **batches[0]) # type: ignore[arg-type]
store.update_subscription(sub.id, subscribed=False, fetch=False)
class Tally:
results: ClassVar[list[dict[str, Any]]] = []
pull.close_boards(FakeConfig(), store, Tally()) # type: ignore[arg-type]
assert store.roles() == []
# And a batch that starts after the switch reads nothing.
assert pull.run_batch(FakeConfig(), store, **batches[0]) == {"roles": 0} # type: ignore[arg-type]
+127 -1
View File
@@ -202,7 +202,7 @@ def test_set_status_validates_and_writes_an_event(
def test_every_tab_renders_with_the_whole_nav(client: TestClient) -> None:
assert [t.label for t in TABS] == ["Roles", "Applications", "Agents", "Companies",
"Jobs", "Schedules", "Messages", "Account"]
"Sources", "Jobs", "Schedules", "Messages", "Account"]
for tab in TABS:
res = client.get(tab.path)
assert res.status_code == 200, f"{tab.path} answered {res.status_code}"
@@ -225,6 +225,14 @@ def test_each_roledesk_tab_is_a_shell_for_roles_js(client: TestClient) -> None:
assert client.get("/static/roles.js").status_code == 200
def test_the_sources_tab_is_a_shell_for_sources_js(client: TestClient) -> None:
page = client.get("/sources").text
assert 'id="searches-list"' in page and 'id="boards-list"' in page
assert page.index("app.js") < page.index("sources.js")
assert 'value="1" selected>Region Hovedstaden' in page # from plugins.search
assert client.get("/static/sources.js").status_code == 200
def test_the_modules_bring_their_own_elements(client: TestClient) -> None:
assert "<desk-jobs>" in client.get("/jobs").text
assert "<desk-schedules>" in client.get("/schedules").text
@@ -320,3 +328,121 @@ def test_roles_by_agent_are_ordered_by_score_and_hide_the_gated(
assert everything[-1]["gated_reason"] == "place unknown"
assert client.get("/api/roles").json()[0]["score"] is None
assert client.get("/api/roles?agent=999").status_code == 404
# ----------------------------------------------------------------- sources
def _fake_search(monkeypatch: pytest.MonkeyPatch, rows: int) -> None:
import plugins.search as search
def run(target: str, params: dict[str, object],
keys: dict[str, object] | None = None) -> tuple[list[dict[str, object]], bool]:
return [{"title": f"Engineer {n}", "company": "Acme", "place": "København",
"url": f"https://x.example/{n}"} for n in range(rows)], True
monkeypatch.setattr(search, "run_search", run)
def test_sources_lists_searches_and_boards_with_their_countries(
client: TestClient, store: Store) -> None:
board = store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
board="greenhouse")
store.upsert_role(posting("1", company="Acme", place="Copenhagen"), board)
store.upsert_role(posting("2", company="Acme", place="Boston, MA"), board)
store.upsert_role(posting("3", company="Acme", place="Aarhus"), board)
store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex",
params={"q": "backend", "area": 1})
res = client.get("/api/sources").json()
[row] = res["boards"]
assert (row["label"], row["on"], row["open_roles"]) == ("Acme", True, 3)
assert row["countries"] == [["DK", 2], ["US", 1]]
assert [s["label"] for s in res["searches"]] == ["Copenhagen"]
assert res["searches"][0]["params"] == {"q": "backend", "area": 1}
def test_switching_a_board_off_closes_its_roles_and_on_keeps_them_closed_until_a_pull(
client: TestClient, store: Store) -> None:
board = store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme",
board="greenhouse")
store.upsert_role(posting("1", company="Acme"), board)
off = client.post(f"/api/sources/{board.id}/switch", json={"on": False}).json()
assert off["closed"] == 1 and off["source"]["on"] is False
assert store.subscriptions(source="ats", fetchable=True) == []
assert store.roles() == []
on = client.post(f"/api/sources/{board.id}/switch", json={"on": True}).json()
assert on["closed"] == 0 and on["source"]["on"] is True
assert client.post("/api/sources/999/switch", json={"on": True}).status_code == 404
def test_a_search_is_tried_saved_updated_and_removed(
client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_fake_search(monkeypatch, 3)
body = {"provider": "jobindex", "name": "Copenhagen backend", "q": "backend", "area": 1}
tried = client.post("/api/searches/try", json=body).json()
assert tried["found"] == 3 and tried["complete"] is True
assert tried["sample"][0] == {"title": "Engineer 0", "company": "Acme",
"place": "København", "url": "https://x.example/0"}
assert store.subscriptions(source="search") == []
saved = client.post("/api/searches", json=body).json()
assert saved["source"]["target"] == "jobindex:copenhagen-backend"
assert saved["source"]["params"] == {"q": "backend", "area": 1}
updated = client.post("/api/searches", json={**body, "q": "python", "pages": 3}).json()
assert updated["source"]["id"] == saved["source"]["id"]
assert updated["source"]["params"] == {"q": "python", "area": 1, "pages": 3}
sub_id = saved["source"]["id"]
assert client.delete(f"/api/searches/{sub_id}").json() == {"closed": 0}
assert client.get("/api/sources").json()["searches"] == []
again = client.post("/api/searches", json=body).json() # the same row, back on
assert again["source"]["id"] == sub_id and again["source"]["on"] is True
def test_a_search_that_finds_nothing_or_is_malformed_is_refused(
client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
_fake_search(monkeypatch, 0)
res = client.post("/api/searches", json={"provider": "jobindex", "name": "X", "q": "x"})
assert res.status_code == 422 and "found nothing" in res.json()["detail"]
res = client.post("/api/searches", json={"provider": "indeed", "name": "X", "q": "x"})
assert res.status_code == 422
res = client.post("/api/searches/try", json={"provider": "jobindex", "q": " "})
assert res.status_code == 422 and "query" in res.json()["detail"]
assert store.subscriptions(source="search") == []
def test_a_board_is_followed_only_if_it_answers(
client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None:
import plugins.ats as ats
monkeypatch.setattr(ats, "fetch_board", lambda target: [{"source_id": "1"}])
res = client.post("/api/boards", json={"board": "ashby:acme", "name": "Acme"})
assert res.status_code == 201
assert res.json()["found"] == 1 and res.json()["source"]["target"] == "ashby:acme"
assert store.company("acme") is not None
again = client.post("/api/boards", json={"board": "ashby:acme", "name": "Acme"})
assert again.status_code == 422 and "already followed" in again.json()["detail"]
bad = client.post("/api/boards", json={"board": "acme", "name": "Other"})
assert bad.status_code == 422 and "host:slug" in bad.json()["detail"]
def test_prune_previews_then_switches_off(client: TestClient, store: Store) -> None:
us = store.add_subscription("ats", "greenhouse:us", label="US", company="US",
board="greenhouse")
dk = store.add_subscription("ats", "greenhouse:dk", label="DK", company="DK",
board="greenhouse")
store.upsert_role(posting("1", company="US", place="Boston, MA"), us)
store.upsert_role(posting("2", company="DK", place="Copenhagen"), dk)
preview = client.get("/api/prune", params={"region": "DK,EU"}).json()
assert [b["label"] for b in preview["boards"]] == ["US"]
assert len(store.subscriptions(source="ats", fetchable=True)) == 2
done = client.post("/api/prune", json={"region": "DK,EU"}).json()
assert (done["pruned"], done["roles_closed"], done["boards_left"]) == (1, 1, 1)
assert client.get("/api/prune", params={"region": " "}).status_code == 422