Sources tab: searches, company boards and the prune by region, in the app

What a pull reads can now be changed on a page instead of only the command
line: try a search and see its first ten results before saving it, edit or
remove one, follow a company's ATS board, switch any board or search on or
off, and preview and apply a prune by region. Each row shows its open roles
and the countries they are in.

The page and `app companies`/`app searches` call the same functions in
app/jobs/sources.py. Switching a target off closes its open roles, and the
pull's settle closes them again if a batch in flight reopened them. Roles
pulled before `country` existed are backfilled at startup.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-24 15:15:04 +02:00
1 parent 047f7cd8ad
commit 6b40702a9a
16 files changed
+1068 -92

No files matched your search

+166 -4
View File
@@ -1,4 +1,5 @@
"""The JSON routes the tabs use: roles, companies, applications, agents.
"""The JSON routes the tabs use: roles, companies, sources, applications,
agents.
`/api/jobs*`, `/api/schedules*`, `/api/notices` and `/ws/jobs` are
`pipeline.web`'s and `/api/account*` is `auth`'s, both wired in `app/main.py`.
@@ -9,15 +10,17 @@ This is roledesk's own API, over the same store. Every refusal is
from __future__ import annotations
from datetime import timedelta
from typing import Annotated
from typing import Annotated, Any
from fastapi import APIRouter, HTTPException, Query, Response
from pydantic import BaseModel, Field
from app import jobs
from app.models import (
AFFINITY, Agent, Application, Company, Gate, ScoredRole, Status, Weights,
AFFINITY, Agent, Application, Company, Gate, ScoredRole, Status, Subscription, Weights,
)
from app.web.deps import StoreDep
from app.storage import Store
from app.web.deps import ConfigDep, StoreDep
router = APIRouter(prefix="/api", tags=["roledesk"])
@@ -87,6 +90,165 @@ def set_affinity(key: str, body: AffinityBody, store: StoreDep) -> Company:
raise HTTPException(404, f"no such company: {key}") from None
# ----------------------------------------------------------------- sources
class SourceOut(BaseModel):
"""A board or a saved search, with what it brings in: its open roles, and
the countries they are in, most first."""
id: int
source: str
target: str
label: str
provider: str
params: dict[str, Any]
on: bool
open_roles: int
countries: list[tuple[str, int]]
class SourcesOut(BaseModel):
searches: list[SourceOut]
boards: list[SourceOut]
#: Whether an adzuna search can run: a key in the config or the environment.
adzuna_key: bool
def _source_out(sub: Subscription, countries: dict[str, int]) -> SourceOut:
return SourceOut(
id=sub.id, source=sub.source, target=sub.target, label=sub.label,
provider=sub.board, params=sub.params, on=bool(sub.subscribed and sub.fetch),
open_roles=sum(countries.values()),
countries=sorted(countries.items(), key=lambda kv: -kv[1]))
def _one_source(store: Store, sub_id: int) -> SourceOut:
sub = store.subscription(sub_id)
if sub is None:
raise HTTPException(404, f"no such board or search: {sub_id}")
return _source_out(sub, store.open_countries_by_subscription().get(sub_id, {}))
@router.get("/sources")
def list_sources(store: StoreDep, config: ConfigDep) -> SourcesOut:
"""Every board, and every search not removed. A board switched off stays
listed, so it can be switched back on."""
counts = store.open_countries_by_subscription()
searches = [_source_out(s, counts.get(s.id, {}))
for s in store.subscriptions(source="search") if s.show]
boards = [_source_out(s, counts.get(s.id, {})) for s in store.subscriptions(source="ats")]
return SourcesOut(searches=searches, boards=boards,
adzuna_key=jobs.sources.has_key(config, "adzuna"))
class SwitchBody(BaseModel):
on: bool
@router.post("/sources/{sub_id}/switch")
def switch_source(sub_id: int, body: SwitchBody, store: StoreDep) -> dict[str, Any]:
"""On or off for the next pull. Off closes the open roles at once; on
reopens the ones the next pull still finds."""
if store.subscription(sub_id) is None:
raise HTTPException(404, f"no such board or search: {sub_id}")
_, closed = jobs.sources.set_fetch(store, sub_id, body.on)
return {"source": _one_source(store, sub_id), "closed": closed}
class SearchBody(BaseModel):
provider: str
name: str = ""
q: str = ""
area: int = 0
country: str = ""
where: str = ""
pages: Annotated[int, Field(ge=0, le=10)] = 0
def _params(body: SearchBody) -> dict[str, Any]:
try:
return jobs.sources.search_params(body.provider, body.q, area=body.area,
country=body.country, where=body.where,
pages=body.pages)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
@router.post("/searches/try")
def try_search(body: SearchBody, config: ConfigDep) -> dict[str, Any]:
"""Run a search once without saving it: how many it finds, whether that is
every match, and the first ten, so a query can be tuned before it is kept."""
try:
rows, complete = jobs.sources.try_search(config, body.provider, body.name,
_params(body))
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
sample = [{k: r.get(k) for k in ("title", "company", "place", "url")} for r in rows[:10]]
return {"found": len(rows), "complete": complete, "sample": sample}
@router.post("/searches")
def save_search(body: SearchBody, store: StoreDep, config: ConfigDep) -> dict[str, Any]:
"""Save a search, or update the one of that provider and name. It is run
once first and refused if it finds nothing."""
try:
sub, found, complete = jobs.sources.save_search(config, store, body.provider,
body.name, _params(body))
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
return {"source": _one_source(store, sub.id), "found": found, "complete": complete}
@router.delete("/searches/{sub_id}")
def remove_search(sub_id: int, store: StoreDep) -> dict[str, Any]:
try:
_, closed = jobs.sources.remove_search(store, sub_id)
except KeyError:
raise HTTPException(404, f"no such search: {sub_id}") from None
return {"closed": closed}
class BoardBody(BaseModel):
board: str
name: str
@router.post("/boards", status_code=201)
def add_board(body: BoardBody, store: StoreDep) -> dict[str, Any]:
"""Follow a company's ATS board. It is fetched first, and refused if it
does not answer or lists nothing."""
try:
sub, found = jobs.sources.add_board(store, body.board, body.name)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
return {"source": _one_source(store, sub.id), "found": found}
@router.get("/prune")
def prune_preview(store: StoreDep, region: str = "europe") -> dict[str, Any]:
"""The boards a prune would switch off: every one on, none of whose open
roles is in the region."""
try:
_, plan = jobs.sources.prune_plan(store, region)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
return {"region": region, "boards": [{"id": sub.id, "label": sub.label,
"target": sub.target, "open_roles": n}
for sub, n in plan]}
class PruneBody(BaseModel):
region: str = "europe"
@router.post("/prune")
def prune(body: PruneBody, store: StoreDep) -> dict[str, Any]:
try:
return jobs.sources.prune(store, body.region)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
# ----------------------------------------------------------- applications
class SaveBody(BaseModel):
+8
View File
@@ -17,6 +17,7 @@ from typing import Annotated
from fastapi import Depends
from starlette.requests import HTTPConnection
from app.config import Config
from app.storage import Store
@@ -25,3 +26,10 @@ def get_store(request: HTTPConnection) -> Store:
StoreDep = Annotated[Store, Depends(get_store)]
def get_config(request: HTTPConnection) -> Config:
return request.app.state.config
ConfigDep = Annotated[Config, Depends(get_config)]
+15 -3
View File
@@ -1,18 +1,21 @@
"""The tabs: eight pages, declared once.
"""The tabs: nine pages, declared once.
`TABS` is the whole navigation. `base.html` renders it, every page marks
itself active by its own `path`, and adding a tab is one entry here plus one
template.
Every page is a shell. Roles, Applications, Agents and Companies are filled
and driven by `static/roles.js` from the JSON routes in `web/api.py`; Jobs,
and driven by `static/roles.js` from the JSON routes in `web/api.py`, Sources
by `static/sources.js`; Jobs,
Schedules and Account are shells around the modules' own elements. What the
server renders is only what a select needs to offer, so the options cannot
drift from what the store accepts.
`FUNCTIONS` is `plugins.ats`'s, not `app`'s: the nine job functions are the
classifier's vocabulary, and this reaches `plugins/` the same way `app.cli`
does, since a plugin is app-agnostic and never reaches back into `app`.
does, since a plugin is app-agnostic and never reaches back into `app`. The
Jobindex areas and Adzuna countries on the Sources tab come from
`plugins.search` for the same reason.
"""
from __future__ import annotations
@@ -32,6 +35,7 @@ if str(_ROOT) not in sys.path:
sys.path.insert(0, str(_ROOT))
from plugins.ats import FUNCTIONS # noqa: E402
from plugins.search import ADZUNA_COUNTRIES, JOBINDEX_AREAS # noqa: E402
TEMPLATE_DIR = Path(__file__).resolve().parent.parent / "templates"
STATIC_DIR = Path(__file__).resolve().parent.parent / "static"
@@ -50,6 +54,7 @@ TABS: tuple[Tab, ...] = (
Tab("/applications", "Applications", "applications.html"),
Tab("/agents", "Agents", "agents.html"),
Tab("/companies", "Companies", "companies.html"),
Tab("/sources", "Sources", "sources.html"),
Tab("/jobs", "Jobs", "jobs.html"),
Tab("/schedules", "Schedules", "schedules.html"),
Tab("/messages", "Messages", "messages.html"),
@@ -61,6 +66,8 @@ TABS: tuple[Tab, ...] = (
FUNCTION_OPTIONS = [(f, f.replace("/", " / ").capitalize()) for f in FUNCTIONS]
REMOTE_OPTIONS = [(r, r.capitalize()) for r in REMOTE if r]
STATUS_OPTIONS = [(s, s.capitalize()) for s in STATUSES]
AREA_OPTIONS = list(JOBINDEX_AREAS.items())
COUNTRY_OPTIONS = [(c, c.upper()) for c in sorted(ADZUNA_COUNTRIES)]
def asset(path: str) -> str:
@@ -107,6 +114,11 @@ def companies(request: Request) -> HTMLResponse:
return page(request, "companies.html")
@router.get("/sources", response_class=HTMLResponse)
def sources(request: Request) -> HTMLResponse:
return page(request, "sources.html", areas=AREA_OPTIONS, countries=COUNTRY_OPTIONS)
@router.get("/jobs", response_class=HTMLResponse)
def jobs(request: Request) -> HTMLResponse:
return page(request, "jobs.html")