Sources tab: searches, company boards and the prune by region, in the app

What a pull reads can now be changed on a page instead of only the command
line: try a search and see its first ten results before saving it, edit or
remove one, follow a company's ATS board, switch any board or search on or
off, and preview and apply a prune by region. Each row shows its open roles
and the countries they are in.

The page and `app companies`/`app searches` call the same functions in
app/jobs/sources.py. Switching a target off closes its open roles, and the
pull's settle closes them again if a batch in flight reopened them. Roles
pulled before `country` existed are backfilled at startup.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-24 15:15:04 +02:00
1 parent 047f7cd8ad
commit 6b40702a9a
16 files changed
+1068 -92

No files matched your search

+166 -4
View File
@@ -1,4 +1,5 @@
"""The JSON routes the tabs use: roles, companies, applications, agents.
"""The JSON routes the tabs use: roles, companies, sources, applications,
agents.
`/api/jobs*`, `/api/schedules*`, `/api/notices` and `/ws/jobs` are
`pipeline.web`'s and `/api/account*` is `auth`'s, both wired in `app/main.py`.
@@ -9,15 +10,17 @@ This is roledesk's own API, over the same store. Every refusal is
from __future__ import annotations
from datetime import timedelta
from typing import Annotated
from typing import Annotated, Any
from fastapi import APIRouter, HTTPException, Query, Response
from pydantic import BaseModel, Field
from app import jobs
from app.models import (
AFFINITY, Agent, Application, Company, Gate, ScoredRole, Status, Weights,
AFFINITY, Agent, Application, Company, Gate, ScoredRole, Status, Subscription, Weights,
)
from app.web.deps import StoreDep
from app.storage import Store
from app.web.deps import ConfigDep, StoreDep
router = APIRouter(prefix="/api", tags=["roledesk"])
@@ -87,6 +90,165 @@ def set_affinity(key: str, body: AffinityBody, store: StoreDep) -> Company:
raise HTTPException(404, f"no such company: {key}") from None
# ----------------------------------------------------------------- sources
class SourceOut(BaseModel):
"""A board or a saved search, with what it brings in: its open roles, and
the countries they are in, most first."""
id: int
source: str
target: str
label: str
provider: str
params: dict[str, Any]
on: bool
open_roles: int
countries: list[tuple[str, int]]
class SourcesOut(BaseModel):
searches: list[SourceOut]
boards: list[SourceOut]
#: Whether an adzuna search can run: a key in the config or the environment.
adzuna_key: bool
def _source_out(sub: Subscription, countries: dict[str, int]) -> SourceOut:
return SourceOut(
id=sub.id, source=sub.source, target=sub.target, label=sub.label,
provider=sub.board, params=sub.params, on=bool(sub.subscribed and sub.fetch),
open_roles=sum(countries.values()),
countries=sorted(countries.items(), key=lambda kv: -kv[1]))
def _one_source(store: Store, sub_id: int) -> SourceOut:
sub = store.subscription(sub_id)
if sub is None:
raise HTTPException(404, f"no such board or search: {sub_id}")
return _source_out(sub, store.open_countries_by_subscription().get(sub_id, {}))
@router.get("/sources")
def list_sources(store: StoreDep, config: ConfigDep) -> SourcesOut:
"""Every board, and every search not removed. A board switched off stays
listed, so it can be switched back on."""
counts = store.open_countries_by_subscription()
searches = [_source_out(s, counts.get(s.id, {}))
for s in store.subscriptions(source="search") if s.show]
boards = [_source_out(s, counts.get(s.id, {})) for s in store.subscriptions(source="ats")]
return SourcesOut(searches=searches, boards=boards,
adzuna_key=jobs.sources.has_key(config, "adzuna"))
class SwitchBody(BaseModel):
on: bool
@router.post("/sources/{sub_id}/switch")
def switch_source(sub_id: int, body: SwitchBody, store: StoreDep) -> dict[str, Any]:
"""On or off for the next pull. Off closes the open roles at once; on
reopens the ones the next pull still finds."""
if store.subscription(sub_id) is None:
raise HTTPException(404, f"no such board or search: {sub_id}")
_, closed = jobs.sources.set_fetch(store, sub_id, body.on)
return {"source": _one_source(store, sub_id), "closed": closed}
class SearchBody(BaseModel):
provider: str
name: str = ""
q: str = ""
area: int = 0
country: str = ""
where: str = ""
pages: Annotated[int, Field(ge=0, le=10)] = 0
def _params(body: SearchBody) -> dict[str, Any]:
try:
return jobs.sources.search_params(body.provider, body.q, area=body.area,
country=body.country, where=body.where,
pages=body.pages)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
@router.post("/searches/try")
def try_search(body: SearchBody, config: ConfigDep) -> dict[str, Any]:
"""Run a search once without saving it: how many it finds, whether that is
every match, and the first ten, so a query can be tuned before it is kept."""
try:
rows, complete = jobs.sources.try_search(config, body.provider, body.name,
_params(body))
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
sample = [{k: r.get(k) for k in ("title", "company", "place", "url")} for r in rows[:10]]
return {"found": len(rows), "complete": complete, "sample": sample}
@router.post("/searches")
def save_search(body: SearchBody, store: StoreDep, config: ConfigDep) -> dict[str, Any]:
"""Save a search, or update the one of that provider and name. It is run
once first and refused if it finds nothing."""
try:
sub, found, complete = jobs.sources.save_search(config, store, body.provider,
body.name, _params(body))
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
return {"source": _one_source(store, sub.id), "found": found, "complete": complete}
@router.delete("/searches/{sub_id}")
def remove_search(sub_id: int, store: StoreDep) -> dict[str, Any]:
try:
_, closed = jobs.sources.remove_search(store, sub_id)
except KeyError:
raise HTTPException(404, f"no such search: {sub_id}") from None
return {"closed": closed}
class BoardBody(BaseModel):
board: str
name: str
@router.post("/boards", status_code=201)
def add_board(body: BoardBody, store: StoreDep) -> dict[str, Any]:
"""Follow a company's ATS board. It is fetched first, and refused if it
does not answer or lists nothing."""
try:
sub, found = jobs.sources.add_board(store, body.board, body.name)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
return {"source": _one_source(store, sub.id), "found": found}
@router.get("/prune")
def prune_preview(store: StoreDep, region: str = "europe") -> dict[str, Any]:
"""The boards a prune would switch off: every one on, none of whose open
roles is in the region."""
try:
_, plan = jobs.sources.prune_plan(store, region)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
return {"region": region, "boards": [{"id": sub.id, "label": sub.label,
"target": sub.target, "open_roles": n}
for sub, n in plan]}
class PruneBody(BaseModel):
region: str = "europe"
@router.post("/prune")
def prune(body: PruneBody, store: StoreDep) -> dict[str, Any]:
try:
return jobs.sources.prune(store, body.region)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
# ----------------------------------------------------------- applications
class SaveBody(BaseModel):