diff --git a/README.md b/README.md index 3a78774..a114d83 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,8 @@ uv run app serve # http://127.0.0.1:8790 ## Searches and companies +The Sources tab does all of this; the same actions exist on the command line: + ```bash uv run app searches add jobindex "Copenhagen golang" --q "golang go-udvikler" --area 1 uv run app searches add adzuna "Berlin backend" --q backend --country de --where Berlin @@ -24,7 +26,7 @@ uv run app searches list uv run app companies prune --in DK,SE,NO,EU # lists; add --apply to unsubscribe ``` -A Jobindex query is its own syntax: words are OR, `"a b"` is a phrase, `+word` is required; `--area 1` is Region Hovedstaden. An Adzuna search needs a free key from developer.adzuna.com in `ADZUNA_APP_ID` and `ADZUNA_APP_KEY`. `searches add` runs the search once and saves it only if it finds something. The `embeddings` extra is what `fit` needs, `agent` is for `extract` on a model, `passkeys` for the Account tab's passkeys. +A Jobindex query is its own syntax: words are OR, `"a b"` is a phrase, `+word` is required; `--area 1` is Region Hovedstaden. An Adzuna search needs a free key from developer.adzuna.com in `ADZUNA_APP_ID` and `ADZUNA_APP_KEY`. `searches add` runs the search once and saves it only if it finds something. Switching a board or search off closes its open roles; the next pull after switching it back on reopens the ones it still lists. The `embeddings` extra is what `fit` needs, `agent` is for `extract` on a model, `passkeys` for the Account tab's passkeys. ## The tabs @@ -34,6 +36,7 @@ A Jobindex query is its own syntax: words are OR, `"a b"` is a phrase, `+word` i | **Applications** | the saved roles and their status, each change recorded as an event | | **Agents** | the search agents: intent, require and prefer gates, weights. Saving one rescores it | | **Companies** | every company whose board is read, its open roles, and its affinity: mute, neutral, like, favourite | +| **Sources** | what a pull reads: saved searches (try, save, edit, remove), company boards (follow one, switch any on or off), and a prune of boards by region, with the countries each one's roles are in | | **Jobs** | a button per job (`pull`, `extract`, `embed`, `score`) and what is running | | **Schedules** | `pull` every 6h by default, editable in place | | **Messages** | everything the server said that nobody has cleared | diff --git a/app/cli.py b/app/cli.py index 84d32d6..87a7224 100644 --- a/app/cli.py +++ b/app/cli.py @@ -83,7 +83,7 @@ def build_parser() -> argparse.ArgumentParser: searches = sub.add_parser("searches", help="saved searches on cross-company job boards") searches_sub = searches.add_subparsers(dest="action", required=True) search_add = searches_sub.add_parser( - "add", help="run a search once, then save it for every pull") + "add", help="run a search once, then save it for every pull (or update it)") search_add.add_argument("provider", choices=["jobindex", "adzuna"]) search_add.add_argument("name", help='what to call it, e.g. "Copenhagen backend"') search_add.add_argument("--q", required=True, @@ -97,7 +97,7 @@ def build_parser() -> argparse.ArgumentParser: help="result pages per pull (default 5, at most 10)") searches_sub.add_parser("list", help="every saved search and its open roles") search_remove = searches_sub.add_parser( - "remove", help="stop running a search; its roles stay until they go stale") + "remove", help="stop running a search and close its open roles") search_remove.add_argument("name") db = sub.add_parser("db", help="database housekeeping") @@ -275,34 +275,15 @@ def add_company(store: Store, board: str, name: str) -> int: """`app companies add greenhouse:acme "Acme"`: fetch the board first and refuse if it does not answer, is not a known host, or the company already exists. Port of seedgen's `-add`.""" - from plugins.ats import careers_url as careers_url_of, fetch_board, is_known_target + from app import jobs from app.models import company_key - if not is_known_target(board): - print(f"companies add: {board!r} is not host:slug for a known ATS host", - file=sys.stderr) - return 1 - key = company_key(name) - if store.company(key) is not None: - print(f"companies add: {key!r} already exists", file=sys.stderr) - return 1 try: - rows = fetch_board(board) - except Exception as exc: - print(f"companies add: {board} did not answer: {exc}", file=sys.stderr) - return 1 - if not rows: - print(f"companies add: {board} answered but lists no roles; nothing added", - file=sys.stderr) - return 1 - host, _, _slug = board.partition(":") - store.add_company(name, key=key, careers_url=careers_url_of(board)) - try: - store.add_subscription("ats", board, label=name, company=name, board=host) + sub, found = jobs.sources.add_board(store, board, name) except ValueError as exc: print(f"companies add: {exc}", file=sys.stderr) return 1 - print(json.dumps({"key": key, "board": board, "roles": len(rows)})) + print(json.dumps({"key": company_key(sub.company), "board": sub.target, "roles": found})) return 0 @@ -325,38 +306,29 @@ def check_companies(store: Store) -> int: def prune_companies(store: Store, region: str, *, apply: bool) -> int: - """`app companies prune --in DK,EU`: every subscribed board none of whose - open roles is in the region, by `country`, which this fills from `place` - first where it is empty. `europe`, the default, is `EUROPE`. `--apply` unsubscribes those boards and closes their open roles, - since a board no longer pulled would otherwise keep them open for good. - `app companies seed` never brings one back: it skips a company that exists. + """`app companies prune --in DK,EU`: every fetched board none of whose open + roles is in the region, which is `europe` unless named. `--apply` + switches them off, which closes their open roles. `app companies seed` + never brings one back: it skips a company that exists. See + `app.jobs.sources.prune_plan` for which boards count.""" + from app import jobs - A board whose open roles all name no country is left alone: it says - nothing about where that company hires.""" - from app.models import EUROPE - - filled = store.backfill_country() - countries = store.open_countries_by_subscription() - wanted = {c.strip().upper() for c in region.split(",") if c.strip()} - if "EUROPE" in wanted: - wanted = (wanted - {"EUROPE"}) | set(EUROPE) - pruned = [(sub, sum(countries[sub.id].values())) - for sub in store.subscriptions(source="ats", fetchable=True) - if set(countries.get(sub.id, {})) - {""} - and not wanted.intersection(countries[sub.id])] - closed = 0 - for sub, count in sorted(pruned, key=lambda p: -p[1]): + try: + filled, plan = jobs.sources.prune_plan(store, region) + except ValueError as exc: + print(f"companies prune: {exc}", file=sys.stderr) + return 1 + for sub, count in plan: print(f"{sub.target:40} {sub.label:30} {count:6} open, none in {region}") - if apply: - store.unsubscribe(sub.id) - closed += store.close_subscription(sub.id) - kept = len(store.subscriptions(source="ats", fetchable=True)) - print(json.dumps({"countries_filled": filled, "pruned": len(pruned), "applied": apply, - "roles_closed": closed, "boards_left": kept})) + result: dict[str, Any] = {"countries_filled": filled, "pruned": len(plan), "applied": apply} + if apply: + result |= jobs.sources.prune(store, region) + print(json.dumps(result)) return 0 def run_searches(args: argparse.Namespace) -> int: + from app import jobs from app.storage import Store config = config_mod.load(args.config) @@ -368,12 +340,12 @@ def run_searches(args: argparse.Namespace) -> int: if sub is None: print(f"searches remove: no search named {args.name!r}", file=sys.stderr) return 1 - store.unsubscribe(sub.id) - print(json.dumps({"removed": sub.label})) + _, closed = jobs.sources.remove_search(store, sub.id) + print(json.dumps({"removed": sub.label, "roles_closed": closed})) return 0 countries = store.open_countries_by_subscription() for sub in store.subscriptions(source="search"): - removed = "" if sub.subscribed and sub.fetch else " (removed)" + removed = "" if sub.subscribed and sub.fetch else " (off)" open_roles = sum(countries.get(sub.id, {}).values()) print(f"{sub.label:30} {sub.target:40} {open_roles:5} open " f"{json.dumps(sub.params, ensure_ascii=False)}{removed}") @@ -382,36 +354,20 @@ def run_searches(args: argparse.Namespace) -> int: def add_search(config: config_mod.Config, store: Store, args: argparse.Namespace) -> int: """`app searches add jobindex "Copenhagen backend" --q '"backend developer"' - --area 1`: run the search once and refuse if it fails or finds nothing, so - a typo in a query is found now rather than by an empty tab later.""" - from app.jobs import search_target - from plugins.search import run_search + --area 1`: run the search once and save it only if it finds something. + A search of the same name is updated in place.""" + from app import jobs - params: dict[str, Any] = {"q": args.q} - if args.provider == "jobindex" and args.area: - params["area"] = args.area - if args.provider == "adzuna": - params["country"] = args.country.lower() - if args.where: - params["where"] = args.where - if args.pages: - params["pages"] = args.pages - target = search_target(args.provider, args.name) try: - rows, complete = run_search(target, params, dict(config.sources.get("search") or {})) - except Exception as exc: - print(f"searches add: {args.provider} did not answer: {exc}", file=sys.stderr) - return 1 - if not rows: - print("searches add: the search found nothing; not saved", file=sys.stderr) - return 1 - try: - store.add_subscription("search", target, label=args.name, board=args.provider, - params=params) + params = jobs.sources.search_params(args.provider, args.q, area=args.area, + country=args.country, where=args.where, + pages=args.pages) + sub, found, complete = jobs.sources.save_search(config, store, args.provider, args.name, + params) except ValueError as exc: print(f"searches add: {exc}", file=sys.stderr) return 1 - print(json.dumps({"search": args.name, "target": target, "roles": len(rows), + print(json.dumps({"search": sub.label, "target": sub.target, "roles": found, "complete": complete}, ensure_ascii=False)) return 0 diff --git a/app/jobs/__init__.py b/app/jobs/__init__.py index 14aaa60..b2118d6 100644 --- a/app/jobs/__init__.py +++ b/app/jobs/__init__.py @@ -7,13 +7,15 @@ task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`. score.py every open role scored for every enabled search agent cascade.py pull → extract → embed → score, what `TASKS` offers, the `pull` job a schedule fires, and that schedule + sources.py what a pull reads, changed by hand: boards and searches + added, switched on or off, pruned by region """ from __future__ import annotations from typing import Any -from app.jobs import cascade, embed, extract, pull, score +from app.jobs import cascade, embed, extract, pull, score, sources from app.jobs.pull import search_target, seed_searches from app.jobs.score import seed_agents @@ -33,4 +35,4 @@ def bootstrap(config_path: str = "") -> tuple[Any, Any]: __all__ = ["BOOTSTRAP", "bootstrap", "cascade", "embed", "extract", "pull", "score", - "search_target", "seed_agents", "seed_searches"] + "search_target", "seed_agents", "seed_searches", "sources"] diff --git a/app/jobs/pull.py b/app/jobs/pull.py index 950ee4f..8b90f21 100644 --- a/app/jobs/pull.py +++ b/app/jobs/pull.py @@ -88,7 +88,10 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str = from plugins.ats import ATSSource from plugins.search import SearchSource - subs = {s.target: s for s in store.subscriptions(source=source) if s.target in targets} + # Fetchable now, not when the pull was queued: a board switched off on the + # Sources tab since then is skipped. + subs = {s.target: s for s in store.subscriptions(source=source, fetchable=True) + if s.target in targets} options = dict(config.sources.get(source) or {}) plugin: ATSSource | SearchSource if source == "search": @@ -124,9 +127,16 @@ def close_boards(config: Config, store: Store, tally: Tally, *, `ats:ok:` was not written during *this* run — it failed, or its whole batch job never got to it — is never closed. A search closes by its list only when that list was complete, and otherwise by `STALE_DAYS`. + + A board or search switched off while its batch was in flight had its + roles reopened by that batch, after the switch closed them, so every + switched-off one is closed again here. """ answered = searched = failed = closed = 0 for source in SOURCES: + for sub in store.subscriptions(source=source): + if not (sub.subscribed and sub.fetch): + closed += store.close_subscription(sub.id) run_marker = store.state.get(f"{source}:run", "") for sub in store.subscriptions(source=source, fetchable=True): stamp = store.state.get(f"{source}:ok:{sub.target}", "") diff --git a/app/jobs/sources.py b/app/jobs/sources.py new file mode 100644 index 0000000..ef3456f --- /dev/null +++ b/app/jobs/sources.py @@ -0,0 +1,171 @@ +"""What a pull reads, changed by hand: add a board or a search, switch one on +or off, prune boards by region. `app companies`, `app searches` and the +Sources tab all call these, so the command line and the page cannot disagree +about what "off" means. + +A board or search switched off has its open roles closed at once. Nothing +else would ever close them, since only a pull that reads a target closes its +roles; switched back on, the next pull reopens every role it still lists. +""" + +from __future__ import annotations + +import os +from typing import Any + +from app.config import Config +from app.models import EUROPE, Subscription, company_key +from app.storage import Store +from app.jobs.pull import search_target + +#: The providers `plugins.search` runs, and which of them needs a key. +PROVIDERS = ("jobindex", "adzuna") + + +def region_codes(region: str) -> set[str]: + """`europe`, or comma-separated ISO codes such as `DK,SE,EU`, as a set of + codes. `europe` may be one entry among codes: `europe,US`.""" + codes = {c.strip().upper() for c in (region or "").split(",") if c.strip()} + if "EUROPE" in codes: + codes = (codes - {"EUROPE"}) | set(EUROPE) + if not codes: + raise ValueError("name a region: europe, or country codes such as DK,SE,EU") + return codes + + +def prune_plan(store: Store, region: str) -> tuple[int, list[tuple[Subscription, int]]]: + """Every fetched board none of whose open roles is in the region, with its + open role count, largest first; and how many roles had `country` filled + from `place` on the way. A board whose roles all name no country is kept: + it says nothing about where that company hires.""" + wanted = region_codes(region) + filled = store.backfill_country() + countries = store.open_countries_by_subscription() + plan = [(sub, sum(countries[sub.id].values())) + for sub in store.subscriptions(source="ats", fetchable=True) + if set(countries.get(sub.id, {})) - {""} + and not wanted.intersection(countries[sub.id])] + return filled, sorted(plan, key=lambda p: -p[1]) + + +def prune(store: Store, region: str) -> dict[str, Any]: + """Switch off every board `prune_plan` names.""" + filled, plan = prune_plan(store, region) + closed = sum(set_fetch(store, sub.id, False)[1] for sub, _ in plan) + return {"countries_filled": filled, "pruned": len(plan), "roles_closed": closed, + "boards_left": len(store.subscriptions(source="ats", fetchable=True))} + + +def set_fetch(store: Store, sub_id: int, fetch: bool) -> tuple[Subscription, int]: + """Switch one board or search on or off. Off closes its open roles and + returns how many; see the module docstring.""" + sub = store.update_subscription(sub_id, subscribed=fetch, fetch=fetch, show=True) + closed = 0 if fetch else store.close_subscription(sub_id) + return sub, closed + + +def add_board(store: Store, board: str, name: str) -> tuple[Subscription, int]: + """Follow one company's ATS board: fetch it first, and refuse if the host + is unknown, the company exists, the board does not answer or lists + nothing. Returns the subscription and how many roles the board lists.""" + from plugins.ats import careers_url, fetch_board, is_known_target + + board, name = board.strip(), name.strip() + if not name: + raise ValueError("a company needs a name") + if not is_known_target(board): + raise ValueError(f"{board!r} is not host:slug for a known ATS host, " + "e.g. greenhouse:acme or ashby:acme") + key = company_key(name) + if store.company(key) is not None: + raise ValueError(f"{name} is already followed") + try: + rows = fetch_board(board) + except Exception as exc: + raise ValueError(f"{board} did not answer: {exc}") from exc + if not rows: + raise ValueError(f"{board} answered but lists no roles; nothing added") + host = board.partition(":")[0] + store.add_company(name, key=key, careers_url=careers_url(board)) + sub = store.add_subscription("ats", board, label=name, company=name, board=host) + return sub, len(rows) + + +def search_params(provider: str, q: str, *, area: int = 0, country: str = "", + where: str = "", pages: int = 0) -> dict[str, Any]: + """The `params` a search subscription stores, with only the keys its + provider reads.""" + if provider not in PROVIDERS: + raise ValueError(f"provider is one of {', '.join(PROVIDERS)}") + if not q.strip(): + raise ValueError("a search needs a query") + params: dict[str, Any] = {"q": q.strip()} + if provider == "jobindex" and area: + params["area"] = int(area) + if provider == "adzuna": + params["country"] = country.strip().lower() + if where.strip(): + params["where"] = where.strip() + if pages: + params["pages"] = int(pages) + return params + + +def has_key(config: Config, provider: str) -> bool: + """Whether `provider` can run: jobindex always, adzuna with a key in the + config or the environment.""" + if provider != "adzuna": + return True + keys = (config.sources.get("search") or {}).get("adzuna") or {} + return bool((keys.get("app_id") or os.environ.get("ADZUNA_APP_ID")) + and (keys.get("app_key") or os.environ.get("ADZUNA_APP_KEY"))) + + +def try_search(config: Config, provider: str, name: str, + params: dict[str, Any]) -> tuple[list[dict[str, Any]], bool]: + """Run a search once, outside any pull. Raises ValueError with the + provider's own reason when it does not answer.""" + from plugins.search import run_search + + try: + return run_search(search_target(provider, name or "preview"), params, + dict(config.sources.get("search") or {})) + except ValueError: + raise + except Exception as exc: + raise ValueError(f"{provider} did not answer: {exc}") from exc + + +def save_search(config: Config, store: Store, provider: str, name: str, + params: dict[str, Any]) -> tuple[Subscription, int, bool]: + """Run a search once and save it only if it finds something, so a typo in a + query shows now and not as an empty tab later. A search of the same + provider and name is updated in place, removed or not: its roles keep + their subscription. Returns the subscription, how many roles it found, and + whether that was every match.""" + name = name.strip() + if not name: + raise ValueError("a search needs a name") + rows, complete = try_search(config, provider, name, params) + if not rows: + raise ValueError("the search found nothing; not saved") + target = search_target(provider, name) + existing = next((s for s in store.subscriptions(source="search") if s.target == target), + None) + if existing is None: + sub = store.add_subscription("search", target, label=name, board=provider, + params=params) + else: + sub = store.update_subscription(existing.id, label=name, params=params, + subscribed=True, fetch=True, show=True) + return sub, len(rows), complete + + +def remove_search(store: Store, sub_id: int) -> tuple[Subscription, int]: + """Stop running a search and hide it. Its open roles close; adding a search + of the same name again brings the row back.""" + sub = store.subscription(sub_id) + if sub is None or sub.source != "search": + raise KeyError(sub_id) + store.unsubscribe(sub_id) + return sub, store.close_subscription(sub_id) diff --git a/app/main.py b/app/main.py index ee112fd..920d894 100644 --- a/app/main.py +++ b/app/main.py @@ -46,6 +46,9 @@ def create_app(config: Config | None = None) -> FastAPI: store = Store(config.database) jobs.seed_agents(config, store) jobs.seed_searches(config, store) + # Roles pulled before `country` was read from `place` have none; after the + # first start only the places the lookup cannot place are left to read. + store.backfill_country() app = FastAPI(title="roledesk", docs_url="/api/docs", summary="ATS boards and job searches in, roles scored per search agent, " diff --git a/app/static/app.css b/app/static/app.css index bab1250..8ce18ca 100644 --- a/app/static/app.css +++ b/app/static/app.css @@ -352,6 +352,26 @@ textarea:focus-visible { .weights label { flex: 1 1 6rem; } .agent-actions { display: flex; gap: .5rem; flex-wrap: wrap; } +/* sources */ + +.section { margin: 1.5rem 0 2rem; } +.section > h2 { margin-bottom: .25rem; } +.source-form { margin-top: 1rem; } +.form-row { display: flex; gap: .75rem; flex-wrap: wrap; } +.form-row label { flex: 0 1 auto; } +.form-row label.grow { flex: 1 1 14rem; } +.source-form select { + background: var(--bg); border: 1px solid var(--line); + border-radius: .3rem; padding: .35rem .5rem; +} +.source-form input[type="number"] { max-width: 6rem; } +.source-form [hidden] { display: none; } +.sample { margin: 0; padding-left: 1.25rem; font-size: .9rem; } +.sample li { margin: .15rem 0; } +.mono { font-family: var(--mono); overflow-wrap: anywhere; } +.row-actions { display: inline-flex; gap: .35rem; } +.row-actions button { padding: .25rem .6rem; } + /* applications and companies */ table.list { width: 100%; border-collapse: collapse; background: var(--panel); } diff --git a/app/static/sources.js b/app/static/sources.js new file mode 100644 index 0000000..6b7f3e6 --- /dev/null +++ b/app/static/sources.js @@ -0,0 +1,366 @@ +/* The Sources tab: saved searches, company boards, and the prune by region. + * + * The template is a shell; the lists come from /api/sources and every action + * is a fetch through `api()` from app.js, so a refusal (a search that finds + * nothing, a board that does not answer) is one toast with the server's own + * sentence. The catch blocks here only put a control back. + */ +(function () { + "use strict"; + + const api = window.api; + const $ = (sel, root = document) => root.querySelector(sel); + + /** @type {{searches: object[], boards: object[], adzuna_key: boolean}} */ + let data = { searches: [], boards: [], adzuna_key: false }; + /** @type {object[]} the boards the last prune preview named */ + let pruneBoards = []; + + function count(n, one, many) { + return `${n} ${n === 1 ? one : many || `${one}s`}`; + } + + function debounce(fn, ms) { + let timer; + return (...args) => { + clearTimeout(timer); + timer = setTimeout(() => fn(...args), ms); + }; + } + + function cell(tr, label, content) { + const td = document.createElement("td"); + td.dataset.label = label; + if (typeof content === "string") td.textContent = content; + else td.appendChild(content); + tr.appendChild(td); + return td; + } + + /** "DK 120 · EU 4 · unknown 2", the three biggest. */ + function countries(source) { + const shown = source.countries.slice(0, 3) + .map(([code, n]) => `${code || "unknown"} ${n}`); + if (source.countries.length > 3) shown.push("…"); + return shown.join(" · "); + } + + function query(source) { + const p = source.params || {}; + const where = source.provider === "adzuna" + ? [p.country && p.country.toUpperCase(), p.where].filter(Boolean).join(", ") + : (p.area ? $(`#s-area option[value="${p.area}"]`)?.textContent : "All of Denmark"); + return `${p.q || ""}${where ? ` — ${where}` : ""}`; + } + + function onSwitch(source, rerender) { + const box = document.createElement("input"); + box.type = "checkbox"; + box.checked = source.on; + box.setAttribute("aria-label", `Pull ${source.label}`); + box.addEventListener("change", async () => { + box.disabled = true; + try { + const result = await api(`/api/sources/${source.id}/switch`, + { method: "POST", body: { on: box.checked } }); + Object.assign(source, result.source); + if (!box.checked && result.closed) { + window.toast.info(`${source.label}: ${count(result.closed, "open role")} closed.`); + } + } catch (err) { + box.checked = !box.checked; + } + box.disabled = false; + rerender(); + }); + return box; + } + + async function load() { + data = await api("/api/sources"); + } + + // ---------------------------------------------------------------- searches + + function renderSearches() { + const body = $("#searches-list"); + body.replaceChildren(); + for (const source of data.searches) { + const tr = document.createElement("tr"); + if (!source.on) tr.classList.add("muted"); + const name = document.createElement("span"); + name.textContent = source.label; + const provider = document.createElement("span"); + provider.className = "badge"; + provider.textContent = source.provider; + const title = document.createElement("span"); + title.append(name, " ", provider); + cell(tr, "Search", title); + cell(tr, "Query", query(source)).className = "mono small"; + cell(tr, "Open roles", String(source.open_roles)).className = "num"; + cell(tr, "Countries", countries(source)).className = "small"; + cell(tr, "On", onSwitch(source, renderSearches)); + const actions = document.createElement("span"); + actions.className = "row-actions"; + const edit = document.createElement("button"); + edit.type = "button"; + edit.className = "secondary"; + edit.textContent = "Edit"; + edit.addEventListener("click", () => fillSearchForm(source)); + const remove = document.createElement("button"); + remove.type = "button"; + remove.className = "secondary"; + remove.textContent = "Remove"; + remove.addEventListener("click", () => removeSearch(source)); + actions.append(edit, remove); + cell(tr, "", actions); + body.appendChild(tr); + } + const open = data.searches.reduce((sum, s) => sum + s.open_roles, 0); + $("#searches-count").textContent = data.searches.length + ? `${count(data.searches.length, "search", "searches")}, ${count(open, "open role")}` + : "No searches yet. Add one below; the starter searches in config.yaml import on the first pull."; + } + + function showProvider() { + const provider = $("#s-provider").value; + for (const node of document.querySelectorAll("#search-form [data-provider]")) { + node.hidden = node.dataset.provider !== provider; + } + $("#adzuna-key").hidden = provider !== "adzuna" || data.adzuna_key; + } + + function fillSearchForm(source) { + const p = (source && source.params) || {}; + $("#search-form-title").textContent = source ? `Edit ${source.label}` : "New search"; + $("#s-provider").value = source ? source.provider : "jobindex"; + $("#s-name").value = source ? source.label : ""; + $("#s-name").readOnly = Boolean(source); + $("#s-provider").disabled = Boolean(source); + $("#s-q").value = p.q || ""; + $("#s-area").value = String(p.area || (source ? 0 : 1)); + $("#s-country").value = p.country || "de"; + $("#s-where").value = p.where || ""; + $("#s-pages").value = String(p.pages || 10); + $("#search-status").textContent = ""; + $("#search-sample").replaceChildren(); + showProvider(); + if (source) $("#search-form").scrollIntoView({ behavior: "smooth", block: "start" }); + } + + function searchBody() { + return { + provider: $("#s-provider").value, + name: $("#s-name").value.trim(), + q: $("#s-q").value.trim(), + area: Number($("#s-area").value) || 0, + country: $("#s-country").value, + where: $("#s-where").value.trim(), + pages: Number($("#s-pages").value) || 0, + }; + } + + function found(result) { + const all = result.complete ? "every match" : "the page limit cut it short"; + return `Found ${count(result.found, "role")} (${all}).`; + } + + async function trySearch() { + const status = $("#search-status"); + const button = $("#s-try"); + button.disabled = true; + status.textContent = "Searching…"; + $("#search-sample").replaceChildren(); + try { + const result = await api("/api/searches/try", { method: "POST", body: searchBody() }); + status.textContent = `${found(result)} The newest ${Math.min(result.found, 10)}:`; + const list = $("#search-sample"); + for (const row of result.sample) { + const li = document.createElement("li"); + const link = document.createElement("a"); + link.href = row.url; + link.target = "_blank"; + link.rel = "noopener"; + link.textContent = row.title; + li.append(link, ` — ${[row.company, row.place].filter(Boolean).join(", ")}`); + list.appendChild(li); + } + } catch (err) { + status.textContent = ""; + } + button.disabled = false; + } + + async function saveSearch(event) { + event.preventDefault(); + const status = $("#search-status"); + const button = $("#search-form button[type=submit]"); + button.disabled = true; + status.textContent = "Running it once before saving…"; + try { + const result = await api("/api/searches", { method: "POST", body: searchBody() }); + await load(); + renderSearches(); + fillSearchForm(null); + status.textContent = `Saved ${result.source.label}. ${found(result)} ` + + "Its roles arrive with the next pull."; + } catch (err) { + status.textContent = ""; + } + button.disabled = false; + } + + async function removeSearch(source) { + if (!window.confirm(`Remove ${source.label}? Its ${count(source.open_roles, "open role")} close.`)) { + return; + } + try { + await api(`/api/searches/${source.id}`, { method: "DELETE" }); + data.searches = data.searches.filter((s) => s.id !== source.id); + renderSearches(); + } catch (err) { + /* already toasted */ + } + } + + // ------------------------------------------------------------------ boards + + function renderBoards() { + const q = $("#b-q").value.trim().toLowerCase(); + const on = $("#b-on").value; + const rows = data.boards + .filter((b) => !q || b.label.toLowerCase().includes(q) || b.target.includes(q)) + .filter((b) => !on || (on === "on") === b.on) + .sort((a, b) => Number(b.on) - Number(a.on) || b.open_roles - a.open_roles + || a.label.localeCompare(b.label)); + const body = $("#boards-list"); + body.replaceChildren(); + for (const board of rows) { + const tr = document.createElement("tr"); + if (!board.on) tr.classList.add("muted"); + const link = document.createElement("a"); + link.href = `/?company=${encodeURIComponent(board.label)}`; + link.title = "This company's open roles"; + link.textContent = board.label; + cell(tr, "Company", link); + cell(tr, "Board", board.target).className = "mono small"; + cell(tr, "Open roles", String(board.open_roles)).className = "num"; + cell(tr, "Countries", countries(board)).className = "small"; + cell(tr, "On", onSwitch(board, renderBoards)); + body.appendChild(tr); + } + const onCount = data.boards.filter((b) => b.on).length; + $("#boards-count").textContent = + `${count(data.boards.length, "board")}, ${onCount} on. Showing ${rows.length}.`; + } + + async function addBoard(event) { + event.preventDefault(); + const status = $("#board-status"); + const button = $("#board-form button[type=submit]"); + button.disabled = true; + status.textContent = "Reading the board once before following it…"; + try { + const result = await api("/api/boards", { + method: "POST", + body: { board: $("#b-board").value.trim(), name: $("#b-name").value.trim() }, + }); + data.boards.push(result.source); + renderBoards(); + $("#board-form").reset(); + status.textContent = `Following ${result.source.label}: ` + + `${count(result.found, "role")} listed. They arrive with the next pull.`; + } catch (err) { + status.textContent = ""; + } + button.disabled = false; + } + + // ------------------------------------------------------------------- prune + + async function previewPrune(event) { + event.preventDefault(); + const status = $("#prune-status"); + const list = $("#prune-list"); + const region = $("#p-region").value.trim(); + list.replaceChildren(); + $("#p-apply").disabled = true; + status.textContent = "Checking every board…"; + try { + const result = await api(`/api/prune?region=${encodeURIComponent(region)}`); + pruneBoards = result.boards; + const roles = pruneBoards.reduce((sum, b) => sum + b.open_roles, 0); + status.textContent = pruneBoards.length + ? `${count(pruneBoards.length, "board")} with no open role in ${region}, ` + + `${count(roles, "open role")} between them:` + : `Every board that is on has a role in ${region}.`; + for (const board of pruneBoards) { + const li = document.createElement("li"); + li.textContent = `${board.label} (${board.target}), ${count(board.open_roles, "open role")}`; + list.appendChild(li); + } + $("#p-apply").disabled = pruneBoards.length === 0; + } catch (err) { + status.textContent = ""; + } + } + + async function applyPrune() { + const region = $("#p-region").value.trim(); + if (!window.confirm(`Switch off ${count(pruneBoards.length, "board")} and close their roles? ` + + "Each can be switched back on in the list above.")) return; + $("#p-apply").disabled = true; + try { + const result = await api("/api/prune", { method: "POST", body: { region } }); + $("#prune-status").textContent = `Switched off ${count(result.pruned, "board")}, ` + + `closed ${count(result.roles_closed, "role")}. ${result.boards_left} boards are on.`; + $("#prune-list").replaceChildren(); + await load(); + renderBoards(); + } catch (err) { + $("#p-apply").disabled = false; + } + } + + // -------------------------------------------------------------------- pull + + async function pullNow() { + const button = $("#pull-now"); + button.disabled = true; + try { + await api("/api/jobs", { method: "POST", body: { kind: "pull", payload: {} } }); + $("#pull-status").replaceChildren("Pull started. "); + const link = document.createElement("a"); + link.href = "/jobs"; + link.textContent = "Follow it on the Jobs tab"; + $("#pull-status").appendChild(link); + } catch (err) { + /* already toasted */ + } + button.disabled = false; + } + + // -------------------------------------------------------------------- init + + document.addEventListener("DOMContentLoaded", async () => { + if (!$("#searches-list")) return; + try { + await load(); + } catch (err) { + return; + } + renderSearches(); + renderBoards(); + fillSearchForm(null); + $("#s-provider").addEventListener("change", showProvider); + $("#s-try").addEventListener("click", trySearch); + $("#search-form").addEventListener("submit", saveSearch); + $("#s-new").addEventListener("click", () => fillSearchForm(null)); + $("#b-q").addEventListener("input", debounce(renderBoards, 150)); + $("#b-on").addEventListener("change", renderBoards); + $("#board-form").addEventListener("submit", addBoard); + $("#prune-form").addEventListener("submit", previewPrune); + $("#p-apply").addEventListener("click", applyPrune); + $("#pull-now").addEventListener("click", pullNow); + }); +})(); diff --git a/app/templates/sources.html b/app/templates/sources.html new file mode 100644 index 0000000..f4fb6f7 --- /dev/null +++ b/app/templates/sources.html @@ -0,0 +1,107 @@ +{% extends "base.html" %} +{% block title %}Sources — roledesk{% endblock %} + +{% block content %} +
+

Sources

+ +
+

What a pull reads: saved searches, which find roles at any company, and company boards, which read every role one company lists. A pull runs on the Schedules tab's cadence. Switching one off closes its open roles at once; switched back on, the next pull reopens the ones it still finds.

+

+ +
+

Searches

+

+ + + + + +
SearchQueryOpen rolesCountriesOn
+ +
+

New search

+
+ + +
+ +
+ + + + +
+ +
+ + + +
+

+
    +
    +
    + +
    +

    Company boards

    +
    + + +
    +

    + + + + + +
    CompanyBoardOpen rolesCountriesOn
    + +
    +

    Follow a company

    +
    + + +
    +
    +

    +
    +
    + +
    +

    Prune boards by region

    +
    + +
    + + +
    +

    +
      +
      +
      +{% endblock %} + +{% block scripts %} + +{% endblock %} diff --git a/app/web/api.py b/app/web/api.py index b73fb6a..2fd5a08 100644 --- a/app/web/api.py +++ b/app/web/api.py @@ -1,4 +1,5 @@ -"""The JSON routes the tabs use: roles, companies, applications, agents. +"""The JSON routes the tabs use: roles, companies, sources, applications, +agents. `/api/jobs*`, `/api/schedules*`, `/api/notices` and `/ws/jobs` are `pipeline.web`'s and `/api/account*` is `auth`'s, both wired in `app/main.py`. @@ -9,15 +10,17 @@ This is roledesk's own API, over the same store. Every refusal is from __future__ import annotations from datetime import timedelta -from typing import Annotated +from typing import Annotated, Any from fastapi import APIRouter, HTTPException, Query, Response from pydantic import BaseModel, Field +from app import jobs from app.models import ( - AFFINITY, Agent, Application, Company, Gate, ScoredRole, Status, Weights, + AFFINITY, Agent, Application, Company, Gate, ScoredRole, Status, Subscription, Weights, ) -from app.web.deps import StoreDep +from app.storage import Store +from app.web.deps import ConfigDep, StoreDep router = APIRouter(prefix="/api", tags=["roledesk"]) @@ -87,6 +90,165 @@ def set_affinity(key: str, body: AffinityBody, store: StoreDep) -> Company: raise HTTPException(404, f"no such company: {key}") from None +# ----------------------------------------------------------------- sources + +class SourceOut(BaseModel): + """A board or a saved search, with what it brings in: its open roles, and + the countries they are in, most first.""" + + id: int + source: str + target: str + label: str + provider: str + params: dict[str, Any] + on: bool + open_roles: int + countries: list[tuple[str, int]] + + +class SourcesOut(BaseModel): + searches: list[SourceOut] + boards: list[SourceOut] + #: Whether an adzuna search can run: a key in the config or the environment. + adzuna_key: bool + + +def _source_out(sub: Subscription, countries: dict[str, int]) -> SourceOut: + return SourceOut( + id=sub.id, source=sub.source, target=sub.target, label=sub.label, + provider=sub.board, params=sub.params, on=bool(sub.subscribed and sub.fetch), + open_roles=sum(countries.values()), + countries=sorted(countries.items(), key=lambda kv: -kv[1])) + + +def _one_source(store: Store, sub_id: int) -> SourceOut: + sub = store.subscription(sub_id) + if sub is None: + raise HTTPException(404, f"no such board or search: {sub_id}") + return _source_out(sub, store.open_countries_by_subscription().get(sub_id, {})) + + +@router.get("/sources") +def list_sources(store: StoreDep, config: ConfigDep) -> SourcesOut: + """Every board, and every search not removed. A board switched off stays + listed, so it can be switched back on.""" + counts = store.open_countries_by_subscription() + searches = [_source_out(s, counts.get(s.id, {})) + for s in store.subscriptions(source="search") if s.show] + boards = [_source_out(s, counts.get(s.id, {})) for s in store.subscriptions(source="ats")] + return SourcesOut(searches=searches, boards=boards, + adzuna_key=jobs.sources.has_key(config, "adzuna")) + + +class SwitchBody(BaseModel): + on: bool + + +@router.post("/sources/{sub_id}/switch") +def switch_source(sub_id: int, body: SwitchBody, store: StoreDep) -> dict[str, Any]: + """On or off for the next pull. Off closes the open roles at once; on + reopens the ones the next pull still finds.""" + if store.subscription(sub_id) is None: + raise HTTPException(404, f"no such board or search: {sub_id}") + _, closed = jobs.sources.set_fetch(store, sub_id, body.on) + return {"source": _one_source(store, sub_id), "closed": closed} + + +class SearchBody(BaseModel): + provider: str + name: str = "" + q: str = "" + area: int = 0 + country: str = "" + where: str = "" + pages: Annotated[int, Field(ge=0, le=10)] = 0 + + +def _params(body: SearchBody) -> dict[str, Any]: + try: + return jobs.sources.search_params(body.provider, body.q, area=body.area, + country=body.country, where=body.where, + pages=body.pages) + except ValueError as exc: + raise HTTPException(422, str(exc)) from exc + + +@router.post("/searches/try") +def try_search(body: SearchBody, config: ConfigDep) -> dict[str, Any]: + """Run a search once without saving it: how many it finds, whether that is + every match, and the first ten, so a query can be tuned before it is kept.""" + try: + rows, complete = jobs.sources.try_search(config, body.provider, body.name, + _params(body)) + except ValueError as exc: + raise HTTPException(422, str(exc)) from exc + sample = [{k: r.get(k) for k in ("title", "company", "place", "url")} for r in rows[:10]] + return {"found": len(rows), "complete": complete, "sample": sample} + + +@router.post("/searches") +def save_search(body: SearchBody, store: StoreDep, config: ConfigDep) -> dict[str, Any]: + """Save a search, or update the one of that provider and name. It is run + once first and refused if it finds nothing.""" + try: + sub, found, complete = jobs.sources.save_search(config, store, body.provider, + body.name, _params(body)) + except ValueError as exc: + raise HTTPException(422, str(exc)) from exc + return {"source": _one_source(store, sub.id), "found": found, "complete": complete} + + +@router.delete("/searches/{sub_id}") +def remove_search(sub_id: int, store: StoreDep) -> dict[str, Any]: + try: + _, closed = jobs.sources.remove_search(store, sub_id) + except KeyError: + raise HTTPException(404, f"no such search: {sub_id}") from None + return {"closed": closed} + + +class BoardBody(BaseModel): + board: str + name: str + + +@router.post("/boards", status_code=201) +def add_board(body: BoardBody, store: StoreDep) -> dict[str, Any]: + """Follow a company's ATS board. It is fetched first, and refused if it + does not answer or lists nothing.""" + try: + sub, found = jobs.sources.add_board(store, body.board, body.name) + except ValueError as exc: + raise HTTPException(422, str(exc)) from exc + return {"source": _one_source(store, sub.id), "found": found} + + +@router.get("/prune") +def prune_preview(store: StoreDep, region: str = "europe") -> dict[str, Any]: + """The boards a prune would switch off: every one on, none of whose open + roles is in the region.""" + try: + _, plan = jobs.sources.prune_plan(store, region) + except ValueError as exc: + raise HTTPException(422, str(exc)) from exc + return {"region": region, "boards": [{"id": sub.id, "label": sub.label, + "target": sub.target, "open_roles": n} + for sub, n in plan]} + + +class PruneBody(BaseModel): + region: str = "europe" + + +@router.post("/prune") +def prune(body: PruneBody, store: StoreDep) -> dict[str, Any]: + try: + return jobs.sources.prune(store, body.region) + except ValueError as exc: + raise HTTPException(422, str(exc)) from exc + + # ----------------------------------------------------------- applications class SaveBody(BaseModel): diff --git a/app/web/deps.py b/app/web/deps.py index 7e0b2cd..2081443 100644 --- a/app/web/deps.py +++ b/app/web/deps.py @@ -17,6 +17,7 @@ from typing import Annotated from fastapi import Depends from starlette.requests import HTTPConnection +from app.config import Config from app.storage import Store @@ -25,3 +26,10 @@ def get_store(request: HTTPConnection) -> Store: StoreDep = Annotated[Store, Depends(get_store)] + + +def get_config(request: HTTPConnection) -> Config: + return request.app.state.config + + +ConfigDep = Annotated[Config, Depends(get_config)] diff --git a/app/web/pages.py b/app/web/pages.py index 6cce5e6..bcbcd2f 100644 --- a/app/web/pages.py +++ b/app/web/pages.py @@ -1,18 +1,21 @@ -"""The tabs: eight pages, declared once. +"""The tabs: nine pages, declared once. `TABS` is the whole navigation. `base.html` renders it, every page marks itself active by its own `path`, and adding a tab is one entry here plus one template. Every page is a shell. Roles, Applications, Agents and Companies are filled -and driven by `static/roles.js` from the JSON routes in `web/api.py`; Jobs, +and driven by `static/roles.js` from the JSON routes in `web/api.py`, Sources +by `static/sources.js`; Jobs, Schedules and Account are shells around the modules' own elements. What the server renders is only what a select needs to offer, so the options cannot drift from what the store accepts. `FUNCTIONS` is `plugins.ats`'s, not `app`'s: the nine job functions are the classifier's vocabulary, and this reaches `plugins/` the same way `app.cli` -does, since a plugin is app-agnostic and never reaches back into `app`. +does, since a plugin is app-agnostic and never reaches back into `app`. The +Jobindex areas and Adzuna countries on the Sources tab come from +`plugins.search` for the same reason. """ from __future__ import annotations @@ -32,6 +35,7 @@ if str(_ROOT) not in sys.path: sys.path.insert(0, str(_ROOT)) from plugins.ats import FUNCTIONS # noqa: E402 +from plugins.search import ADZUNA_COUNTRIES, JOBINDEX_AREAS # noqa: E402 TEMPLATE_DIR = Path(__file__).resolve().parent.parent / "templates" STATIC_DIR = Path(__file__).resolve().parent.parent / "static" @@ -50,6 +54,7 @@ TABS: tuple[Tab, ...] = ( Tab("/applications", "Applications", "applications.html"), Tab("/agents", "Agents", "agents.html"), Tab("/companies", "Companies", "companies.html"), + Tab("/sources", "Sources", "sources.html"), Tab("/jobs", "Jobs", "jobs.html"), Tab("/schedules", "Schedules", "schedules.html"), Tab("/messages", "Messages", "messages.html"), @@ -61,6 +66,8 @@ TABS: tuple[Tab, ...] = ( FUNCTION_OPTIONS = [(f, f.replace("/", " / ").capitalize()) for f in FUNCTIONS] REMOTE_OPTIONS = [(r, r.capitalize()) for r in REMOTE if r] STATUS_OPTIONS = [(s, s.capitalize()) for s in STATUSES] +AREA_OPTIONS = list(JOBINDEX_AREAS.items()) +COUNTRY_OPTIONS = [(c, c.upper()) for c in sorted(ADZUNA_COUNTRIES)] def asset(path: str) -> str: @@ -107,6 +114,11 @@ def companies(request: Request) -> HTMLResponse: return page(request, "companies.html") +@router.get("/sources", response_class=HTMLResponse) +def sources(request: Request) -> HTMLResponse: + return page(request, "sources.html", areas=AREA_OPTIONS, countries=COUNTRY_OPTIONS) + + @router.get("/jobs", response_class=HTMLResponse) def jobs(request: Request) -> HTMLResponse: return page(request, "jobs.html") diff --git a/docs/decisions.md b/docs/decisions.md index 86c1f7d..d834041 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -48,7 +48,11 @@ What was decided for roledesk and the one reason for each. The longer reasoning **Jobindex ads are read from the feed, not followed to the employer.** Each item links to Jobindex's own page, and its `/c?t=` redirect leads to the employer's site or ATS. Following it per role would count as a click in Jobindex's statistics and costs a request per role, so discovering a company's ATS board from a search result is left for later. -**`app companies prune` cuts the list by where it hires.** A board none of whose open roles is in the region (`--in`, Europe by default, or codes such as `DK,SE,EU`) is unsubscribed and its open roles closed, since a board no longer pulled would keep them open for good. A board whose roles name no country is kept, because it says nothing about where the company hires. +**`app companies prune` cuts the list by where it hires.** A board none of whose open roles is in the region (`--in`, Europe by default, or codes such as `DK,SE,EU`) is switched off. A board whose roles name no country is kept, because it says nothing about where the company hires. + +**Switching a board or search off closes its open roles.** Only a pull that reads a target closes its roles, so a target no longer read would keep them open for good. Switched back on, the next pull reopens what it still lists. A batch already in flight when the switch happens reopens them again, so the pull's settle closes every switched-off target once more. + +**The Sources tab and the CLI share `app.jobs.sources`.** Adding a board or a search, switching one, and pruning are one function each, called by `app companies`, `app searches` and `/api/sources*`, so the two cannot disagree about what "off" does. A search or board is run once before it is saved, and refused if it does not answer or finds nothing. **SmartRecruiters' board list has no description.** Its `/postings` endpoint gives title, location and department; the job text is only on the per-posting endpoint, and one extra request per role would multiply a board's request count by its role count. `description` stays empty for this host. diff --git a/plugins/search.py b/plugins/search.py index bce4af5..28fe253 100644 --- a/plugins/search.py +++ b/plugins/search.py @@ -89,6 +89,14 @@ def _get(client: httpx.Client, url: str, params: dict[str, Any]) -> bytes: JOBINDEX_RSS = "https://www.jobindex.dk/jobsoegning.rss" JOBINDEX_PAGE = 20 +#: Jobindex's `geoareaid` for the regions and the biggest towns, as checked +#: against the titles its feeds answer with. 0 is all of Denmark. +JOBINDEX_AREAS: dict[int, str] = { + 0: "All of Denmark", 1: "Region Hovedstaden", 4: "Region Sjælland", + 2: "Region Midtjylland", 5: "Region Syddanmark", 3: "Region Nordjylland", + 8: "Aarhus", 7: "Aalborg", +} + _ALT = re.compile(r'\balt="([^"]+)"', re.IGNORECASE) _H4 = re.compile(r"

      (.*?)

      ", re.IGNORECASE | re.DOTALL) _AREA = re.compile(r'class="jix_robotjob--area">(.*?)', re.IGNORECASE | re.DOTALL) @@ -112,7 +120,7 @@ def _remote(text: str) -> str | None: def _text(fragment: str) -> str: """HTML to plain text. Jobindex breaks titles after a slash with a zero-width space, which would otherwise end up in `title_key`.""" - return html.unescape(strip_html(fragment)).replace("​", "").strip() + return html.unescape(strip_html(fragment)).replace("\u200b", "").strip() def parse_jobindex(body: bytes) -> list[Row]: diff --git a/tests/test_search.py b/tests/test_search.py index 84dbeca..ef2690e 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -260,3 +260,21 @@ def test_search_source_stamps_complete_and_ids(monkeypatch: pytest.MonkeyPatch) assert items[0].raw["search"] == "jobindex:cph" assert source.state["search:complete:jobindex:cph"] == "1" assert source.state["search:ids:jobindex:cph"] == '["jobindex:h1"]' + + +def test_a_search_switched_off_mid_pull_ends_the_pull_closed( + store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + sub = store.add_subscription("search", "jobindex:cph", label="Copenhagen", + board="jobindex", params={"q": "backend"}) + _stub_search(monkeypatch, ["h1"], complete=False) + batches = pull._boards(FakeConfig(), store) # type: ignore[arg-type] + pull.run_batch(FakeConfig(), store, **batches[0]) # type: ignore[arg-type] + store.update_subscription(sub.id, subscribed=False, fetch=False) + + class Tally: + results: ClassVar[list[dict[str, Any]]] = [] + + pull.close_boards(FakeConfig(), store, Tally()) # type: ignore[arg-type] + assert store.roles() == [] + # And a batch that starts after the switch reads nothing. + assert pull.run_batch(FakeConfig(), store, **batches[0]) == {"roles": 0} # type: ignore[arg-type] diff --git a/tests/test_web.py b/tests/test_web.py index bbffbd0..5c69407 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -202,7 +202,7 @@ def test_set_status_validates_and_writes_an_event( def test_every_tab_renders_with_the_whole_nav(client: TestClient) -> None: assert [t.label for t in TABS] == ["Roles", "Applications", "Agents", "Companies", - "Jobs", "Schedules", "Messages", "Account"] + "Sources", "Jobs", "Schedules", "Messages", "Account"] for tab in TABS: res = client.get(tab.path) assert res.status_code == 200, f"{tab.path} answered {res.status_code}" @@ -225,6 +225,14 @@ def test_each_roledesk_tab_is_a_shell_for_roles_js(client: TestClient) -> None: assert client.get("/static/roles.js").status_code == 200 +def test_the_sources_tab_is_a_shell_for_sources_js(client: TestClient) -> None: + page = client.get("/sources").text + assert 'id="searches-list"' in page and 'id="boards-list"' in page + assert page.index("app.js") < page.index("sources.js") + assert 'value="1" selected>Region Hovedstaden' in page # from plugins.search + assert client.get("/static/sources.js").status_code == 200 + + def test_the_modules_bring_their_own_elements(client: TestClient) -> None: assert "" in client.get("/jobs").text assert "" in client.get("/schedules").text @@ -320,3 +328,121 @@ def test_roles_by_agent_are_ordered_by_score_and_hide_the_gated( assert everything[-1]["gated_reason"] == "place unknown" assert client.get("/api/roles").json()[0]["score"] is None assert client.get("/api/roles?agent=999").status_code == 404 + + +# ----------------------------------------------------------------- sources + +def _fake_search(monkeypatch: pytest.MonkeyPatch, rows: int) -> None: + import plugins.search as search + + def run(target: str, params: dict[str, object], + keys: dict[str, object] | None = None) -> tuple[list[dict[str, object]], bool]: + return [{"title": f"Engineer {n}", "company": "Acme", "place": "København", + "url": f"https://x.example/{n}"} for n in range(rows)], True + + monkeypatch.setattr(search, "run_search", run) + + +def test_sources_lists_searches_and_boards_with_their_countries( + client: TestClient, store: Store) -> None: + board = store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme", + board="greenhouse") + store.upsert_role(posting("1", company="Acme", place="Copenhagen"), board) + store.upsert_role(posting("2", company="Acme", place="Boston, MA"), board) + store.upsert_role(posting("3", company="Acme", place="Aarhus"), board) + store.add_subscription("search", "jobindex:cph", label="Copenhagen", board="jobindex", + params={"q": "backend", "area": 1}) + + res = client.get("/api/sources").json() + [row] = res["boards"] + assert (row["label"], row["on"], row["open_roles"]) == ("Acme", True, 3) + assert row["countries"] == [["DK", 2], ["US", 1]] + assert [s["label"] for s in res["searches"]] == ["Copenhagen"] + assert res["searches"][0]["params"] == {"q": "backend", "area": 1} + + +def test_switching_a_board_off_closes_its_roles_and_on_keeps_them_closed_until_a_pull( + client: TestClient, store: Store) -> None: + board = store.add_subscription("ats", "greenhouse:acme", label="Acme", company="Acme", + board="greenhouse") + store.upsert_role(posting("1", company="Acme"), board) + + off = client.post(f"/api/sources/{board.id}/switch", json={"on": False}).json() + assert off["closed"] == 1 and off["source"]["on"] is False + assert store.subscriptions(source="ats", fetchable=True) == [] + assert store.roles() == [] + + on = client.post(f"/api/sources/{board.id}/switch", json={"on": True}).json() + assert on["closed"] == 0 and on["source"]["on"] is True + assert client.post("/api/sources/999/switch", json={"on": True}).status_code == 404 + + +def test_a_search_is_tried_saved_updated_and_removed( + client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + _fake_search(monkeypatch, 3) + body = {"provider": "jobindex", "name": "Copenhagen backend", "q": "backend", "area": 1} + + tried = client.post("/api/searches/try", json=body).json() + assert tried["found"] == 3 and tried["complete"] is True + assert tried["sample"][0] == {"title": "Engineer 0", "company": "Acme", + "place": "København", "url": "https://x.example/0"} + assert store.subscriptions(source="search") == [] + + saved = client.post("/api/searches", json=body).json() + assert saved["source"]["target"] == "jobindex:copenhagen-backend" + assert saved["source"]["params"] == {"q": "backend", "area": 1} + + updated = client.post("/api/searches", json={**body, "q": "python", "pages": 3}).json() + assert updated["source"]["id"] == saved["source"]["id"] + assert updated["source"]["params"] == {"q": "python", "area": 1, "pages": 3} + + sub_id = saved["source"]["id"] + assert client.delete(f"/api/searches/{sub_id}").json() == {"closed": 0} + assert client.get("/api/sources").json()["searches"] == [] + again = client.post("/api/searches", json=body).json() # the same row, back on + assert again["source"]["id"] == sub_id and again["source"]["on"] is True + + +def test_a_search_that_finds_nothing_or_is_malformed_is_refused( + client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + _fake_search(monkeypatch, 0) + res = client.post("/api/searches", json={"provider": "jobindex", "name": "X", "q": "x"}) + assert res.status_code == 422 and "found nothing" in res.json()["detail"] + res = client.post("/api/searches", json={"provider": "indeed", "name": "X", "q": "x"}) + assert res.status_code == 422 + res = client.post("/api/searches/try", json={"provider": "jobindex", "q": " "}) + assert res.status_code == 422 and "query" in res.json()["detail"] + assert store.subscriptions(source="search") == [] + + +def test_a_board_is_followed_only_if_it_answers( + client: TestClient, store: Store, monkeypatch: pytest.MonkeyPatch) -> None: + import plugins.ats as ats + + monkeypatch.setattr(ats, "fetch_board", lambda target: [{"source_id": "1"}]) + res = client.post("/api/boards", json={"board": "ashby:acme", "name": "Acme"}) + assert res.status_code == 201 + assert res.json()["found"] == 1 and res.json()["source"]["target"] == "ashby:acme" + assert store.company("acme") is not None + + again = client.post("/api/boards", json={"board": "ashby:acme", "name": "Acme"}) + assert again.status_code == 422 and "already followed" in again.json()["detail"] + bad = client.post("/api/boards", json={"board": "acme", "name": "Other"}) + assert bad.status_code == 422 and "host:slug" in bad.json()["detail"] + + +def test_prune_previews_then_switches_off(client: TestClient, store: Store) -> None: + us = store.add_subscription("ats", "greenhouse:us", label="US", company="US", + board="greenhouse") + dk = store.add_subscription("ats", "greenhouse:dk", label="DK", company="DK", + board="greenhouse") + store.upsert_role(posting("1", company="US", place="Boston, MA"), us) + store.upsert_role(posting("2", company="DK", place="Copenhagen"), dk) + + preview = client.get("/api/prune", params={"region": "DK,EU"}).json() + assert [b["label"] for b in preview["boards"]] == ["US"] + assert len(store.subscriptions(source="ats", fetchable=True)) == 2 + + done = client.post("/api/prune", json={"region": "DK,EU"}).json() + assert (done["pruned"], done["roles_closed"], done["boards_left"]) == (1, 1, 1) + assert client.get("/api/prune", params={"region": " "}).status_code == 422