diff --git a/app/cli.py b/app/cli.py index 3b09ba6..b7d3db1 100644 --- a/app/cli.py +++ b/app/cli.py @@ -3,7 +3,7 @@ app serve [--host H] [--port P] [--reload] app job pull|extract|embed|score [--payload JSON] app worker [--workers http=8,cpu=1] [--until-idle] - app companies seed|add|edit|checked|remove|by-hand|set-board|detach|discover|check + app companies seed|add|edit|checked|remove|by-hand|set-board|detach|discover|check|size app searches add|list|remove app agents list|prompt app stages [extract|summarise on|off] @@ -32,6 +32,7 @@ from typing import TYPE_CHECKING, Any from app import config as config_mod if TYPE_CHECKING: + from app.models import Subscription from app.storage import Store #: The one plugin this app ships lives outside the `app` package, in @@ -118,6 +119,10 @@ def build_parser() -> argparse.ArgumentParser: help="add or update the company: its board when one is readable, " "else the host it runs on") companies_sub.add_parser("check", help="report boards that do not answer, now or in the pull") + size = companies_sub.add_parser( + "size", help="size a company's board again, or --missing every board without a size") + size.add_argument("key", nargs="?", default="", help="the company key, e.g. infineon") + size.add_argument("--missing", action="store_true", help="every board with no size yet") searches = sub.add_parser("searches", help="saved searches on cross-company job boards") searches_sub = searches.add_subparsers(dest="action", required=True) @@ -335,6 +340,8 @@ def run_companies(args: argparse.Namespace) -> int: return detach_board(store, args.key) if args.action == "discover": return discover_company(store, args.name, args.domain, add=args.add) + if args.action == "size": + return size_boards(store, args.key, missing=args.missing) return check_companies(store) @@ -503,6 +510,43 @@ def detach_board(store: Store, key: str) -> int: return 0 +def size_boards(store: Store, key: str, *, missing: bool) -> int: + """`app companies size infineon`: size its board again and keep it, as + the Companies row's Recheck does, and print the size and what a pull asks + the board for. `--missing` sizes every board shown that has no size yet, + a line each, and exits 1 if any did not answer. The next pull reads a + board so; nothing is queued from here.""" + from app import jobs + + if bool(key) == missing: + print("companies size: name a company key, or --missing", file=sys.stderr) + return 2 + if missing: + subs = [s for s in store.subscriptions(source="ats") if s.show] + known = jobs.reach.sizes(store, subs) + subs = [s for s in subs if s.target not in known] + else: + sub = store.company_subscription(key) + if sub is None or not sub.show: + print(f"companies size: {key!r} has no board", file=sys.stderr) + return 1 + subs = [sub] + sized: list[tuple[Subscription, int]] = [] + for sub in subs: + try: + sized.append((sub, jobs.sources.size_board(store, sub.target))) + except ValueError as exc: + print(f"companies size: {exc}", file=sys.stderr) + plan = jobs.reach.plan(store, [sub for sub, _size in sized]) + for sub, size in sized: + print(json.dumps({"board": sub.target, "size": size, + "reach": jobs.reach.line(plan, sub)})) + dead = len(subs) - len(sized) + if missing: + print(f"sized {len(sized)} of {len(subs)} boards without a size") + return 1 if dead else 0 + + def check_companies(store: Store) -> int: """`app companies check`: every `ats` board, fetched once. Prints a line per board that did not answer now, answered incompletely, or has diff --git a/app/static/companies.js b/app/static/companies.js index 4fe012c..f4653ea 100644 --- a/app/static/companies.js +++ b/app/static/companies.js @@ -28,6 +28,11 @@ * posted to `/api/sources/{id}/reach`. The wide reach (Europe) and no role * narrowing are the company's two overrides instead, and the board's own * countries win over All countries. + * + * Beside that line stands the board's size, set when it was added and + * changed only by a Recheck ("999 roles · checked 30 Sep"). Recheck, beside + * Change, sizes the board again (`/api/sources/{id}/size`) and queues a + * pull, since the size decides whether the board is filtered. */ (function () { "use strict"; @@ -67,6 +72,41 @@ // ---------------------------------------------------------------- the list + /** "30 Sep", or "30 Sep 2025" when it is not this year. A day alone + ("2026-09-30") is read as local midnight, not UTC's. */ + function checkedOn(iso) { + const text = String(iso || ""); + const at = new Date(text.length === 10 ? `${text}T00:00` : text); + if (Number.isNaN(at.getTime())) return ""; + const options = { day: "numeric", month: "short" }; + if (at.getFullYear() !== new Date().getFullYear()) options.year = "numeric"; + return at.toLocaleDateString("en-GB", options); + } + + /** The row's Recheck: size the board again, which may change what it is + asked for, and read it with the pull that follows. */ + function recheckButton(company, rerender) { + const button = document.createElement("button"); + button.type = "button"; + button.className = "linky"; + button.textContent = "Recheck"; + button.title = "Count the board's roles again"; + button.addEventListener("click", async () => { + button.disabled = true; + try { + const result = await api(`/api/sources/${company.subscription_id}/size`, + { method: "POST" }); + window.toast.info(`${company.name}: ${count(result.size, "role")}, ` + + `reads ${result.reach}. Reading now, or with the pull right after.`); + await loadCompanies(agentId()); + rerender(); + } catch (err) { + button.disabled = false; + } + }); + return button; + } + function subLine(company, rerender) { const sub = document.createElement("span"); sub.className = "sub"; @@ -104,6 +144,10 @@ } sub.append(` · ${count(company.open_roles, "open role")}`); if (company.reach) sub.append(` · reads ${company.reach}`); + if (company.size != null) { + const checked = checkedOn(company.size_checked); + sub.append(` · ${count(company.size, "role")}${checked ? ` · checked ${checked}` : ""}`); + } if (company.failing_since) { // The board has failed on every pull since then; after 21 days its // roles close. @@ -115,6 +159,10 @@ sub.appendChild(failing); } sub.append(" "); + if (company.subscription_id != null) { + sub.appendChild(recheckButton(company, rerender)); + sub.append(" "); + } const edit = document.createElement("button"); edit.type = "button"; edit.className = "linky"; diff --git a/app/web/api.py b/app/web/api.py index e6706fb..edb764d 100644 --- a/app/web/api.py +++ b/app/web/api.py @@ -491,6 +491,10 @@ class CompanyOut(BaseModel): #: what a pull asks its board for (`jobs.reach.line`). reach_params: dict[str, Any] = {} reach: str = "" + #: The roles the board listed when it was last sized, and that day + #: (ISO); None and "" before it is sized (`jobs.reach.SIZE`). + size: int | None = None + size_checked: str = "" def _company_boards(store: Store) -> dict[str, tuple[int, str, bool, str]]: @@ -534,7 +538,10 @@ def _company_outs(store: Store, agent: int | None) -> list[CompanyOut]: all_roles=bool(c.all_roles), reach_params={k: v for k, v in (sub.params if sub else {}).items() if k in jobs.reach.OVERRIDE_KEYS}, - reach=jobs.reach.line(plan, sub) if sub else "")) + reach=jobs.reach.line(plan, sub) if sub else "", + size=plan.sizes.get(target) if sub else None, + size_checked=store.state.get( + jobs.reach.SIZE_CHECKED + target, "") if sub else "")) return out @@ -897,6 +904,25 @@ def set_reach(sub_id: int, body: ReachBody, store: StoreDep) -> dict[str, Any]: "reach": jobs.reach.line(jobs.reach.plan(store, [sub]), sub)} +@router.post("/sources/{sub_id}/size") +def recheck_size(sub_id: int, store: StoreDep, queue: QueueDep) -> dict[str, Any]: + """Size a board again, the one way its size changes once it was added + (`jobs.sources.size_board`): its reach may change with it, so a pull is + queued right after, as the other routes that change what is read do. 422 + when the board does not answer; the size it had stays.""" + sub = store.subscription(sub_id) + if sub is None or sub.source != "ats": + raise HTTPException(404, f"no such board: {sub_id}") + try: + size = jobs.sources.size_board(store, sub.target) + except ValueError as exc: + raise HTTPException(422, str(exc)) from exc + queue.enqueue("pull") + checked = store.state.get(jobs.reach.SIZE_CHECKED + sub.target, "") + return {"size": size, "size_checked": checked, + "reach": jobs.reach.line(jobs.reach.plan(store, [sub]), sub)} + + class ReaderOut(BaseModel): """What a reader declares (`plugins/readers/_contract.py`), for the Sources page: `filters` holds only the dimensions it filters.""" diff --git a/tests/test_board_size.py b/tests/test_board_size.py index 329b46c..a522f53 100644 --- a/tests/test_board_size.py +++ b/tests/test_board_size.py @@ -1,5 +1,5 @@ """A board's size, set once: `ats.board_size` per kind of host, `set_board` -that keeps it, the pull that sizes a board +and the Recheck (route and CLI) that keep it, the pull that sizes a board with none before it reads it, and the Companies line that names what a site's own ids leave out. No network: `ats._get` or the runner's listing is stubbed.""" @@ -8,19 +8,25 @@ from __future__ import annotations import json import sys +from collections.abc import Iterator from datetime import UTC, datetime from pathlib import Path from typing import Any, ClassVar from urllib.parse import parse_qs, urlsplit import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient ROOT = Path(__file__).resolve().parents[1] if str(ROOT) not in sys.path: sys.path.insert(0, str(ROOT)) from pipeline import Tally # noqa: E402 +from app.cli import main # noqa: E402 +from app.config import Config, ServerConfig # noqa: E402 from app.jobs import pull, reach, sources # noqa: E402 +from app.main import create_app # noqa: E402 from app.models import Gate # noqa: E402 from app.storage import Store # noqa: E402 from plugins import ats # noqa: E402 @@ -341,3 +347,82 @@ def test_a_boards_own_reach_is_named_the_same_and_remote_alone_is_read_whole( "set here: DK · NO not on this site · all roles" assert reach.line(plan, _sub(store, "workday:other.wd3/Jobs")) == \ "all places, no remote option on this site · all roles" + + +# ------------------------------------------------------ Recheck and the CLI + +@pytest.fixture +def app(tmp_path: Path) -> FastAPI: + return create_app(Config(server=ServerConfig(host="127.0.0.1", port=0), + database=tmp_path / "roledesk.db")) + + +@pytest.fixture +def client(app: FastAPI) -> Iterator[TestClient]: + with TestClient(app) as c: + yield c + + +def test_recheck_sizes_the_board_again_and_queues_a_pull( + app: FastAPI, client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: + store: Store = app.state.store + _wolf(store) + _board(store, BOSCH, "Bosch", size=120) + store.state[f"ats:sizechecked:{BOSCH}"] = "2026-09-01" + sub_id = _sub(store, BOSCH).id + [row] = client.get("/api/companies").json() + assert (row["size"], row["size_checked"], row["reach"]) == ( + 120, "2026-09-01", "under 300 roles: read whole") + + monkeypatch.setattr(ats, "_get", SmartRecruiters(4830)) + res = client.post(f"/api/sources/{sub_id}/size") + assert res.status_code == 200, res.text + assert res.json() == {"size": 4830, "size_checked": TODAY, + "reach": "follows agents: DK, SE · all roles"} + assert app.state.queue.jobs.last("pull", states=("queued", "running", "done")) is not None + [row] = client.get("/api/companies").json() + assert (row["size"], row["size_checked"]) == (4830, TODAY) + + # A board that does not answer keeps the size it had. + monkeypatch.setattr(ats, "_get", SmartRecruiters(5, down=True)) + res = client.post(f"/api/sources/{sub_id}/size") + assert res.status_code == 422 and "did not answer" in res.json()["detail"] + assert store.state[f"ats:size:{BOSCH}"] == "4830" + assert client.post("/api/sources/9999/size").status_code == 404 + + +def _cli(tmp_path: Path, *argv: str) -> int: + config = tmp_path / "config.yaml" + if not config.exists(): + config.write_text(f"database: {tmp_path / 't.db'}\n") + return main(["-c", str(config), "companies", *argv]) + + +def test_the_cli_sizes_one_board_or_every_board_without_a_size( + store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str]) -> None: + _wolf(store) + _board(store, BOSCH, "Bosch", size=12) + _board(store, "smartrecruiters:tiny", "Tiny") + _board(store, "smartrecruiters:gone", "Gone") + monkeypatch.setattr(ats, "_get", SmartRecruiters(4830)) + assert _cli(tmp_path, "size", "bosch") == 0 + assert json.loads(capsys.readouterr().out) == { + "board": BOSCH, "size": 4830, "reach": "follows agents: DK, SE · all roles"} + + def some(client: Any, url: str, host: str, slug: str, **kwargs: Any) -> bytes: + if slug == "gone": + raise OSError("connection refused") + return SmartRecruiters(40)(client, url, host, slug) + + monkeypatch.setattr(ats, "_get", some) + assert _cli(tmp_path, "size", "--missing") == 1 + out, err = capsys.readouterr() + assert out.splitlines() == [ + json.dumps({"board": "smartrecruiters:tiny", "size": 40, + "reach": "under 300 roles: read whole"}), + "sized 1 of 2 boards without a size"] + assert "smartrecruiters:gone did not answer" in err + assert store.state[f"ats:size:{BOSCH}"] == "4830" + assert _cli(tmp_path, "size") == 2 + assert _cli(tmp_path, "size", "nobody") == 1 diff --git a/tests/test_web.py b/tests/test_web.py index 0d7d0b3..04f6f43 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -603,7 +603,8 @@ def test_companies_lists_board_and_open_role_count( "careers_opened_at": "", "failing_since": "", "failing_reason": "", "from_search": False, "hand_reason": "", "by_hand": False, "all_countries": False, "all_roles": False, - "reach_params": {}, "reach": "size not known yet: sized on the next pull"} + "reach_params": {}, "reach": "size not known yet: sized on the next pull", + "size": None, "size_checked": ""} [other] = [c for c in res.json() if c["key"] == "other"] assert (other["board"], other["subscription_id"], other["on"], other["ats"]) == ( "", None, None, "") @@ -1453,7 +1454,7 @@ def test_a_company_is_added_by_hand_with_a_reason_and_a_link( "country": "", "city": "", "watch": None, "careers_opened_at": "", "failing_since": "", "failing_reason": "", "from_search": False, "hand_reason": "added", "by_hand": True, "all_countries": False, "all_roles": False, - "reach_params": {}, "reach": ""} + "reach_params": {}, "reach": "", "size": None, "size_checked": ""} assert store.company("revolut").hand_reason == "added" # type: ignore[union-attr] other = client.post("/api/companies", json={"name": "Meta", "hand_reason": "blocked"})