Recheck a board's size from the Companies row, the route and app companies size

POST /api/sources/{id}/size sizes the board again, keeps the day and queues a pull; /api/companies returns size and size_checked, and the row shows "999 roles · checked 30 Sep" with Recheck beside Change. app companies size <key> does the same from the CLI, and --missing sizes every board without a size.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 13:44:54 +02:00
1 parent a0882e9659
commit 6085017943
5 files changed
+209 -5

No files matched your search

+45 -1
View File
@@ -3,7 +3,7 @@
app serve [--host H] [--port P] [--reload] app serve [--host H] [--port P] [--reload]
app job pull|extract|embed|score [--payload JSON] app job pull|extract|embed|score [--payload JSON]
app worker [--workers http=8,cpu=1] [--until-idle] app worker [--workers http=8,cpu=1] [--until-idle]
app companies seed|add|edit|checked|remove|by-hand|set-board|detach|discover|check app companies seed|add|edit|checked|remove|by-hand|set-board|detach|discover|check|size
app searches add|list|remove app searches add|list|remove
app agents list|prompt app agents list|prompt
app stages [extract|summarise on|off] app stages [extract|summarise on|off]
@@ -32,6 +32,7 @@ from typing import TYPE_CHECKING, Any
from app import config as config_mod from app import config as config_mod
if TYPE_CHECKING: if TYPE_CHECKING:
from app.models import Subscription
from app.storage import Store from app.storage import Store
#: The one plugin this app ships lives outside the `app` package, in #: The one plugin this app ships lives outside the `app` package, in
@@ -118,6 +119,10 @@ def build_parser() -> argparse.ArgumentParser:
help="add or update the company: its board when one is readable, " help="add or update the company: its board when one is readable, "
"else the host it runs on") "else the host it runs on")
companies_sub.add_parser("check", help="report boards that do not answer, now or in the pull") companies_sub.add_parser("check", help="report boards that do not answer, now or in the pull")
size = companies_sub.add_parser(
"size", help="size a company's board again, or --missing every board without a size")
size.add_argument("key", nargs="?", default="", help="the company key, e.g. infineon")
size.add_argument("--missing", action="store_true", help="every board with no size yet")
searches = sub.add_parser("searches", help="saved searches on cross-company job boards") searches = sub.add_parser("searches", help="saved searches on cross-company job boards")
searches_sub = searches.add_subparsers(dest="action", required=True) searches_sub = searches.add_subparsers(dest="action", required=True)
@@ -335,6 +340,8 @@ def run_companies(args: argparse.Namespace) -> int:
return detach_board(store, args.key) return detach_board(store, args.key)
if args.action == "discover": if args.action == "discover":
return discover_company(store, args.name, args.domain, add=args.add) return discover_company(store, args.name, args.domain, add=args.add)
if args.action == "size":
return size_boards(store, args.key, missing=args.missing)
return check_companies(store) return check_companies(store)
@@ -503,6 +510,43 @@ def detach_board(store: Store, key: str) -> int:
return 0 return 0
def size_boards(store: Store, key: str, *, missing: bool) -> int:
"""`app companies size infineon`: size its board again and keep it, as
the Companies row's Recheck does, and print the size and what a pull asks
the board for. `--missing` sizes every board shown that has no size yet,
a line each, and exits 1 if any did not answer. The next pull reads a
board so; nothing is queued from here."""
from app import jobs
if bool(key) == missing:
print("companies size: name a company key, or --missing", file=sys.stderr)
return 2
if missing:
subs = [s for s in store.subscriptions(source="ats") if s.show]
known = jobs.reach.sizes(store, subs)
subs = [s for s in subs if s.target not in known]
else:
sub = store.company_subscription(key)
if sub is None or not sub.show:
print(f"companies size: {key!r} has no board", file=sys.stderr)
return 1
subs = [sub]
sized: list[tuple[Subscription, int]] = []
for sub in subs:
try:
sized.append((sub, jobs.sources.size_board(store, sub.target)))
except ValueError as exc:
print(f"companies size: {exc}", file=sys.stderr)
plan = jobs.reach.plan(store, [sub for sub, _size in sized])
for sub, size in sized:
print(json.dumps({"board": sub.target, "size": size,
"reach": jobs.reach.line(plan, sub)}))
dead = len(subs) - len(sized)
if missing:
print(f"sized {len(sized)} of {len(subs)} boards without a size")
return 1 if dead else 0
def check_companies(store: Store) -> int: def check_companies(store: Store) -> int:
"""`app companies check`: every `ats` board, fetched once. Prints a """`app companies check`: every `ats` board, fetched once. Prints a
line per board that did not answer now, answered incompletely, or has line per board that did not answer now, answered incompletely, or has
+48
View File
@@ -28,6 +28,11 @@
* posted to `/api/sources/{id}/reach`. The wide reach (Europe) and no role * posted to `/api/sources/{id}/reach`. The wide reach (Europe) and no role
* narrowing are the company's two overrides instead, and the board's own * narrowing are the company's two overrides instead, and the board's own
* countries win over All countries. * countries win over All countries.
*
* Beside that line stands the board's size, set when it was added and
* changed only by a Recheck ("999 roles · checked 30 Sep"). Recheck, beside
* Change, sizes the board again (`/api/sources/{id}/size`) and queues a
* pull, since the size decides whether the board is filtered.
*/ */
(function () { (function () {
"use strict"; "use strict";
@@ -67,6 +72,41 @@
// ---------------------------------------------------------------- the list // ---------------------------------------------------------------- the list
/** "30 Sep", or "30 Sep 2025" when it is not this year. A day alone
("2026-09-30") is read as local midnight, not UTC's. */
function checkedOn(iso) {
const text = String(iso || "");
const at = new Date(text.length === 10 ? `${text}T00:00` : text);
if (Number.isNaN(at.getTime())) return "";
const options = { day: "numeric", month: "short" };
if (at.getFullYear() !== new Date().getFullYear()) options.year = "numeric";
return at.toLocaleDateString("en-GB", options);
}
/** The row's Recheck: size the board again, which may change what it is
asked for, and read it with the pull that follows. */
function recheckButton(company, rerender) {
const button = document.createElement("button");
button.type = "button";
button.className = "linky";
button.textContent = "Recheck";
button.title = "Count the board's roles again";
button.addEventListener("click", async () => {
button.disabled = true;
try {
const result = await api(`/api/sources/${company.subscription_id}/size`,
{ method: "POST" });
window.toast.info(`${company.name}: ${count(result.size, "role")}, `
+ `reads ${result.reach}. Reading now, or with the pull right after.`);
await loadCompanies(agentId());
rerender();
} catch (err) {
button.disabled = false;
}
});
return button;
}
function subLine(company, rerender) { function subLine(company, rerender) {
const sub = document.createElement("span"); const sub = document.createElement("span");
sub.className = "sub"; sub.className = "sub";
@@ -104,6 +144,10 @@
} }
sub.append(` · ${count(company.open_roles, "open role")}`); sub.append(` · ${count(company.open_roles, "open role")}`);
if (company.reach) sub.append(` · reads ${company.reach}`); if (company.reach) sub.append(` · reads ${company.reach}`);
if (company.size != null) {
const checked = checkedOn(company.size_checked);
sub.append(` · ${count(company.size, "role")}${checked ? ` · checked ${checked}` : ""}`);
}
if (company.failing_since) { if (company.failing_since) {
// The board has failed on every pull since then; after 21 days its // The board has failed on every pull since then; after 21 days its
// roles close. // roles close.
@@ -115,6 +159,10 @@
sub.appendChild(failing); sub.appendChild(failing);
} }
sub.append(" "); sub.append(" ");
if (company.subscription_id != null) {
sub.appendChild(recheckButton(company, rerender));
sub.append(" ");
}
const edit = document.createElement("button"); const edit = document.createElement("button");
edit.type = "button"; edit.type = "button";
edit.className = "linky"; edit.className = "linky";
+27 -1
View File
@@ -491,6 +491,10 @@ class CompanyOut(BaseModel):
#: what a pull asks its board for (`jobs.reach.line`). #: what a pull asks its board for (`jobs.reach.line`).
reach_params: dict[str, Any] = {} reach_params: dict[str, Any] = {}
reach: str = "" reach: str = ""
#: The roles the board listed when it was last sized, and that day
#: (ISO); None and "" before it is sized (`jobs.reach.SIZE`).
size: int | None = None
size_checked: str = ""
def _company_boards(store: Store) -> dict[str, tuple[int, str, bool, str]]: def _company_boards(store: Store) -> dict[str, tuple[int, str, bool, str]]:
@@ -534,7 +538,10 @@ def _company_outs(store: Store, agent: int | None) -> list[CompanyOut]:
all_roles=bool(c.all_roles), all_roles=bool(c.all_roles),
reach_params={k: v for k, v in (sub.params if sub else {}).items() reach_params={k: v for k, v in (sub.params if sub else {}).items()
if k in jobs.reach.OVERRIDE_KEYS}, if k in jobs.reach.OVERRIDE_KEYS},
reach=jobs.reach.line(plan, sub) if sub else "")) reach=jobs.reach.line(plan, sub) if sub else "",
size=plan.sizes.get(target) if sub else None,
size_checked=store.state.get(
jobs.reach.SIZE_CHECKED + target, "") if sub else ""))
return out return out
@@ -897,6 +904,25 @@ def set_reach(sub_id: int, body: ReachBody, store: StoreDep) -> dict[str, Any]:
"reach": jobs.reach.line(jobs.reach.plan(store, [sub]), sub)} "reach": jobs.reach.line(jobs.reach.plan(store, [sub]), sub)}
@router.post("/sources/{sub_id}/size")
def recheck_size(sub_id: int, store: StoreDep, queue: QueueDep) -> dict[str, Any]:
"""Size a board again, the one way its size changes once it was added
(`jobs.sources.size_board`): its reach may change with it, so a pull is
queued right after, as the other routes that change what is read do. 422
when the board does not answer; the size it had stays."""
sub = store.subscription(sub_id)
if sub is None or sub.source != "ats":
raise HTTPException(404, f"no such board: {sub_id}")
try:
size = jobs.sources.size_board(store, sub.target)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
queue.enqueue("pull")
checked = store.state.get(jobs.reach.SIZE_CHECKED + sub.target, "")
return {"size": size, "size_checked": checked,
"reach": jobs.reach.line(jobs.reach.plan(store, [sub]), sub)}
class ReaderOut(BaseModel): class ReaderOut(BaseModel):
"""What a reader declares (`plugins/readers/_contract.py`), for the """What a reader declares (`plugins/readers/_contract.py`), for the
Sources page: `filters` holds only the dimensions it filters.""" Sources page: `filters` holds only the dimensions it filters."""
+86 -1
View File
@@ -1,5 +1,5 @@
"""A board's size, set once: `ats.board_size` per kind of host, `set_board` """A board's size, set once: `ats.board_size` per kind of host, `set_board`
that keeps it, the pull that sizes a board and the Recheck (route and CLI) that keep it, the pull that sizes a board
with none before it reads it, and the Companies line that names what a with none before it reads it, and the Companies line that names what a
site's own ids leave out. No network: `ats._get` or the runner's listing is site's own ids leave out. No network: `ats._get` or the runner's listing is
stubbed.""" stubbed."""
@@ -8,19 +8,25 @@ from __future__ import annotations
import json import json
import sys import sys
from collections.abc import Iterator
from datetime import UTC, datetime from datetime import UTC, datetime
from pathlib import Path from pathlib import Path
from typing import Any, ClassVar from typing import Any, ClassVar
from urllib.parse import parse_qs, urlsplit from urllib.parse import parse_qs, urlsplit
import pytest import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
ROOT = Path(__file__).resolve().parents[1] ROOT = Path(__file__).resolve().parents[1]
if str(ROOT) not in sys.path: if str(ROOT) not in sys.path:
sys.path.insert(0, str(ROOT)) sys.path.insert(0, str(ROOT))
from pipeline import Tally # noqa: E402 from pipeline import Tally # noqa: E402
from app.cli import main # noqa: E402
from app.config import Config, ServerConfig # noqa: E402
from app.jobs import pull, reach, sources # noqa: E402 from app.jobs import pull, reach, sources # noqa: E402
from app.main import create_app # noqa: E402
from app.models import Gate # noqa: E402 from app.models import Gate # noqa: E402
from app.storage import Store # noqa: E402 from app.storage import Store # noqa: E402
from plugins import ats # noqa: E402 from plugins import ats # noqa: E402
@@ -341,3 +347,82 @@ def test_a_boards_own_reach_is_named_the_same_and_remote_alone_is_read_whole(
"set here: DK · NO not on this site · all roles" "set here: DK · NO not on this site · all roles"
assert reach.line(plan, _sub(store, "workday:other.wd3/Jobs")) == \ assert reach.line(plan, _sub(store, "workday:other.wd3/Jobs")) == \
"all places, no remote option on this site · all roles" "all places, no remote option on this site · all roles"
# ------------------------------------------------------ Recheck and the CLI
@pytest.fixture
def app(tmp_path: Path) -> FastAPI:
return create_app(Config(server=ServerConfig(host="127.0.0.1", port=0),
database=tmp_path / "roledesk.db"))
@pytest.fixture
def client(app: FastAPI) -> Iterator[TestClient]:
with TestClient(app) as c:
yield c
def test_recheck_sizes_the_board_again_and_queues_a_pull(
app: FastAPI, client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None:
store: Store = app.state.store
_wolf(store)
_board(store, BOSCH, "Bosch", size=120)
store.state[f"ats:sizechecked:{BOSCH}"] = "2026-09-01"
sub_id = _sub(store, BOSCH).id
[row] = client.get("/api/companies").json()
assert (row["size"], row["size_checked"], row["reach"]) == (
120, "2026-09-01", "under 300 roles: read whole")
monkeypatch.setattr(ats, "_get", SmartRecruiters(4830))
res = client.post(f"/api/sources/{sub_id}/size")
assert res.status_code == 200, res.text
assert res.json() == {"size": 4830, "size_checked": TODAY,
"reach": "follows agents: DK, SE · all roles"}
assert app.state.queue.jobs.last("pull", states=("queued", "running", "done")) is not None
[row] = client.get("/api/companies").json()
assert (row["size"], row["size_checked"]) == (4830, TODAY)
# A board that does not answer keeps the size it had.
monkeypatch.setattr(ats, "_get", SmartRecruiters(5, down=True))
res = client.post(f"/api/sources/{sub_id}/size")
assert res.status_code == 422 and "did not answer" in res.json()["detail"]
assert store.state[f"ats:size:{BOSCH}"] == "4830"
assert client.post("/api/sources/9999/size").status_code == 404
def _cli(tmp_path: Path, *argv: str) -> int:
config = tmp_path / "config.yaml"
if not config.exists():
config.write_text(f"database: {tmp_path / 't.db'}\n")
return main(["-c", str(config), "companies", *argv])
def test_the_cli_sizes_one_board_or_every_board_without_a_size(
store: Store, tmp_path: Path, monkeypatch: pytest.MonkeyPatch,
capsys: pytest.CaptureFixture[str]) -> None:
_wolf(store)
_board(store, BOSCH, "Bosch", size=12)
_board(store, "smartrecruiters:tiny", "Tiny")
_board(store, "smartrecruiters:gone", "Gone")
monkeypatch.setattr(ats, "_get", SmartRecruiters(4830))
assert _cli(tmp_path, "size", "bosch") == 0
assert json.loads(capsys.readouterr().out) == {
"board": BOSCH, "size": 4830, "reach": "follows agents: DK, SE · all roles"}
def some(client: Any, url: str, host: str, slug: str, **kwargs: Any) -> bytes:
if slug == "gone":
raise OSError("connection refused")
return SmartRecruiters(40)(client, url, host, slug)
monkeypatch.setattr(ats, "_get", some)
assert _cli(tmp_path, "size", "--missing") == 1
out, err = capsys.readouterr()
assert out.splitlines() == [
json.dumps({"board": "smartrecruiters:tiny", "size": 40,
"reach": "under 300 roles: read whole"}),
"sized 1 of 2 boards without a size"]
assert "smartrecruiters:gone did not answer" in err
assert store.state[f"ats:size:{BOSCH}"] == "4830"
assert _cli(tmp_path, "size") == 2
assert _cli(tmp_path, "size", "nobody") == 1
+3 -2
View File
@@ -603,7 +603,8 @@ def test_companies_lists_board_and_open_role_count(
"careers_opened_at": "", "failing_since": "", "failing_reason": "", "careers_opened_at": "", "failing_since": "", "failing_reason": "",
"from_search": False, "hand_reason": "", "by_hand": False, "from_search": False, "hand_reason": "", "by_hand": False,
"all_countries": False, "all_roles": False, "all_countries": False, "all_roles": False,
"reach_params": {}, "reach": "size not known yet: sized on the next pull"} "reach_params": {}, "reach": "size not known yet: sized on the next pull",
"size": None, "size_checked": ""}
[other] = [c for c in res.json() if c["key"] == "other"] [other] = [c for c in res.json() if c["key"] == "other"]
assert (other["board"], other["subscription_id"], other["on"], other["ats"]) == ( assert (other["board"], other["subscription_id"], other["on"], other["ats"]) == (
"", None, None, "") "", None, None, "")
@@ -1453,7 +1454,7 @@ def test_a_company_is_added_by_hand_with_a_reason_and_a_link(
"country": "", "city": "", "watch": None, "careers_opened_at": "", "country": "", "city": "", "watch": None, "careers_opened_at": "",
"failing_since": "", "failing_reason": "", "from_search": False, "failing_since": "", "failing_reason": "", "from_search": False,
"hand_reason": "added", "by_hand": True, "all_countries": False, "all_roles": False, "hand_reason": "added", "by_hand": True, "all_countries": False, "all_roles": False,
"reach_params": {}, "reach": ""} "reach_params": {}, "reach": "", "size": None, "size_checked": ""}
assert store.company("revolut").hand_reason == "added" # type: ignore[union-attr] assert store.company("revolut").hand_reason == "added" # type: ignore[union-attr]
other = client.post("/api/companies", json={"name": "Meta", "hand_reason": "blocked"}) other = client.post("/api/companies", json={"name": "Meta", "hand_reason": "blocked"})