Drop pruning boards by region: a fix for the US seed, and a wrong one

`app companies prune`, `/api/prune` and the Sources disclosure switched off every board with no open role in a region. It existed because the first seed was 186 US boards; the European seed makes it moot, and it was wrong on its own terms: a board with no European role today (a company between openings, or a fetch that timed out) was switched off for good. A company that hires nowhere near should not be in the list, and a ranking's country gate handles the rest. `EUROPE` in models had no other reader and goes with it.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Fable 5.1 committed 2026-09-28 15:35:18 +02:00
1 parent 3aee48997c
commit e602eb3388
12 files changed
+11 -233

No files matched your search

+1 -31
View File
@@ -3,7 +3,7 @@
app serve [--host H] [--port P] [--reload]
app job pull|extract|embed|score [--payload JSON]
app worker [--workers http=8,cpu=1] [--until-idle]
app companies seed|add|discover|check|prune
app companies seed|add|discover|check
app searches add|list|remove
app db init
@@ -79,12 +79,6 @@ def build_parser() -> argparse.ArgumentParser:
discover.add_argument("--add", action="store_true",
help="add the company, with its board when one is readable")
companies_sub.add_parser("check", help="report boards that do not answer")
prune = companies_sub.add_parser(
"prune", help="unsubscribe every board with no open role in the region")
prune.add_argument("--in", dest="region", default="europe",
help="'europe' or country codes, e.g. DK,SE,NO,EU (default europe)")
prune.add_argument("--apply", action="store_true",
help="unsubscribe and close their roles; without it, only list them")
searches = sub.add_parser("searches", help="saved searches on cross-company job boards")
searches_sub = searches.add_subparsers(dest="action", required=True)
@@ -242,8 +236,6 @@ def run_companies(args: argparse.Namespace) -> int:
return add_company(store, args.board, args.name)
if args.action == "discover":
return discover_company(store, args.name, args.domain, add=args.add)
if args.action == "prune":
return prune_companies(store, args.region, apply=args.apply)
return check_companies(store)
@@ -346,28 +338,6 @@ def check_companies(store: Store) -> int:
return 1 if dead else 0
def prune_companies(store: Store, region: str, *, apply: bool) -> int:
"""`app companies prune --in DK,EU`: every fetched board none of whose open
roles is in the region, which is `europe` unless named. `--apply`
switches them off, which closes their open roles. `app companies seed`
never brings one back: it skips a company that exists. See
`app.jobs.sources.prune_plan` for which boards count."""
from app import jobs
try:
filled, plan = jobs.sources.prune_plan(store, region)
except ValueError as exc:
print(f"companies prune: {exc}", file=sys.stderr)
return 1
for sub, count in plan:
print(f"{sub.target:40} {sub.label:30} {count:6} open, none in {region}")
result: dict[str, Any] = {"countries_filled": filled, "pruned": len(plan), "applied": apply}
if apply:
result |= jobs.sources.prune(store, region)
print(json.dumps(result))
return 0
def run_searches(args: argparse.Namespace) -> int:
from app import jobs
from app.storage import Store
+1 -1
View File
@@ -9,7 +9,7 @@ task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`.
cascade.py pull → embed → score → extract → score → summarise, what `TASKS`
offers, the `pull` job a schedule fires, and that schedule
sources.py what a pull reads, changed by hand: boards and searches
added, switched on or off, pruned by region
added, switched on or off
discover.py which ATS a company's careers site runs on, from its pages
"""
+3 -38
View File
@@ -1,7 +1,6 @@
"""What a pull reads, changed by hand: add a board or a search, switch one on
or off, prune boards by region. `app companies`, `app searches` and the
settings pages all call these, so the command line and the pages cannot disagree
about what "off" means.
or off. `app companies`, `app searches` and the settings pages all call these,
so the command line and the pages cannot disagree about what "off" means.
A board or search switched off has its open roles closed at once. Nothing
else would ever close them, since only a pull that reads a target closes its
@@ -14,7 +13,7 @@ import os
from typing import TYPE_CHECKING, Any
from app.config import Config
from app.models import EUROPE, Subscription, company_key
from app.models import Subscription, company_key
from app.storage import Store
from app.jobs.pull import search_target
@@ -25,40 +24,6 @@ if TYPE_CHECKING:
PROVIDERS = ("jobindex", "adzuna")
def region_codes(region: str) -> set[str]:
"""`europe`, or comma-separated ISO codes such as `DK,SE,EU`, as a set of
codes. `europe` may be one entry among codes: `europe,US`."""
codes = {c.strip().upper() for c in (region or "").split(",") if c.strip()}
if "EUROPE" in codes:
codes = (codes - {"EUROPE"}) | set(EUROPE)
if not codes:
raise ValueError("name a region: europe, or country codes such as DK,SE,EU")
return codes
def prune_plan(store: Store, region: str) -> tuple[int, list[tuple[Subscription, int]]]:
"""Every fetched board none of whose open roles is in the region, with its
open role count, largest first; and how many roles had `country` filled
from `place` on the way. A board whose roles all name no country is kept:
it says nothing about where that company hires."""
wanted = region_codes(region)
filled = store.backfill_country()
countries = store.open_countries_by_subscription()
plan = [(sub, sum(countries[sub.id].values()))
for sub in store.subscriptions(source="ats", fetchable=True)
if set(countries.get(sub.id, {})) - {""}
and not wanted.intersection(countries[sub.id])]
return filled, sorted(plan, key=lambda p: -p[1])
def prune(store: Store, region: str) -> dict[str, Any]:
"""Switch off every board `prune_plan` names."""
filled, plan = prune_plan(store, region)
closed = sum(set_fetch(store, sub.id, False)[1] for sub, _ in plan)
return {"countries_filled": filled, "pruned": len(plan), "roles_closed": closed,
"boards_left": len(store.subscriptions(source="ats", fetchable=True))}
def set_fetch(store: Store, sub_id: int, fetch: bool) -> tuple[Subscription, int]:
"""Switch one board or search on or off. Off closes its open roles and
returns how many; see the module docstring."""
-9
View File
@@ -66,15 +66,6 @@ def company_key(company: str) -> str:
# ---------------------------------------------------------------- geography
#: The EU, the EEA, the UK and Switzerland: what "a European role" means to a
#: gate on `country`. `EU` is ISO's reserved code, used here for a role open
#: anywhere in Europe ("Remote - Europe", "EMEA") that names no one country.
EUROPE: tuple[str, ...] = (
"AT", "BE", "BG", "HR", "CY", "CZ", "DK", "EE", "FI", "FR", "DE", "GR", "HU", "IE",
"IT", "LV", "LT", "LU", "MT", "NL", "PL", "PT", "RO", "SK", "SI", "ES", "SE",
"IS", "LI", "NO", "GB", "CH", "EU",
)
#: Folded names a board writes in its location line, to an ISO code: every
#: European country in English and its own language, the cities the boards
#: name most, and the non-European places that fill the seed boards, so those
+1 -56
View File
@@ -1,6 +1,5 @@
/* The Company boards section of Sources: every company, its board, its open
* roles, its affinity, following a new one, and pruning boards with nothing
* open in a region.
* roles, its affinity, and following a new one.
*
* The list is app.js's shared company list (`/api/companies`), already
* joined with each company's board subscription — its id and whether it is
@@ -174,58 +173,6 @@
button.disabled = false;
}
// -------------------------------------------------------------------- prune
/** @type {object[]} the boards the last preview named. */
let pruneBoards = [];
async function previewPrune(event) {
event.preventDefault();
const status = $("#prune-status");
const list = $("#prune-list");
const region = $("#p-region").value.trim();
list.replaceChildren();
$("#p-apply").disabled = true;
$("#p-apply").textContent = "Switch off these boards";
status.textContent = "Checking every board…";
try {
const result = await api(`/api/prune?region=${encodeURIComponent(region)}`);
pruneBoards = result.boards;
const roles = pruneBoards.reduce((sum, b) => sum + b.open_roles, 0);
status.textContent = pruneBoards.length
? `${count(pruneBoards.length, "board")} with no open role in ${region}, `
+ `${count(roles, "open role")} between them:`
: `Every board that is on has a role in ${region}.`;
for (const board of pruneBoards) {
const li = document.createElement("li");
li.textContent = `${board.label} (${board.target}), ${count(board.open_roles, "open role")}`;
list.appendChild(li);
}
$("#p-apply").disabled = pruneBoards.length === 0;
$("#p-apply").textContent = `Switch off these ${count(pruneBoards.length, "board")}`;
} catch (err) {
status.textContent = "";
}
}
async function applyPrune() {
const region = $("#p-region").value.trim();
if (!window.confirm(`Switch off ${count(pruneBoards.length, "board")} and close their roles? `
+ "Each can be switched back on in the list above.")) return;
$("#p-apply").disabled = true;
try {
const result = await api("/api/prune", { method: "POST", body: { region } });
$("#prune-status").textContent = `Switched off ${count(result.pruned, "board")}, `
+ `closed ${count(result.roles_closed, "role")}. ${result.boards_left} boards are on.`;
$("#prune-list").replaceChildren();
$("#p-apply").textContent = "Switch off these boards";
await loadCompanies(agentId());
render();
} catch (err) {
$("#p-apply").disabled = false;
}
}
// -------------------------------------------------------------------- init
async function switchAgent() {
@@ -250,7 +197,5 @@
$("#c-filter").addEventListener("change", render);
$("#c-agent").addEventListener("change", switchAgent);
$("#board-form").addEventListener("submit", addBoard);
$("#prune-form").addEventListener("submit", previewPrune);
$("#p-apply").addEventListener("click", applyPrune);
});
})();
+2 -3
View File
@@ -1,8 +1,7 @@
/* The Job boards section of Sources: one query on one board per row, on
* Jobindex or Adzuna. The UI says "job board" for what the API and the CLI
* call a search (`/api/searches`, `app searches`); company boards and the
* prune by region are companies.js's section, since a board is a company's
* own, not a query's.
* call a search (`/api/searches`, `app searches`); company boards are
* companies.js's section, since a board is a company's own, not a query's.
*
* The list comes from `/api/sources`, and every action is a fetch through
* `api()` from app.js, so a refusal (a query that finds nothing) is one
-16
View File
@@ -104,22 +104,6 @@
<p id="companies-count" class="count" role="status"></p>
<div id="companies-list" class="crows"></div>
<details class="prune">
<summary>Switch off boards with no roles in a region</summary>
<p class="lede">A board none of whose open roles is in the region is switched off; a board whose roles name no country is kept.</p>
<form id="prune-form" class="agent-form source-form" autocomplete="off">
<label>Region
<input type="text" id="p-region" value="europe" spellcheck="false">
<span class="hint"><code>europe</code>, or country codes such as <code>DK,SE,NO,EU</code>. EU is a role open anywhere in Europe.</span></label>
<div class="agent-actions">
<button type="submit" class="secondary">Preview</button>
<button type="button" id="p-apply" disabled>Switch off these boards</button>
</div>
<p id="prune-status" class="count" role="status"></p>
<ul id="prune-list" class="sample"></ul>
</form>
</details>
</section>
<section class="card" id="runs" data-section>
-25
View File
@@ -463,31 +463,6 @@ def add_board(body: BoardBody, store: StoreDep) -> dict[str, Any]:
return {"source": _one_source(store, sub.id), "found": found}
@router.get("/prune")
def prune_preview(store: StoreDep, region: str = "europe") -> dict[str, Any]:
"""The boards a prune would switch off: every one on, none of whose open
roles is in the region."""
try:
_, plan = jobs.sources.prune_plan(store, region)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
return {"region": region, "boards": [{"id": sub.id, "label": sub.label,
"target": sub.target, "open_roles": n}
for sub, n in plan]}
class PruneBody(BaseModel):
region: str = "europe"
@router.post("/prune")
def prune(body: PruneBody, store: StoreDep) -> dict[str, Any]:
try:
return jobs.sources.prune(store, body.region)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
# ----------------------------------------------------------- applications
class SaveBody(BaseModel):