From e602eb33881a6216b736ee858bad7d64799ceb4d Mon Sep 17 00:00:00 2001 From: Wolf Date: Mon, 28 Sep 2026 15:35:18 +0200 Subject: [PATCH] Drop pruning boards by region: a fix for the US seed, and a wrong one `app companies prune`, `/api/prune` and the Sources disclosure switched off every board with no open role in a region. It existed because the first seed was 186 US boards; the European seed makes it moot, and it was wrong on its own terms: a board with no European role today (a company between openings, or a fetch that timed out) was switched off for good. A company that hires nowhere near should not be in the list, and a ranking's country gate handles the rest. `EUROPE` in models had no other reader and goes with it. Co-Authored-By: Claude Fable 5.1 --- README.md | 3 +- app/cli.py | 32 +--------------- app/jobs/__init__.py | 2 +- app/jobs/sources.py | 41 ++------------------- app/models.py | 9 ----- app/static/companies.js | 57 +---------------------------- app/static/sources.js | 5 +-- app/templates/settings/sources.html | 16 -------- app/web/api.py | 25 ------------- docs/decisions.md | 4 +- tests/test_search.py | 30 --------------- tests/test_web.py | 20 +--------- 12 files changed, 11 insertions(+), 233 deletions(-) diff --git a/README.md b/README.md index 308ddb2..a90d694 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,6 @@ The Sources sub-tab does all of this; the same actions exist on the command line uv run app searches add jobindex "Copenhagen golang" --q "golang go-udvikler" --area 1 uv run app searches add adzuna "Berlin backend" --q backend --country de --where Berlin uv run app searches list -uv run app companies prune --in DK,SE,NO,EU # lists; add --apply to unsubscribe uv run app companies discover "Pleo" pleo.io --add # find its board on its own site, then follow it ``` @@ -45,7 +44,7 @@ The cog opens `/settings`: a rail of pages on the left, the page beside it, stic | | | |---|---| | **Search agents** | a card per agent, newest first: name, an On/Off switch, what it looks for, and its weights. Edit turns the card into the form in place (name, intent, weights, require/prefer gates behind an Advanced disclosure); New agent puts a blank card on top. Saving or switching on rescores; deleting one takes its votes, liked companies and scores with it | -| **Sources** | one page showing one section at a time, picked in the rail or by the URL fragment: job boards (queries on Jobindex or Adzuna: try, save, edit, remove), company boards (every company whose board is read, its open roles and its affinity for the picked agent, following a company, pruning boards by region) and runs (Pull now, what is running, the schedule, one stage at a time, and the message log) | +| **Sources** | one page showing one section at a time, picked in the rail or by the URL fragment: job boards (queries on Jobindex or Adzuna: try, save, edit, remove), company boards (every company whose board is read, its open roles and its affinity for the picked agent, following a company) and runs (Pull now, what is running, the schedule, one stage at a time, and the message log) | | **Account** | the password, sessions and passkeys. Open on loopback, locked on anything else | Likes, hidden companies and votes belong to a search agent, not to the account: two agents rank the same roles differently, and one may hide a company the other favours. An agent also learns from votes: its `taste` weight scores a role by how close it is to the roles the agent liked minus the ones it passed. diff --git a/app/cli.py b/app/cli.py index 5541c86..109d79e 100644 --- a/app/cli.py +++ b/app/cli.py @@ -3,7 +3,7 @@ app serve [--host H] [--port P] [--reload] app job pull|extract|embed|score [--payload JSON] app worker [--workers http=8,cpu=1] [--until-idle] - app companies seed|add|discover|check|prune + app companies seed|add|discover|check app searches add|list|remove app db init @@ -79,12 +79,6 @@ def build_parser() -> argparse.ArgumentParser: discover.add_argument("--add", action="store_true", help="add the company, with its board when one is readable") companies_sub.add_parser("check", help="report boards that do not answer") - prune = companies_sub.add_parser( - "prune", help="unsubscribe every board with no open role in the region") - prune.add_argument("--in", dest="region", default="europe", - help="'europe' or country codes, e.g. DK,SE,NO,EU (default europe)") - prune.add_argument("--apply", action="store_true", - help="unsubscribe and close their roles; without it, only list them") searches = sub.add_parser("searches", help="saved searches on cross-company job boards") searches_sub = searches.add_subparsers(dest="action", required=True) @@ -242,8 +236,6 @@ def run_companies(args: argparse.Namespace) -> int: return add_company(store, args.board, args.name) if args.action == "discover": return discover_company(store, args.name, args.domain, add=args.add) - if args.action == "prune": - return prune_companies(store, args.region, apply=args.apply) return check_companies(store) @@ -346,28 +338,6 @@ def check_companies(store: Store) -> int: return 1 if dead else 0 -def prune_companies(store: Store, region: str, *, apply: bool) -> int: - """`app companies prune --in DK,EU`: every fetched board none of whose open - roles is in the region, which is `europe` unless named. `--apply` - switches them off, which closes their open roles. `app companies seed` - never brings one back: it skips a company that exists. See - `app.jobs.sources.prune_plan` for which boards count.""" - from app import jobs - - try: - filled, plan = jobs.sources.prune_plan(store, region) - except ValueError as exc: - print(f"companies prune: {exc}", file=sys.stderr) - return 1 - for sub, count in plan: - print(f"{sub.target:40} {sub.label:30} {count:6} open, none in {region}") - result: dict[str, Any] = {"countries_filled": filled, "pruned": len(plan), "applied": apply} - if apply: - result |= jobs.sources.prune(store, region) - print(json.dumps(result)) - return 0 - - def run_searches(args: argparse.Namespace) -> int: from app import jobs from app.storage import Store diff --git a/app/jobs/__init__.py b/app/jobs/__init__.py index 8182e1f..206d9ac 100644 --- a/app/jobs/__init__.py +++ b/app/jobs/__init__.py @@ -9,7 +9,7 @@ task in `pipeline.TASKS` and the default schedule in `pipeline.SCHEDULES`. cascade.py pull → embed → score → extract → score → summarise, what `TASKS` offers, the `pull` job a schedule fires, and that schedule sources.py what a pull reads, changed by hand: boards and searches - added, switched on or off, pruned by region + added, switched on or off discover.py which ATS a company's careers site runs on, from its pages """ diff --git a/app/jobs/sources.py b/app/jobs/sources.py index 3a6a79b..8637670 100644 --- a/app/jobs/sources.py +++ b/app/jobs/sources.py @@ -1,7 +1,6 @@ """What a pull reads, changed by hand: add a board or a search, switch one on -or off, prune boards by region. `app companies`, `app searches` and the -settings pages all call these, so the command line and the pages cannot disagree -about what "off" means. +or off. `app companies`, `app searches` and the settings pages all call these, +so the command line and the pages cannot disagree about what "off" means. A board or search switched off has its open roles closed at once. Nothing else would ever close them, since only a pull that reads a target closes its @@ -14,7 +13,7 @@ import os from typing import TYPE_CHECKING, Any from app.config import Config -from app.models import EUROPE, Subscription, company_key +from app.models import Subscription, company_key from app.storage import Store from app.jobs.pull import search_target @@ -25,40 +24,6 @@ if TYPE_CHECKING: PROVIDERS = ("jobindex", "adzuna") -def region_codes(region: str) -> set[str]: - """`europe`, or comma-separated ISO codes such as `DK,SE,EU`, as a set of - codes. `europe` may be one entry among codes: `europe,US`.""" - codes = {c.strip().upper() for c in (region or "").split(",") if c.strip()} - if "EUROPE" in codes: - codes = (codes - {"EUROPE"}) | set(EUROPE) - if not codes: - raise ValueError("name a region: europe, or country codes such as DK,SE,EU") - return codes - - -def prune_plan(store: Store, region: str) -> tuple[int, list[tuple[Subscription, int]]]: - """Every fetched board none of whose open roles is in the region, with its - open role count, largest first; and how many roles had `country` filled - from `place` on the way. A board whose roles all name no country is kept: - it says nothing about where that company hires.""" - wanted = region_codes(region) - filled = store.backfill_country() - countries = store.open_countries_by_subscription() - plan = [(sub, sum(countries[sub.id].values())) - for sub in store.subscriptions(source="ats", fetchable=True) - if set(countries.get(sub.id, {})) - {""} - and not wanted.intersection(countries[sub.id])] - return filled, sorted(plan, key=lambda p: -p[1]) - - -def prune(store: Store, region: str) -> dict[str, Any]: - """Switch off every board `prune_plan` names.""" - filled, plan = prune_plan(store, region) - closed = sum(set_fetch(store, sub.id, False)[1] for sub, _ in plan) - return {"countries_filled": filled, "pruned": len(plan), "roles_closed": closed, - "boards_left": len(store.subscriptions(source="ats", fetchable=True))} - - def set_fetch(store: Store, sub_id: int, fetch: bool) -> tuple[Subscription, int]: """Switch one board or search on or off. Off closes its open roles and returns how many; see the module docstring.""" diff --git a/app/models.py b/app/models.py index e741afa..daf276f 100644 --- a/app/models.py +++ b/app/models.py @@ -66,15 +66,6 @@ def company_key(company: str) -> str: # ---------------------------------------------------------------- geography -#: The EU, the EEA, the UK and Switzerland: what "a European role" means to a -#: gate on `country`. `EU` is ISO's reserved code, used here for a role open -#: anywhere in Europe ("Remote - Europe", "EMEA") that names no one country. -EUROPE: tuple[str, ...] = ( - "AT", "BE", "BG", "HR", "CY", "CZ", "DK", "EE", "FI", "FR", "DE", "GR", "HU", "IE", - "IT", "LV", "LT", "LU", "MT", "NL", "PL", "PT", "RO", "SK", "SI", "ES", "SE", - "IS", "LI", "NO", "GB", "CH", "EU", -) - #: Folded names a board writes in its location line, to an ISO code: every #: European country in English and its own language, the cities the boards #: name most, and the non-European places that fill the seed boards, so those diff --git a/app/static/companies.js b/app/static/companies.js index 7f16956..8c8aab5 100644 --- a/app/static/companies.js +++ b/app/static/companies.js @@ -1,6 +1,5 @@ /* The Company boards section of Sources: every company, its board, its open - * roles, its affinity, following a new one, and pruning boards with nothing - * open in a region. + * roles, its affinity, and following a new one. * * The list is app.js's shared company list (`/api/companies`), already * joined with each company's board subscription — its id and whether it is @@ -174,58 +173,6 @@ button.disabled = false; } - // -------------------------------------------------------------------- prune - - /** @type {object[]} the boards the last preview named. */ - let pruneBoards = []; - - async function previewPrune(event) { - event.preventDefault(); - const status = $("#prune-status"); - const list = $("#prune-list"); - const region = $("#p-region").value.trim(); - list.replaceChildren(); - $("#p-apply").disabled = true; - $("#p-apply").textContent = "Switch off these boards"; - status.textContent = "Checking every board…"; - try { - const result = await api(`/api/prune?region=${encodeURIComponent(region)}`); - pruneBoards = result.boards; - const roles = pruneBoards.reduce((sum, b) => sum + b.open_roles, 0); - status.textContent = pruneBoards.length - ? `${count(pruneBoards.length, "board")} with no open role in ${region}, ` - + `${count(roles, "open role")} between them:` - : `Every board that is on has a role in ${region}.`; - for (const board of pruneBoards) { - const li = document.createElement("li"); - li.textContent = `${board.label} (${board.target}), ${count(board.open_roles, "open role")}`; - list.appendChild(li); - } - $("#p-apply").disabled = pruneBoards.length === 0; - $("#p-apply").textContent = `Switch off these ${count(pruneBoards.length, "board")}`; - } catch (err) { - status.textContent = ""; - } - } - - async function applyPrune() { - const region = $("#p-region").value.trim(); - if (!window.confirm(`Switch off ${count(pruneBoards.length, "board")} and close their roles? ` - + "Each can be switched back on in the list above.")) return; - $("#p-apply").disabled = true; - try { - const result = await api("/api/prune", { method: "POST", body: { region } }); - $("#prune-status").textContent = `Switched off ${count(result.pruned, "board")}, ` - + `closed ${count(result.roles_closed, "role")}. ${result.boards_left} boards are on.`; - $("#prune-list").replaceChildren(); - $("#p-apply").textContent = "Switch off these boards"; - await loadCompanies(agentId()); - render(); - } catch (err) { - $("#p-apply").disabled = false; - } - } - // -------------------------------------------------------------------- init async function switchAgent() { @@ -250,7 +197,5 @@ $("#c-filter").addEventListener("change", render); $("#c-agent").addEventListener("change", switchAgent); $("#board-form").addEventListener("submit", addBoard); - $("#prune-form").addEventListener("submit", previewPrune); - $("#p-apply").addEventListener("click", applyPrune); }); })(); diff --git a/app/static/sources.js b/app/static/sources.js index 97a36de..cd3d319 100644 --- a/app/static/sources.js +++ b/app/static/sources.js @@ -1,8 +1,7 @@ /* The Job boards section of Sources: one query on one board per row, on * Jobindex or Adzuna. The UI says "job board" for what the API and the CLI - * call a search (`/api/searches`, `app searches`); company boards and the - * prune by region are companies.js's section, since a board is a company's - * own, not a query's. + * call a search (`/api/searches`, `app searches`); company boards are + * companies.js's section, since a board is a company's own, not a query's. * * The list comes from `/api/sources`, and every action is a fetch through * `api()` from app.js, so a refusal (a query that finds nothing) is one diff --git a/app/templates/settings/sources.html b/app/templates/settings/sources.html index 1806a2e..a837115 100644 --- a/app/templates/settings/sources.html +++ b/app/templates/settings/sources.html @@ -104,22 +104,6 @@

- -
- Switch off boards with no roles in a region -

A board none of whose open roles is in the region is switched off; a board whose roles name no country is kept.

-
- -
- - -
-

-
    -
    -
    diff --git a/app/web/api.py b/app/web/api.py index 1e3d1a5..fea6a47 100644 --- a/app/web/api.py +++ b/app/web/api.py @@ -463,31 +463,6 @@ def add_board(body: BoardBody, store: StoreDep) -> dict[str, Any]: return {"source": _one_source(store, sub.id), "found": found} -@router.get("/prune") -def prune_preview(store: StoreDep, region: str = "europe") -> dict[str, Any]: - """The boards a prune would switch off: every one on, none of whose open - roles is in the region.""" - try: - _, plan = jobs.sources.prune_plan(store, region) - except ValueError as exc: - raise HTTPException(422, str(exc)) from exc - return {"region": region, "boards": [{"id": sub.id, "label": sub.label, - "target": sub.target, "open_roles": n} - for sub, n in plan]} - - -class PruneBody(BaseModel): - region: str = "europe" - - -@router.post("/prune") -def prune(body: PruneBody, store: StoreDep) -> dict[str, Any]: - try: - return jobs.sources.prune(store, body.region) - except ValueError as exc: - raise HTTPException(422, str(exc)) from exc - - # ----------------------------------------------------------- applications class SaveBody(BaseModel): diff --git a/docs/decisions.md b/docs/decisions.md index 412bd15..6e428b6 100644 --- a/docs/decisions.md +++ b/docs/decisions.md @@ -54,11 +54,9 @@ What was decided for roledesk and the one reason for each. The longer reasoning **Jobindex ads are read from the feed, not followed to the employer.** Each item links to Jobindex's own page, and its `/c?t=` redirect leads to the employer's site or ATS. Following it per role would count as a click in Jobindex's statistics and costs a request per role, so discovering a company's ATS board from a search result is left for later. -**`app companies prune` cuts the list by where it hires.** A board none of whose open roles is in the region (`--in`, Europe by default, or codes such as `DK,SE,EU`) is switched off. A board whose roles name no country is kept, because it says nothing about where the company hires. - **Switching a board or search off closes its open roles.** Only a pull that reads a target closes its roles, so a target no longer read would keep them open for good. Switched back on, the next pull reopens what it still lists. A batch already in flight when the switch happens reopens them again, so the pull's settle closes every switched-off target once more. -**The Sources tab and the CLI share `app.jobs.sources`.** Adding a board or a search, switching one, and pruning are one function each, called by `app companies`, `app searches` and `/api/sources*`, so the two cannot disagree about what "off" does. A search or board is run once before it is saved, and refused if it does not answer or finds nothing. +**The Sources tab and the CLI share `app.jobs.sources`.** Adding a board or a search and switching one are one function each, called by `app companies`, `app searches` and `/api/sources*`, so the two cannot disagree about what "off" does. A search or board is run once before it is saved, and refused if it does not answer or finds nothing. **SmartRecruiters' board list has no description.** Its `/postings` endpoint gives title, location and department; the job text is only on the per-posting endpoint, and one extra request per role would multiply a board's request count by its role count. `description` stays empty for this host. diff --git a/tests/test_search.py b/tests/test_search.py index ef2690e..8e1362f 100644 --- a/tests/test_search.py +++ b/tests/test_search.py @@ -18,7 +18,6 @@ if str(ROOT) not in sys.path: from desk.models import Item # noqa: E402 from plugins import search # noqa: E402 from plugins.search import SearchSource # noqa: E402 -from app import cli # noqa: E402 from app.config import SearchSeed # noqa: E402 from app.jobs import pull # noqa: E402 from app.models import country_of # noqa: E402 @@ -222,35 +221,6 @@ def test_seed_searches_imports_once(store: Store) -> None: # -------------------------------------------------------------------- cli -def test_prune_unsubscribes_boards_with_no_role_in_the_region( - store: Store, capsys: pytest.CaptureFixture[str]) -> None: - def board(slug: str, *places: str) -> int: - sub = store.add_subscription("ats", f"greenhouse:{slug}", label=slug, company=slug, - board="greenhouse") - for n, place in enumerate(places): - it = Item(source_id=f"greenhouse:{slug}:{n}", location="https://x.example/1", - title=f"Engineer {n}", body="", raw={"company": slug, "place": place}) - it.source = "ats" - store.upsert_role(it, sub) - return sub.id - - us = board("usonly", "Boston, MA", "Remote - US") - both = board("both", "San Francisco, CA", "London, UK") - vague = board("vague", "Remote", "Hybrid") - - cli.prune_companies(store, "europe", apply=False) - assert "usonly" in capsys.readouterr().out - assert len(store.subscriptions(source="ats", fetchable=True)) == 3 - - cli.prune_companies(store, "europe", apply=True) - fetchable = {s.id for s in store.subscriptions(source="ats", fetchable=True)} - assert fetchable == {both, vague} - assert all(r.subscription_id != us for r in store.roles()) - - cli.prune_companies(store, "DK,EU", apply=True) - assert {s.id for s in store.subscriptions(source="ats", fetchable=True)} == {vague} - - def test_search_source_stamps_complete_and_ids(monkeypatch: pytest.MonkeyPatch) -> None: _stub_search(monkeypatch, ["h1"], complete=True) source = SearchSource({"searches": [{"url": "jobindex:cph", "label": "Copenhagen", diff --git a/tests/test_web.py b/tests/test_web.py index d07acf7..d76f7ac 100644 --- a/tests/test_web.py +++ b/tests/test_web.py @@ -638,11 +638,10 @@ def test_the_sources_page_has_a_section_per_plugin(client: TestClient) -> None: assert 'id="searches-list"' in page and 'id="search-form"' in page assert 'value="1" selected>Region Hovedstaden' in page assert "job board" in page.lower() and "saved search" not in page.lower() - # Company boards: the follow form, the agent select, the list, the prune tool. + # Company boards: the follow form, the agent select, the list. assert 'id="companies-list"' in page and 'id="c-agent"' in page assert 'id="b-host"' in page and 'id="b-slug"' in page and 'id="b-name"' in page assert '