Merge board size set once, Recheck, and the Companies line naming what a site does not know (wp-a-size)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
WolfandClaude Opus 5.5 committed 2026-09-30 13:45:27 +02:00
commit 8c66aeb445
11 files changed
+786 -70

No files matched your search

+45 -1
View File
@@ -3,7 +3,7 @@
app serve [--host H] [--port P] [--reload]
app job pull|extract|embed|score [--payload JSON]
app worker [--workers http=8,cpu=1] [--until-idle]
app companies seed|add|edit|checked|remove|by-hand|set-board|detach|discover|check
app companies seed|add|edit|checked|remove|by-hand|set-board|detach|discover|check|size
app searches add|list|remove
app agents list|prompt
app stages [extract|summarise on|off]
@@ -32,6 +32,7 @@ from typing import TYPE_CHECKING, Any
from app import config as config_mod
if TYPE_CHECKING:
from app.models import Subscription
from app.storage import Store
#: The one plugin this app ships lives outside the `app` package, in
@@ -118,6 +119,10 @@ def build_parser() -> argparse.ArgumentParser:
help="add or update the company: its board when one is readable, "
"else the host it runs on")
companies_sub.add_parser("check", help="report boards that do not answer, now or in the pull")
size = companies_sub.add_parser(
"size", help="size a company's board again, or --missing every board without a size")
size.add_argument("key", nargs="?", default="", help="the company key, e.g. infineon")
size.add_argument("--missing", action="store_true", help="every board with no size yet")
searches = sub.add_parser("searches", help="saved searches on cross-company job boards")
searches_sub = searches.add_subparsers(dest="action", required=True)
@@ -335,6 +340,8 @@ def run_companies(args: argparse.Namespace) -> int:
return detach_board(store, args.key)
if args.action == "discover":
return discover_company(store, args.name, args.domain, add=args.add)
if args.action == "size":
return size_boards(store, args.key, missing=args.missing)
return check_companies(store)
@@ -503,6 +510,43 @@ def detach_board(store: Store, key: str) -> int:
return 0
def size_boards(store: Store, key: str, *, missing: bool) -> int:
"""`app companies size infineon`: size its board again and keep it, as
the Companies row's Recheck does, and print the size and what a pull asks
the board for. `--missing` sizes every board shown that has no size yet,
a line each, and exits 1 if any did not answer. The next pull reads a
board so; nothing is queued from here."""
from app import jobs
if bool(key) == missing:
print("companies size: name a company key, or --missing", file=sys.stderr)
return 2
if missing:
subs = [s for s in store.subscriptions(source="ats") if s.show]
known = jobs.reach.sizes(store, subs)
subs = [s for s in subs if s.target not in known]
else:
sub = store.company_subscription(key)
if sub is None or not sub.show:
print(f"companies size: {key!r} has no board", file=sys.stderr)
return 1
subs = [sub]
sized: list[tuple[Subscription, int]] = []
for sub in subs:
try:
sized.append((sub, jobs.sources.size_board(store, sub.target)))
except ValueError as exc:
print(f"companies size: {exc}", file=sys.stderr)
plan = jobs.reach.plan(store, [sub for sub, _size in sized])
for sub, size in sized:
print(json.dumps({"board": sub.target, "size": size,
"reach": jobs.reach.line(plan, sub)}))
dead = len(subs) - len(sized)
if missing:
print(f"sized {len(sized)} of {len(subs)} boards without a size")
return 1 if dead else 0
def check_companies(store: Store) -> int:
"""`app companies check`: every `ats` board, fetched once. Prints a
line per board that did not answer now, answered incompletely, or has
+113 -19
View File
@@ -21,8 +21,13 @@ on. Computed once per pull (`plan`) and handed to each board's batch as the
it skips the `any` gates; it does not widen what is fetched.
- **Board override**: a board whose subscription `params` name a country, a
place or remote gets no reach here; `ask_for` reads the params instead.
- **Small boards**: a board that listed under `SMALL` roles when last read
whole gets no filter; filtering it saves little and risks closing roles.
- **Small boards**: a board of under `SMALL` roles gets no filter;
filtering it saves little and risks closing roles. A board's size is set
once, when it is added or rechecked (`plugins/ats.py`, `board_size`), and
never by a pull. A board with no size yet is handed the reach it would
have as a big one, with `unsized` and `small`, and its batch sizes it
before it reads it, so a big new board is not read whole first; a board
whose reader cannot narrow that reach is sized by its whole read instead.
A reach of more countries than a `one`-filter host should be asked one by
one (the wide reach's Europe on SmartRecruiters) is dropped by `ask_for`,
@@ -43,14 +48,15 @@ board sends the roles that match one of them rather than every role:
changes nothing on the narrow reach today, which takes no terms.
- Only a reader that declares `query="one"` takes them, one series a term;
`ask_for` has the rule for terms times the reach's series.
- A read with terms is never the whole board: it sets no `ats:size`, and
the pull closes nothing against it (`app/jobs/pull.py`).
- A read with terms is never the whole board: the pull closes nothing
against it (`app/jobs/pull.py`).
- The search providers take none. Every saved search has a query of its
own, which terms would replace; a search is a query already.
"""
from __future__ import annotations
import json
from collections.abc import Iterable, Mapping
from dataclasses import dataclass, field
from typing import Any
@@ -65,9 +71,15 @@ from plugins.readers import _runner
#: whole.
SMALL = 300
#: `state` key of a board's size, `ats:size:<target>`: the roles its last
#: unfiltered read listed, written by `plugins/ats.py`.
#: `state` keys of a board's size, `ats:size:<target>`, the roles it listed
#: when it was sized, and of the day it was sized (ISO), both written by
#: `plugins/ats.py` (`keep_size`).
SIZE = "ats:size:"
SIZE_CHECKED = "ats:sizechecked:"
#: `state` key of the host ids a board's site states (`{"checked",
#: "facets"}`), kept by the runner (`_runner.facets_for`).
FACETS = "ats:facets:"
#: The params that make a board's own reach (see `_runner._PARAM_DIMENSION`).
OVERRIDE_KEYS = frozenset(("countries", "country", "places", "place", "remote"))
@@ -239,7 +251,9 @@ class Plan:
"""The reach of one pull: `base` for every board, `wide` for the boards
of the companies in `all_countries`, each board's size by target, the
fetch `terms` of the wide reads, which skip the companies in
`all_roles`, and `wide_names`, what the agents call the wide reach."""
`all_roles`, `wide_names`, what the agents call the wide reach, and
`facets`, the host ids the runner keeps for a board on a host that
filters by them, for the Companies line."""
base: Reach
wide: Reach = EVERYWHERE
@@ -248,6 +262,7 @@ class Plan:
terms: tuple[str, ...] = ()
all_roles: frozenset[str] = frozenset()
wide_names: str = ""
facets: dict[str, dict[str, dict[str, str]]] = field(default_factory=dict)
def why(self, sub: Subscription) -> tuple[Reach, str]:
"""The reach `sub` is read with, and why: `override`, `small`,
@@ -260,6 +275,10 @@ class Plan:
return EVERYWHERE, "unknown"
if size < SMALL:
return EVERYWHERE, "small"
return self._big(sub)
def _big(self, sub: Subscription) -> tuple[Reach, str]:
"""The reach of `sub` as a board of `SMALL` roles or more."""
if company_key(sub.company or sub.label) in self.all_countries:
return self.wide, "countries"
return self.base, "agents"
@@ -274,30 +293,51 @@ class Plan:
fetch terms, and unless its company is set to All roles, a board on
the wide reach carries them and `wide`, and a board on the agents'
reach carries them for `ask_for` to use if it drops the countries as
too many (see the module docstring)."""
too many (see the module docstring). A board with no size carries
`unsized` and `small`, and the reach of a big board unless its params
set their own, for its batch to size it first."""
found, why = self.why(sub)
if why == "unknown":
found, why = self._big(sub)
out = found.params()
if self.terms and why in ("countries", "agents") and self.narrows_roles(sub):
out = (out or {}) | {"terms": list(self.terms)}
if why == "countries":
out["wide"] = True
if sub.target not in self.sizes:
out = (out or {}) | {"unsized": True, "small": SMALL}
return out
def sizes(store: Store, subs: Iterable[Subscription]) -> dict[str, int]:
"""Each board's size: the roles its last unfiltered read listed
(`ats:size:<target>`), else, for a board read before that was kept and
never filtered, its open roles. A board never read has none."""
"""Each board's size as kept (`ats:size:<target>`); a board never sized
has none."""
state = store.state
counts = store.open_countries_by_subscription()
out: dict[str, int] = {}
for sub in subs:
kept = state.get(f"{SIZE}{sub.target}", "")
if kept.isdigit():
out[sub.target] = int(kept)
elif (state.get(f"ats:ok:{sub.target}") and not state.get(f"ats:filtered:{sub.target}")
and not state.get(f"ats:narrowed:{sub.target}")):
out[sub.target] = sum(counts.get(sub.id, {}).values())
return out
def site_facets(store: Store, subs: Iterable[Subscription]) -> dict[str, dict[str, dict[str, str]]]:
"""The host ids the runner keeps for each board on a host whose country
or remote filter is `ids` (`ats:facets:<target>`); a board with none kept
is left out."""
out: dict[str, dict[str, dict[str, str]]] = {}
for sub in subs:
module = readers.HOSTS.get(sub.target.partition(":")[0])
if module is None or not _runner.declared(module) \
or "ids" not in (module.READS.filters.country, module.READS.filters.remote):
continue
try:
kept = json.loads(store.state.get(f"{FACETS}{sub.target}") or "{}")
except (TypeError, ValueError):
continue
found = kept.get("facets") if isinstance(kept, dict) else None
if isinstance(found, dict):
out[sub.target] = found
return out
@@ -321,7 +361,8 @@ def plan(store: Store, subs: Iterable[Subscription] | None = None) -> Plan:
all_countries=frozenset(c.key for c in rows if c.all_countries),
terms=fetch_terms(agents),
all_roles=frozenset(c.key for c in rows if c.all_roles),
wide_names=wide_names(agents, gaz) if agents else "everywhere")
wide_names=wide_names(agents, gaz) if agents else "everywhere",
facets=site_facets(store, subs))
def reaches(store: Store, subs: Iterable[Subscription]) -> Mapping[str, dict[str, Any]]:
@@ -338,14 +379,58 @@ _WHOLE = {"small": f"under {SMALL} roles", "unknown": "size not known yet",
"agents": "an agent takes any place"}
def _codes(codes: Iterable[str]) -> list[str]:
"""Country codes as the line names them: up to four, else a count."""
found = sorted(codes)
return found if len(found) <= 4 else [f"{len(found)} countries"]
def _places(ask: readers.Ask) -> str:
words = sorted(ask.countries) if len(ask.countries) <= 4 \
else [f"{len(ask.countries)} countries"]
words = _codes(ask.countries)
if ask.remote:
words.append("remote")
return ", ".join(words)
def _site(module: Any, ask: readers.Ask, facets: Mapping[str, Mapping[str, str]] | None,
lead: str, roles: str) -> str:
"""The line of a board whose site does not know every value `ask` wants
on an `ids` filter, from the host ids the runner keeps (`facets`, None
before any are kept), as `_runner.series` drops them: what is asked,
then what the site does not know ("DK · SE not on this site · no remote
option"), after `lead` and before `roles`; or, with nothing left to
ask, "site lists none of DK, SE: nothing asked". "" when the site knows
every value, or `lead` is "" and something is still asked."""
if facets is None:
return ""
reads = module.READS.filters
known, missing = list(ask.countries), []
if reads.country == "ids" and ask.countries and "country" in facets:
listed = facets.get("country") or {}
known = [c for c in ask.countries if c in listed]
missing = [c for c in ask.countries if c not in listed]
no_remote = bool(ask.remote) and reads.remote == "ids" and not facets.get("remote")
if not (missing or no_remote):
return ""
if no_remote and not (ask.countries or ask.places) and "remote" not in facets:
# Remote alone, on a site that states no remote facet, is read whole.
return f"all places, no remote option on this site · {roles}" if lead else ""
remote = bool(ask.remote) and not no_remote
if not (known or ask.places or remote):
if not missing:
return "site has no remote option: nothing asked"
named = _codes(missing) + (["remote"] if no_remote else [])
return f"site lists none of {', '.join(named)}: nothing asked"
if not lead:
return ""
words = [", ".join(_codes(known) + list(ask.places) + (["remote"] if remote else []))]
if missing:
words.append(f"{', '.join(_codes(missing))} not on this site")
if no_remote:
words.append("no remote option")
return f"{lead}: {' · '.join(w for w in words if w)} · {roles}"
def _roles(ask: readers.Ask) -> str:
"""The role-type half of the line: the agents' terms the board is
asked for, or all roles."""
@@ -363,14 +448,23 @@ def line(found: Plan, sub: Subscription) -> str:
· all roles", "set here: DE · all roles", "all places, too many countries
to ask · follows agents' terms: engineer", or, for a board read whole,
why: "under 300 roles: read whole", "Greenhouse cannot filter: read
whole"."""
whole". A board with no size yet says it is sized on the next pull. On
a host that filters by its own ids, the line names what the site does
not know once the runner has asked it (`_site`): "follows agents: DK ·
SE not on this site · no remote option · all roles", or "site lists
none of DK, SE: nothing asked"."""
module = readers.HOSTS.get(sub.target.partition(":")[0])
if module is None:
return ""
given, why = found.why(sub)
if why == "unknown":
return f"{_WHOLE[why]}: sized on the next pull"
ask = _runner.ask_for(module, sub.params, found.for_subscription(sub))
roles = _roles(ask)
if _runner.filtered(ask):
lead = {"agents": "follows agents", "override": "set here"}.get(why, "")
if site := _site(module, ask, found.facets.get(sub.target), lead, roles):
return site
if why == "countries":
named = found.wide_names or f"{len(found.wide.countries)} countries"
return f"all countries ({named}) · {roles}"
+29 -10
View File
@@ -46,10 +46,11 @@ def set_board(store: Store, key: str, board: str, *, name: str = "",
check: bool = True, **fields: str) -> tuple[Subscription, int | None, int]:
"""Read company `key` from `board` (`host:slug`): attach it to a company
that has none, replace the one it has, or create the company from `name`
when there is no row. The board is fetched first and refused if its host
is unknown or it does not answer; a board listing zero roles is kept, since
a company between openings is still one to follow. `check=False` skips the
fetch, for the seed, whose boards were read once when it was written.
when there is no row. The board is sized first (`board_size`) and refused
if its host is unknown or it does not answer; a board listing zero roles
is kept, since a company between openings is still one to follow.
`check=False` skips the request, for the seed, whose boards were read
once when it was written; such a board is sized by its first pull.
The row's `ats` and `ats_slug` become the board's host and slug, its
`careers_url` the board's page where it was empty or the old board's, and
@@ -59,7 +60,7 @@ def set_board(store: Store, key: str, board: str, *, name: str = "",
change in one transaction (`Store.set_company_board`), and replacing a
board closes the old one's open roles. Returns the subscription, how many
roles the board lists (None unchecked) and how many roles closed."""
from plugins.ats import careers_url, fetch_board, is_known_target
from plugins.ats import board_size, careers_url, is_known_target, keep_size
board, name = board.strip(), name.strip()
key = key.strip() or company_key(name)
@@ -72,10 +73,10 @@ def set_board(store: Store, key: str, board: str, *, name: str = "",
raise ValueError(f"no company {key!r}; a new company needs a name")
if company_key(name) != key:
raise ValueError(f"{name!r} is not the company {key!r}")
rows: list[Any] | None = None
size: int | None = None
if check:
try:
rows, _complete = fetch_board(board)
size = board_size(board)
except Exception as exc:
raise ValueError(f"{board} did not answer: {exc}") from exc
host, _, slug = board.partition(":")
@@ -95,7 +96,23 @@ def set_board(store: Store, key: str, board: str, *, name: str = "",
assert sub is not None
if before is not None and before.target != board:
_forget_failing(store, before.target)
return sub, None if rows is None else len(rows), closed
if size is not None:
keep_size(store.state, board, size)
return sub, size, closed
def size_board(store: Store, board: str) -> int:
"""Size `board` again (`plugins.ats.board_size`) and keep it with the
day: how a board's size changes once it was added (a Recheck, `app
companies size`). Raises ValueError when it does not answer."""
from plugins.ats import board_size, keep_size
try:
size = board_size(board)
except Exception as exc:
raise ValueError(f"{board} did not answer: {exc}") from exc
keep_size(store.state, board, size)
return size
def detach_board(store: Store, key: str) -> int:
@@ -118,9 +135,11 @@ def _forget_failing(store: Store, target: str) -> None:
record of `target` failing would otherwise outlive the board, and count
in `app companies check` for good. The `ats` plugin's robots.txt verdict
and its lists of roles read and failed in detail go with it, since they
are about that board alone and would only grow stale."""
are about that board alone and would only grow stale, as would its size,
which a board attached again gets anew."""
store.state.pop(FAILING + target, None)
for key in ("ats:robots:", "ats:detailed:", "ats:detailfailed:"):
for key in ("ats:robots:", "ats:detailed:", "ats:detailfailed:", "ats:size:",
"ats:sizechecked:"):
store.state.pop(key + target, None)
+48
View File
@@ -28,6 +28,11 @@
* posted to `/api/sources/{id}/reach`. The wide reach (Europe) and no role
* narrowing are the company's two overrides instead, and the board's own
* countries win over All countries.
*
* Beside that line stands the board's size, set when it was added and
* changed only by a Recheck ("999 roles · checked 30 Sep"). Recheck, beside
* Change, sizes the board again (`/api/sources/{id}/size`) and queues a
* pull, since the size decides whether the board is filtered.
*/
(function () {
"use strict";
@@ -67,6 +72,41 @@
// ---------------------------------------------------------------- the list
/** "30 Sep", or "30 Sep 2025" when it is not this year. A day alone
("2026-09-30") is read as local midnight, not UTC's. */
function checkedOn(iso) {
const text = String(iso || "");
const at = new Date(text.length === 10 ? `${text}T00:00` : text);
if (Number.isNaN(at.getTime())) return "";
const options = { day: "numeric", month: "short" };
if (at.getFullYear() !== new Date().getFullYear()) options.year = "numeric";
return at.toLocaleDateString("en-GB", options);
}
/** The row's Recheck: size the board again, which may change what it is
asked for, and read it with the pull that follows. */
function recheckButton(company, rerender) {
const button = document.createElement("button");
button.type = "button";
button.className = "linky";
button.textContent = "Recheck";
button.title = "Count the board's roles again";
button.addEventListener("click", async () => {
button.disabled = true;
try {
const result = await api(`/api/sources/${company.subscription_id}/size`,
{ method: "POST" });
window.toast.info(`${company.name}: ${count(result.size, "role")}, `
+ `reads ${result.reach}. Reading now, or with the pull right after.`);
await loadCompanies(agentId());
rerender();
} catch (err) {
button.disabled = false;
}
});
return button;
}
function subLine(company, rerender) {
const sub = document.createElement("span");
sub.className = "sub";
@@ -104,6 +144,10 @@
}
sub.append(` · ${count(company.open_roles, "open role")}`);
if (company.reach) sub.append(` · reads ${company.reach}`);
if (company.size != null) {
const checked = checkedOn(company.size_checked);
sub.append(` · ${count(company.size, "role")}${checked ? ` · checked ${checked}` : ""}`);
}
if (company.failing_since) {
// The board has failed on every pull since then; after 21 days its
// roles close.
@@ -115,6 +159,10 @@
sub.appendChild(failing);
}
sub.append(" ");
if (company.subscription_id != null) {
sub.appendChild(recheckButton(company, rerender));
sub.append(" ");
}
const edit = document.createElement("button");
edit.type = "button";
edit.className = "linky";
+27 -1
View File
@@ -491,6 +491,10 @@ class CompanyOut(BaseModel):
#: what a pull asks its board for (`jobs.reach.line`).
reach_params: dict[str, Any] = {}
reach: str = ""
#: The roles the board listed when it was last sized, and that day
#: (ISO); None and "" before it is sized (`jobs.reach.SIZE`).
size: int | None = None
size_checked: str = ""
def _company_boards(store: Store) -> dict[str, tuple[int, str, bool, str]]:
@@ -534,7 +538,10 @@ def _company_outs(store: Store, agent: int | None) -> list[CompanyOut]:
all_roles=bool(c.all_roles),
reach_params={k: v for k, v in (sub.params if sub else {}).items()
if k in jobs.reach.OVERRIDE_KEYS},
reach=jobs.reach.line(plan, sub) if sub else ""))
reach=jobs.reach.line(plan, sub) if sub else "",
size=plan.sizes.get(target) if sub else None,
size_checked=store.state.get(
jobs.reach.SIZE_CHECKED + target, "") if sub else ""))
return out
@@ -897,6 +904,25 @@ def set_reach(sub_id: int, body: ReachBody, store: StoreDep) -> dict[str, Any]:
"reach": jobs.reach.line(jobs.reach.plan(store, [sub]), sub)}
@router.post("/sources/{sub_id}/size")
def recheck_size(sub_id: int, store: StoreDep, queue: QueueDep) -> dict[str, Any]:
"""Size a board again, the one way its size changes once it was added
(`jobs.sources.size_board`): its reach may change with it, so a pull is
queued right after, as the other routes that change what is read do. 422
when the board does not answer; the size it had stays."""
sub = store.subscription(sub_id)
if sub is None or sub.source != "ats":
raise HTTPException(404, f"no such board: {sub_id}")
try:
size = jobs.sources.size_board(store, sub.target)
except ValueError as exc:
raise HTTPException(422, str(exc)) from exc
queue.enqueue("pull")
checked = store.state.get(jobs.reach.SIZE_CHECKED + sub.target, "")
return {"size": size, "size_checked": checked,
"reach": jobs.reach.line(jobs.reach.plan(store, [sub]), sub)}
class ReaderOut(BaseModel):
"""What a reader declares (`plugins/readers/_contract.py`), for the
Sources page: `filters` holds only the dimensions it filters."""