Merge branch 'worktree-agent-ac3c9345edd29a1c6'

# Conflicts:
#	app/cli.py
#	app/jobs/sources.py
#	app/static/app.css
#	app/static/companies.js
#	app/web/api.py
#	tests/test_web.py
This commit is contained in:
Wolf committed 2026-09-29 14:15:10 +02:00
commit c6246af994
21 files changed
+1017 -155

No files matched your search

+29 -11
View File
@@ -87,7 +87,7 @@ def build_parser() -> argparse.ArgumentParser:
discover.add_argument("--add", action="store_true",
help="add or update the company: its board when one is readable, "
"else the host it runs on")
companies_sub.add_parser("check", help="report boards that do not answer")
companies_sub.add_parser("check", help="report boards that do not answer, now or in the pull")
searches = sub.add_parser("searches", help="saved searches on cross-company job boards")
searches_sub = searches.add_subparsers(dest="action", required=True)
@@ -185,8 +185,17 @@ def run_job(args: argparse.Namespace) -> int:
bootstrap=jobs.BOOTSTRAP, bootstrap_arg=str(config.path or ""),
tz=config.timezone, schedules=False)
if chained or args.kind not in JOBS:
task_id = queue.start_task(cascade.chain(args.kind) if chained
else TASKS[args.kind]())
first = cascade.chain(args.kind) if chained else TASKS[args.kind]()
if args.kind == "pull":
# The same claim the `pull` job makes, so a pull here and one in
# the server's worker cannot both fetch.
run = jobs.pull.claim_run(store)
if not run:
print(json.dumps({"task": "pull", "result": jobs.pull.ALREADY_RUNNING,
"running": jobs.pull.running(store)}))
return 1
first = cascade.pull_chain(run)
task_id = queue.start_task(first)
queue.drain()
started = queue.jobs.of_task(task_id)
if not started:
@@ -333,19 +342,28 @@ def detach_board(store: Store, key: str) -> int:
def check_companies(store: Store) -> int:
"""`app companies check`: every `ats` board, fetched once. Prints a
line per board that did not answer and exits 1 if any did not."""
from plugins.ats import fetch_board
line per board that did not answer now, answered incompletely, or has
been failing in the pull, with what the pull recorded beside the live
result, and exits 1 if any did not answer now."""
from app import jobs
from plugins.ats import failure_reason, fetch_board
subs = [s for s in store.subscriptions(source="ats") if s.show]
dead = []
failing = jobs.pull.failing_boards(store)
dead = 0
for sub in subs:
try:
fetch_board(sub.target)
rows, complete = fetch_board(sub.target)
live = f"{len(rows)} roles" + ("" if complete else ", incomplete")
except Exception as exc:
dead.append((sub.target, sub.label, str(exc)))
for target, label, reason in dead:
print(f"{target} ({label}): {reason}")
print(f"checked {len(subs)} boards, {len(dead)} dead")
complete, live = False, f"not answering: {failure_reason(exc)}"
dead += 1
since, reason = failing.get(sub.target, ("", ""))
if complete and not since:
continue
stored = f"pull: failing since {since[:10]}: {reason}" if since else "pull: answering"
print(f"{sub.target} ({sub.label}): now {live}; {stored}")
print(f"checked {len(subs)} boards, {dead} dead, {len(failing)} failing in the pull")
return 1 if dead else 0
+33 -6
View File
@@ -34,7 +34,7 @@ from typing import Any
from app.jobs import embed, extract, pull, score, summarise # noqa: F401 -- registers the jobs
from desk.tools import agent
from pipeline import NULL, TASKS, RunContext, Tally, Task, declare, job
from pipeline import NULL, TASKS, Notice, RunContext, Tally, Task, declare, job
STEP = "app.jobs.cascade:step"
@@ -129,17 +129,44 @@ def chain(name: str) -> Task:
return task(CASCADES[name][0], cascade=name, at=0)
def pull_chain(run: str) -> Task:
"""The pull cascade for a run `pull.claim_run` has claimed: its finder
reads under that marker rather than claiming one of its own."""
return replace(chain("pull"), finder=pull.pull_task(run).finder)
def already_running(since: str) -> Notice:
"""The notice for a pull refused because another is in flight."""
return Notice(
code="pull.running", level="info", kind="expected", key="pull",
text=f"Not started: {pull.ALREADY_RUNNING}"
+ (f" (since {since[:19].replace('T', ' ')} UTC)." if since else "."),
where="app.jobs.pull")
@job("pull", "Pulling every board", resource="http")
def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]:
"""Ask for the pull cascade once this job is done. Its own work is nil.
"""Claim a pull run and ask for the pull cascade once this job is done.
Its own work is nil.
A pull whose claim is refused, because another run is still unsettled,
queues nothing, embed to summarise included, and says so in its result
and a notice: a second cascade would fetch every board again and could
send the same roles to the model twice.
Under `NULL` there is no queue to take the chain, so this raises rather
than report a pull that never happened; `app job pull` starts the chain
on a queue of its own instead of calling this.
than report a pull that never happened; `app job pull` claims the run and
starts the chain on a queue of its own instead of calling this.
"""
if not context.task(chain("pull")):
run = pull.claim_run(store)
if not run:
since = pull.running(store)
context.notice(already_running(since))
return {"skipped": pull.ALREADY_RUNNING, "running": since}
if not context.task(pull_chain(run)):
pull.release_run(store, run)
raise RuntimeError("no queue to start the pull cascade on")
return {"started": " > ".join(CASCADES["pull"])}
return {"started": " > ".join(CASCADES["pull"]), "run": run}
#: What `app job <name>` and `/api/jobs` start besides `pull`: one stage
+149 -39
View File
@@ -7,15 +7,28 @@ sized for exactly this. The final job's `settle` then closes every board a
complete fetch of it did not list again — see `close_boards`.
Closure is read from `self.state`, which each plugin itself stamps per target
on a successful parse (`ats:ok:<target>`, `ats:ids:<target>`, and the same
under `search:`), rather than from what each `pull_batch` job happens to
return. A batch job that raises partway through still leaves the boards it had
already finished stamped, so one bad board in a batch of ten does not cost the
other nine their closure.
on a successful parse (`ats:ok:<target>`, `ats:complete:<target>`,
`ats:ids:<target>`, and the same under `search:`), rather than from what each
`pull_batch` job happens to return. A batch job that raises partway through
still leaves the boards it had already finished stamped, so one bad board in a
batch of ten does not cost the other nine their closure. A board that answered
but was not read to its stated total (a paged host that hit a 429 or the page
cap) stores what it read and closes nothing.
A search closes the same way when its results ran out before its page limit,
since it then listed everything that matches. One that hit the limit did not,
so its roles close only once no search has seen them for `STALE_DAYS`.
A search closes the same way when it listed everything that matches (see
`plugins/search.py` for how each provider knows). One that did not closes its
roles only once no search has seen them for `STALE_DAYS`. A board
that has failed on every fetch for `STALE_DAYS` (`ats:failing:<target>`)
closes all of its roles, and its next answer reopens what it lists.
One pull at a time. A run is claimed in the database (`claim_run`: `pull:run`
set only while the previous run is settled, in one statement), so the
schedule, the Run button and `app job pull` in another process cannot start a
second fetch while one is in flight; a claim older than `ABANDONED` with no
settle is a run that died and is taken over. The marker travels in every
batch's payload, the plugins stamp with the marker they were handed, and the
settle compares against the marker its own batches carried, so neither a
later run's marker nor its id lists reach this run's closure.
"""
from __future__ import annotations
@@ -23,6 +36,7 @@ from __future__ import annotations
import json
import logging
from datetime import UTC, datetime, timedelta
from functools import partial
from typing import Any
from pipeline import NULL, RunContext, Tally, Task, job
@@ -43,37 +57,85 @@ SETTLE = "app.jobs.pull:close_boards"
#: The plugins a pull reads, by the `source` their subscriptions carry.
SOURCES = ("ats", "search")
#: How long a search role stays open once no search lists it any more.
#: How long a search role stays open once no search lists it any more, and
#: how long a board may fail on every fetch before its roles close.
STALE_DAYS = 21
#: The `state` prefix the `ats` plugin records a failing board under:
#: `ats:failing:<target>` -> `{"since", "reason"}`, removed on its next answer.
FAILING = "ats:failing:"
#: The `state` key that records the starter searches were imported.
SEARCHES_SEEDED = "searches:seeded"
#: The marker of the pull run claimed last, and of the one settled last. A run
#: is in flight while the two differ.
RUN = "pull:run"
SETTLED = "pull:settled"
#: How old an unsettled claim may be before it counts as a run that died (a
#: killed process, a killed cascade) and the next pull takes over. A pull of
#: every board takes minutes.
ABANDONED = timedelta(hours=1)
#: What a refused pull says, in its result and its notice.
ALREADY_RUNNING = "a pull is already running"
def _now() -> str:
return datetime.now(UTC).isoformat(timespec="seconds")
"""A run marker: to the microsecond, so two pulls a second apart are two
runs and a stamp from one never matches the other."""
return datetime.now(UTC).isoformat(timespec="microseconds")
def _boards(config: Config, store: Store) -> list[dict[str, Any]]:
def claim_run(store: Store) -> str:
"""Start a pull run: a new marker in `pull:run`, returned, unless another
run is still unsettled and younger than `ABANDONED`, which returns "".
One conditional write, so two starters in two processes cannot both win."""
marker = _now()
stale = (datetime.now(UTC) - ABANDONED).isoformat(timespec="microseconds")
return marker if store.claim_state(RUN, marker, released=SETTLED, stale=stale) else ""
def running(store: Store) -> str:
"""The marker of the pull run in flight, or "" when the last one settled."""
run = store.state.get(RUN, "")
return run if run and run != store.state.get(SETTLED, "") else ""
def release_run(store: Store, run: str) -> None:
"""Mark `run` settled, unless a later run has taken over since: letting go
of a claim that is no longer ours would release the one that is."""
if store.state.get(RUN, "") == run:
store.state[SETTLED] = run
def _boards(config: Config, store: Store, run: str = "") -> list[dict[str, Any]]:
"""Every fetchable `ats` and `search` subscription, ten to a batch of one
source. Stamps the run marker `close_boards` will look for, before any
batch runs — finders run synchronously, in whoever starts the task, ahead
of every job it queues."""
source, each batch carrying the run marker its plugin stamps with and
`close_boards` compares against. `run` is a marker the caller claimed
already (the `pull` job, `app job pull`); without one the finder claims
its own, and queues nothing when another run is in flight. A run with
nothing to read is settled here, since no settle will see a batch of it."""
seed_searches(config, store)
batches: list[dict[str, Any]] = []
for source in SOURCES:
subs = store.subscriptions(source=source, fetchable=True)
if not subs:
continue
store.state[f"{source}:run"] = _now()
batches += [{"targets": [s.target for s in subs[i:i + BATCH_SIZE]], "source": source}
for i in range(0, len(subs), BATCH_SIZE)]
return batches
targets = [(source, [s.target for s in store.subscriptions(source=source, fetchable=True)])
for source in SOURCES]
if not any(found for _, found in targets):
if run:
release_run(store, run)
return []
run = run or claim_run(store)
if not run:
log.warning("pull: %s since %s; this one reads nothing", ALREADY_RUNNING,
running(store))
return []
return [{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run}
for source, found in targets for i in range(0, len(found), BATCH_SIZE)]
@job("pull_batch", "Reading boards", resource="http")
def run_batch(config: Config, store: Store, *, targets: list[str], source: str = "ats",
context: RunContext = NULL) -> dict[str, Any]:
run: str = "", context: RunContext = NULL) -> dict[str, Any]:
"""Fetch this batch's boards or searches and `upsert_role` every item
they list.
@@ -85,6 +147,10 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
reads `options[subscription_field]` — so one job never re-fetches another
job's boards, and `context.check()` runs before each fetch: a Kill lands
between two boards rather than after the tenth.
`run` is the marker of the run that queued this batch, handed to the
plugin as its `run` option, so its stamps belong to that run whatever the
stored marker says by the time the batch runs.
"""
from plugins.ats import ATSSource
from plugins.search import SearchSource
@@ -99,7 +165,7 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
if sub is None:
continue
context.check()
options = dict(config.sources.get(source) or {})
options = dict(config.sources.get(source) or {}) | {"run": run}
plugin: ATSSource | SearchSource
if source == "search":
options["searches"] = [{"url": target, "label": sub.label, "params": sub.params}]
@@ -112,7 +178,18 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
item.source = source
store.upsert_role(item, sub)
stored += 1
return {"roles": stored}
return {"roles": stored, "run": run}
def _run_of(tally: Tally, context: RunContext) -> str:
"""The marker this settle's own batches were queued with: off their
payloads, which a batch that raised still has, else off their results
(the only thing a settle called directly is given)."""
for batch in context.waited_for:
run = (getattr(batch, "payload", None) or {}).get("run")
if run:
return str(run)
return next((str(r["run"]) for r in tally.results if r.get("run")), "")
def close_boards(config: Config, store: Store, tally: Tally, *,
@@ -124,38 +201,70 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
`pull_batch` job that raised partway through still leaves the boards it
had already parsed stamped — see the module docstring. A board whose
`ats:ok:<target>` was not written during *this* run — it failed, or its
whole batch job never got to it — is never closed. A search closes by its
list only when that list was complete, and otherwise by `STALE_DAYS`.
whole batch job never got to it — is never closed by its list, and a board
that answered incompletely is not either. A search closes by its list only
when that list was complete, and otherwise by `STALE_DAYS`. A board that
has failed on every fetch since `STALE_DAYS` ago closes all its roles.
The stamps are compared with this run's own marker, the one its batches
carried, never with the stored `pull:run`, which a later run may have
replaced. A settle with no batch behind it (a refused pull, nothing to
read) closes nothing by list or by age. Last, the run is marked settled,
which lets the next pull start.
A board or search switched off while its batch was in flight had its
roles reopened by that batch, after the switch closed them, so every
switched-off one is closed again here.
"""
answered = searched = failed = closed = 0
answered = searched = failed = incomplete = closed = 0
stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds")
failing = failing_boards(store)
run_marker = _run_of(tally, context)
for source in SOURCES:
for sub in store.subscriptions(source=source):
if not (sub.subscribed and sub.fetch):
closed += store.close_subscription(sub.id)
run_marker = store.state.get(f"{source}:run", "")
if not run_marker:
continue
for sub in store.subscriptions(source=source, fetchable=True):
stamp = store.state.get(f"{source}:ok:{sub.target}", "")
if not (stamp and stamp == run_marker):
failed += 1
since = failing.get(sub.target, ("", ""))[0] if source == "ats" else ""
if since and since < stale:
closed += store.close_subscription(sub.id)
continue
if source == "search":
searched += 1
if not store.state.get(f"search:complete:{sub.target}", ""):
continue
else:
answered += 1
if not store.state.get(f"{source}:complete:{sub.target}", ""):
if source == "ats":
incomplete += 1
continue
seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]"))
closed += store.mark_closed(source, seen_ids, sub.id)
if searched:
stale = datetime.now(UTC) - timedelta(days=STALE_DAYS)
closed += store.close_unseen("search", stale.isoformat(timespec="seconds"))
closed += store.close_unseen("search", stale)
if run_marker:
release_run(store, run_marker)
roles = sum(int(result.get("roles", 0)) for result in tally.results)
return {"boards": answered, "searches": searched, "failed": failed, "roles": roles,
"closed": closed}
return {"boards": answered, "searches": searched, "failed": failed,
"incomplete": incomplete, "roles": roles, "closed": closed}
def failing_boards(store: Store) -> dict[str, tuple[str, str]]:
"""`target -> (since, reason)` for every `ats` board whose last fetch
failed: when the failures began, and what the latest one said."""
out: dict[str, tuple[str, str]] = {}
for key, value in store.state_items(FAILING).items():
try:
record = json.loads(value)
except ValueError:
continue
out[key[len(FAILING):]] = (str(record.get("since") or ""),
str(record.get("reason") or ""))
return out
def search_target(provider: str, name: str) -> str:
@@ -184,8 +293,9 @@ def seed_searches(config: Config, store: Store) -> list[str]:
return added
def pull_task() -> Task:
def pull_task(run: str = "") -> Task:
"""The pull on its own. `app.jobs.cascade` makes it the first stage
of the chain the `pull` job starts."""
return Task(kind="pull_batch", finder=_boards, name="pull",
of the chain the `pull` job starts. `run` is a marker `claim_run` already
gave the caller; without one the finder claims its own."""
return Task(kind="pull_batch", finder=partial(_boards, run=run), name="pull",
label="Reading every board", settle=SETTLE)
+1 -1
View File
@@ -68,7 +68,7 @@ def set_board(store: Store, key: str, board: str, *, name: str = "",
rows: list[Any] | None = None
if check:
try:
rows = fetch_board(board)
rows, _complete = fetch_board(board)
except Exception as exc:
raise ValueError(f"{board} did not answer: {exc}") from exc
host, _, slug = board.partition(":")
+1
View File
@@ -1211,6 +1211,7 @@ table.applications td[data-label="Saved"] { white-space: nowrap; color: var(--di
border-radius: .3rem; padding: .35rem .5rem;
}
.crow .board-edit input { flex: 1 1 8rem; min-width: 0; }
.crow .sub .failing { color: var(--warn); }
.linky {
padding: 0; border: 0; background: none; font-size: .8rem;
+12 -1
View File
@@ -85,7 +85,18 @@
sub.append(" · ");
sub.appendChild(link);
}
sub.append(` · ${count(company.open_roles, "open role")} `);
sub.append(` · ${count(company.open_roles, "open role")}`);
if (company.failing_since) {
// The board has failed on every pull since then; after 21 days its
// roles close.
const failing = document.createElement("span");
failing.className = "failing";
failing.textContent = `not answering since ${company.failing_since.slice(0, 10)}`
+ (company.failing_reason ? `: ${company.failing_reason}` : "");
sub.append(" · ");
sub.appendChild(failing);
}
sub.append(" ");
const edit = document.createElement("button");
edit.type = "button";
edit.className = "linky";
+20
View File
@@ -1319,6 +1319,26 @@ class Store:
`self.state`: it can keep a counter without being able to read roles."""
return _StateMap(self)
def state_items(self, prefix: str) -> dict[str, str]:
"""Every `state` key that starts with `prefix`, with its value, in one
query, for a reader that wants one kind of key across every target."""
return {str(r["key"]): str(r["value"]) for r in self._rows(
"SELECT key, value FROM state WHERE substr(key, 1, ?) = ? ORDER BY key",
(len(prefix), prefix))}
def claim_state(self, key: str, value: str, *, released: str, stale: str) -> bool:
"""Set `key` to `value` only while it is free: unset, equal to the
value under `released` (the claim last let go of), or no later than
`stale`. Returns whether it was set. One statement, so of two
processes claiming at once exactly one wins."""
return self._write(
"""INSERT INTO state (key, value) VALUES (?, ?)
ON CONFLICT(key) DO UPDATE SET value = excluded.value
WHERE state.value <= ?
OR state.value = COALESCE(
(SELECT r.value FROM state AS r WHERE r.key = ?), '')""",
(key, value, stale, released)) > 0
def _check_status(status: str) -> None:
if status not in STATUSES:
+15 -8
View File
@@ -265,7 +265,9 @@ class CompanyOut(BaseModel):
still open. `ats` is the host its careers site runs on whether or not the
app reads it, and `ats_slug` its name there, so a followed company with
no `board` and an `ats` is one the app cannot read yet; `careers_url` is
where a person can."""
where a person can. `failing_since` and `failing_reason` say since when
the board has failed on every fetch and why it last did, both empty
while it answers."""
key: str
name: str
@@ -278,20 +280,22 @@ class CompanyOut(BaseModel):
ats_slug: str = ""
careers_url: str = ""
country: str = ""
failing_since: str = ""
failing_reason: str = ""
def _company_boards(store: Store) -> dict[str, tuple[int, str, bool]]:
"""`company_key -> (subscription id, ats host, on)`, read off each
def _company_boards(store: Store) -> dict[str, tuple[int, str, bool, str]]:
"""`company_key -> (subscription id, ats host, on, target)`, read off each
company's own subscription. Same join as `Store.company_boards`, with the
id and switch state a company row needs to draw its own on/off control
rather than fetching `/api/sources` as well. A detached board (`show`
0) is no board."""
boards: dict[str, tuple[int, str, bool]] = {}
boards: dict[str, tuple[int, str, bool, str]] = {}
for sub in store.subscriptions(source="ats"):
if not sub.show:
continue
boards.setdefault(company_key(sub.company), (sub.id, sub.board,
bool(sub.subscribed and sub.fetch)))
boards.setdefault(company_key(sub.company), (
sub.id, sub.board, bool(sub.subscribed and sub.fetch), sub.target))
return boards
@@ -301,14 +305,17 @@ def list_companies(store: StoreDep, agent: int | None = None) -> list[CompanyOut
absent) has it; 0 throughout when there is no agent."""
boards = _company_boards(store)
counts = store.open_role_counts()
failing = jobs.pull.failing_boards(store)
out = []
for c in store.companies(_agent_id(store, agent)):
sub_id, host, on = boards.get(c.key, (None, "", None))
sub_id, host, on, target = boards.get(c.key, (None, "", None, ""))
since, reason = failing.get(target, ("", ""))
out.append(CompanyOut(key=c.key, name=c.name, board=host,
subscription_id=sub_id, on=on, affinity=c.affinity,
open_roles=counts.get(c.key, 0), ats=c.ats,
ats_slug=c.ats_slug, careers_url=c.careers_url,
country=c.country))
country=c.country,
failing_since=since, failing_reason=reason))
return out