Merge branch 'worktree-agent-ac3c9345edd29a1c6'
# Conflicts: # app/cli.py # app/jobs/sources.py # app/static/app.css # app/static/companies.js # app/web/api.py # tests/test_web.py
This commit is contained in:
commit
c6246af994
21 files changed
+1017
-155
No files matched your search
+29
-11
@@ -87,7 +87,7 @@ def build_parser() -> argparse.ArgumentParser:
|
||||
discover.add_argument("--add", action="store_true",
|
||||
help="add or update the company: its board when one is readable, "
|
||||
"else the host it runs on")
|
||||
companies_sub.add_parser("check", help="report boards that do not answer")
|
||||
companies_sub.add_parser("check", help="report boards that do not answer, now or in the pull")
|
||||
|
||||
searches = sub.add_parser("searches", help="saved searches on cross-company job boards")
|
||||
searches_sub = searches.add_subparsers(dest="action", required=True)
|
||||
@@ -185,8 +185,17 @@ def run_job(args: argparse.Namespace) -> int:
|
||||
bootstrap=jobs.BOOTSTRAP, bootstrap_arg=str(config.path or ""),
|
||||
tz=config.timezone, schedules=False)
|
||||
if chained or args.kind not in JOBS:
|
||||
task_id = queue.start_task(cascade.chain(args.kind) if chained
|
||||
else TASKS[args.kind]())
|
||||
first = cascade.chain(args.kind) if chained else TASKS[args.kind]()
|
||||
if args.kind == "pull":
|
||||
# The same claim the `pull` job makes, so a pull here and one in
|
||||
# the server's worker cannot both fetch.
|
||||
run = jobs.pull.claim_run(store)
|
||||
if not run:
|
||||
print(json.dumps({"task": "pull", "result": jobs.pull.ALREADY_RUNNING,
|
||||
"running": jobs.pull.running(store)}))
|
||||
return 1
|
||||
first = cascade.pull_chain(run)
|
||||
task_id = queue.start_task(first)
|
||||
queue.drain()
|
||||
started = queue.jobs.of_task(task_id)
|
||||
if not started:
|
||||
@@ -333,19 +342,28 @@ def detach_board(store: Store, key: str) -> int:
|
||||
|
||||
def check_companies(store: Store) -> int:
|
||||
"""`app companies check`: every `ats` board, fetched once. Prints a
|
||||
line per board that did not answer and exits 1 if any did not."""
|
||||
from plugins.ats import fetch_board
|
||||
line per board that did not answer now, answered incompletely, or has
|
||||
been failing in the pull, with what the pull recorded beside the live
|
||||
result, and exits 1 if any did not answer now."""
|
||||
from app import jobs
|
||||
from plugins.ats import failure_reason, fetch_board
|
||||
|
||||
subs = [s for s in store.subscriptions(source="ats") if s.show]
|
||||
dead = []
|
||||
failing = jobs.pull.failing_boards(store)
|
||||
dead = 0
|
||||
for sub in subs:
|
||||
try:
|
||||
fetch_board(sub.target)
|
||||
rows, complete = fetch_board(sub.target)
|
||||
live = f"{len(rows)} roles" + ("" if complete else ", incomplete")
|
||||
except Exception as exc:
|
||||
dead.append((sub.target, sub.label, str(exc)))
|
||||
for target, label, reason in dead:
|
||||
print(f"{target} ({label}): {reason}")
|
||||
print(f"checked {len(subs)} boards, {len(dead)} dead")
|
||||
complete, live = False, f"not answering: {failure_reason(exc)}"
|
||||
dead += 1
|
||||
since, reason = failing.get(sub.target, ("", ""))
|
||||
if complete and not since:
|
||||
continue
|
||||
stored = f"pull: failing since {since[:10]}: {reason}" if since else "pull: answering"
|
||||
print(f"{sub.target} ({sub.label}): now {live}; {stored}")
|
||||
print(f"checked {len(subs)} boards, {dead} dead, {len(failing)} failing in the pull")
|
||||
return 1 if dead else 0
|
||||
|
||||
|
||||
|
||||
+33
-6
@@ -34,7 +34,7 @@ from typing import Any
|
||||
|
||||
from app.jobs import embed, extract, pull, score, summarise # noqa: F401 -- registers the jobs
|
||||
from desk.tools import agent
|
||||
from pipeline import NULL, TASKS, RunContext, Tally, Task, declare, job
|
||||
from pipeline import NULL, TASKS, Notice, RunContext, Tally, Task, declare, job
|
||||
|
||||
STEP = "app.jobs.cascade:step"
|
||||
|
||||
@@ -129,17 +129,44 @@ def chain(name: str) -> Task:
|
||||
return task(CASCADES[name][0], cascade=name, at=0)
|
||||
|
||||
|
||||
def pull_chain(run: str) -> Task:
|
||||
"""The pull cascade for a run `pull.claim_run` has claimed: its finder
|
||||
reads under that marker rather than claiming one of its own."""
|
||||
return replace(chain("pull"), finder=pull.pull_task(run).finder)
|
||||
|
||||
|
||||
def already_running(since: str) -> Notice:
|
||||
"""The notice for a pull refused because another is in flight."""
|
||||
return Notice(
|
||||
code="pull.running", level="info", kind="expected", key="pull",
|
||||
text=f"Not started: {pull.ALREADY_RUNNING}"
|
||||
+ (f" (since {since[:19].replace('T', ' ')} UTC)." if since else "."),
|
||||
where="app.jobs.pull")
|
||||
|
||||
|
||||
@job("pull", "Pulling every board", resource="http")
|
||||
def run_pull(config: Any, store: Any, *, context: RunContext = NULL) -> dict[str, Any]:
|
||||
"""Ask for the pull cascade once this job is done. Its own work is nil.
|
||||
"""Claim a pull run and ask for the pull cascade once this job is done.
|
||||
Its own work is nil.
|
||||
|
||||
A pull whose claim is refused, because another run is still unsettled,
|
||||
queues nothing, embed to summarise included, and says so in its result
|
||||
and a notice: a second cascade would fetch every board again and could
|
||||
send the same roles to the model twice.
|
||||
|
||||
Under `NULL` there is no queue to take the chain, so this raises rather
|
||||
than report a pull that never happened; `app job pull` starts the chain
|
||||
on a queue of its own instead of calling this.
|
||||
than report a pull that never happened; `app job pull` claims the run and
|
||||
starts the chain on a queue of its own instead of calling this.
|
||||
"""
|
||||
if not context.task(chain("pull")):
|
||||
run = pull.claim_run(store)
|
||||
if not run:
|
||||
since = pull.running(store)
|
||||
context.notice(already_running(since))
|
||||
return {"skipped": pull.ALREADY_RUNNING, "running": since}
|
||||
if not context.task(pull_chain(run)):
|
||||
pull.release_run(store, run)
|
||||
raise RuntimeError("no queue to start the pull cascade on")
|
||||
return {"started": " > ".join(CASCADES["pull"])}
|
||||
return {"started": " > ".join(CASCADES["pull"]), "run": run}
|
||||
|
||||
|
||||
#: What `app job <name>` and `/api/jobs` start besides `pull`: one stage
|
||||
|
||||
+149
-39
@@ -7,15 +7,28 @@ sized for exactly this. The final job's `settle` then closes every board a
|
||||
complete fetch of it did not list again — see `close_boards`.
|
||||
|
||||
Closure is read from `self.state`, which each plugin itself stamps per target
|
||||
on a successful parse (`ats:ok:<target>`, `ats:ids:<target>`, and the same
|
||||
under `search:`), rather than from what each `pull_batch` job happens to
|
||||
return. A batch job that raises partway through still leaves the boards it had
|
||||
already finished stamped, so one bad board in a batch of ten does not cost the
|
||||
other nine their closure.
|
||||
on a successful parse (`ats:ok:<target>`, `ats:complete:<target>`,
|
||||
`ats:ids:<target>`, and the same under `search:`), rather than from what each
|
||||
`pull_batch` job happens to return. A batch job that raises partway through
|
||||
still leaves the boards it had already finished stamped, so one bad board in a
|
||||
batch of ten does not cost the other nine their closure. A board that answered
|
||||
but was not read to its stated total (a paged host that hit a 429 or the page
|
||||
cap) stores what it read and closes nothing.
|
||||
|
||||
A search closes the same way when its results ran out before its page limit,
|
||||
since it then listed everything that matches. One that hit the limit did not,
|
||||
so its roles close only once no search has seen them for `STALE_DAYS`.
|
||||
A search closes the same way when it listed everything that matches (see
|
||||
`plugins/search.py` for how each provider knows). One that did not closes its
|
||||
roles only once no search has seen them for `STALE_DAYS`. A board
|
||||
that has failed on every fetch for `STALE_DAYS` (`ats:failing:<target>`)
|
||||
closes all of its roles, and its next answer reopens what it lists.
|
||||
|
||||
One pull at a time. A run is claimed in the database (`claim_run`: `pull:run`
|
||||
set only while the previous run is settled, in one statement), so the
|
||||
schedule, the Run button and `app job pull` in another process cannot start a
|
||||
second fetch while one is in flight; a claim older than `ABANDONED` with no
|
||||
settle is a run that died and is taken over. The marker travels in every
|
||||
batch's payload, the plugins stamp with the marker they were handed, and the
|
||||
settle compares against the marker its own batches carried, so neither a
|
||||
later run's marker nor its id lists reach this run's closure.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -23,6 +36,7 @@ from __future__ import annotations
|
||||
import json
|
||||
import logging
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from functools import partial
|
||||
from typing import Any
|
||||
|
||||
from pipeline import NULL, RunContext, Tally, Task, job
|
||||
@@ -43,37 +57,85 @@ SETTLE = "app.jobs.pull:close_boards"
|
||||
#: The plugins a pull reads, by the `source` their subscriptions carry.
|
||||
SOURCES = ("ats", "search")
|
||||
|
||||
#: How long a search role stays open once no search lists it any more.
|
||||
#: How long a search role stays open once no search lists it any more, and
|
||||
#: how long a board may fail on every fetch before its roles close.
|
||||
STALE_DAYS = 21
|
||||
|
||||
#: The `state` prefix the `ats` plugin records a failing board under:
|
||||
#: `ats:failing:<target>` -> `{"since", "reason"}`, removed on its next answer.
|
||||
FAILING = "ats:failing:"
|
||||
|
||||
#: The `state` key that records the starter searches were imported.
|
||||
SEARCHES_SEEDED = "searches:seeded"
|
||||
|
||||
#: The marker of the pull run claimed last, and of the one settled last. A run
|
||||
#: is in flight while the two differ.
|
||||
RUN = "pull:run"
|
||||
SETTLED = "pull:settled"
|
||||
|
||||
#: How old an unsettled claim may be before it counts as a run that died (a
|
||||
#: killed process, a killed cascade) and the next pull takes over. A pull of
|
||||
#: every board takes minutes.
|
||||
ABANDONED = timedelta(hours=1)
|
||||
|
||||
#: What a refused pull says, in its result and its notice.
|
||||
ALREADY_RUNNING = "a pull is already running"
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now(UTC).isoformat(timespec="seconds")
|
||||
"""A run marker: to the microsecond, so two pulls a second apart are two
|
||||
runs and a stamp from one never matches the other."""
|
||||
return datetime.now(UTC).isoformat(timespec="microseconds")
|
||||
|
||||
|
||||
def _boards(config: Config, store: Store) -> list[dict[str, Any]]:
|
||||
def claim_run(store: Store) -> str:
|
||||
"""Start a pull run: a new marker in `pull:run`, returned, unless another
|
||||
run is still unsettled and younger than `ABANDONED`, which returns "".
|
||||
One conditional write, so two starters in two processes cannot both win."""
|
||||
marker = _now()
|
||||
stale = (datetime.now(UTC) - ABANDONED).isoformat(timespec="microseconds")
|
||||
return marker if store.claim_state(RUN, marker, released=SETTLED, stale=stale) else ""
|
||||
|
||||
|
||||
def running(store: Store) -> str:
|
||||
"""The marker of the pull run in flight, or "" when the last one settled."""
|
||||
run = store.state.get(RUN, "")
|
||||
return run if run and run != store.state.get(SETTLED, "") else ""
|
||||
|
||||
|
||||
def release_run(store: Store, run: str) -> None:
|
||||
"""Mark `run` settled, unless a later run has taken over since: letting go
|
||||
of a claim that is no longer ours would release the one that is."""
|
||||
if store.state.get(RUN, "") == run:
|
||||
store.state[SETTLED] = run
|
||||
|
||||
|
||||
def _boards(config: Config, store: Store, run: str = "") -> list[dict[str, Any]]:
|
||||
"""Every fetchable `ats` and `search` subscription, ten to a batch of one
|
||||
source. Stamps the run marker `close_boards` will look for, before any
|
||||
batch runs — finders run synchronously, in whoever starts the task, ahead
|
||||
of every job it queues."""
|
||||
source, each batch carrying the run marker its plugin stamps with and
|
||||
`close_boards` compares against. `run` is a marker the caller claimed
|
||||
already (the `pull` job, `app job pull`); without one the finder claims
|
||||
its own, and queues nothing when another run is in flight. A run with
|
||||
nothing to read is settled here, since no settle will see a batch of it."""
|
||||
seed_searches(config, store)
|
||||
batches: list[dict[str, Any]] = []
|
||||
for source in SOURCES:
|
||||
subs = store.subscriptions(source=source, fetchable=True)
|
||||
if not subs:
|
||||
continue
|
||||
store.state[f"{source}:run"] = _now()
|
||||
batches += [{"targets": [s.target for s in subs[i:i + BATCH_SIZE]], "source": source}
|
||||
for i in range(0, len(subs), BATCH_SIZE)]
|
||||
return batches
|
||||
targets = [(source, [s.target for s in store.subscriptions(source=source, fetchable=True)])
|
||||
for source in SOURCES]
|
||||
if not any(found for _, found in targets):
|
||||
if run:
|
||||
release_run(store, run)
|
||||
return []
|
||||
run = run or claim_run(store)
|
||||
if not run:
|
||||
log.warning("pull: %s since %s; this one reads nothing", ALREADY_RUNNING,
|
||||
running(store))
|
||||
return []
|
||||
return [{"targets": found[i:i + BATCH_SIZE], "source": source, "run": run}
|
||||
for source, found in targets for i in range(0, len(found), BATCH_SIZE)]
|
||||
|
||||
|
||||
@job("pull_batch", "Reading boards", resource="http")
|
||||
def run_batch(config: Config, store: Store, *, targets: list[str], source: str = "ats",
|
||||
context: RunContext = NULL) -> dict[str, Any]:
|
||||
run: str = "", context: RunContext = NULL) -> dict[str, Any]:
|
||||
"""Fetch this batch's boards or searches and `upsert_role` every item
|
||||
they list.
|
||||
|
||||
@@ -85,6 +147,10 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
|
||||
reads `options[subscription_field]` — so one job never re-fetches another
|
||||
job's boards, and `context.check()` runs before each fetch: a Kill lands
|
||||
between two boards rather than after the tenth.
|
||||
|
||||
`run` is the marker of the run that queued this batch, handed to the
|
||||
plugin as its `run` option, so its stamps belong to that run whatever the
|
||||
stored marker says by the time the batch runs.
|
||||
"""
|
||||
from plugins.ats import ATSSource
|
||||
from plugins.search import SearchSource
|
||||
@@ -99,7 +165,7 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
|
||||
if sub is None:
|
||||
continue
|
||||
context.check()
|
||||
options = dict(config.sources.get(source) or {})
|
||||
options = dict(config.sources.get(source) or {}) | {"run": run}
|
||||
plugin: ATSSource | SearchSource
|
||||
if source == "search":
|
||||
options["searches"] = [{"url": target, "label": sub.label, "params": sub.params}]
|
||||
@@ -112,7 +178,18 @@ def run_batch(config: Config, store: Store, *, targets: list[str], source: str =
|
||||
item.source = source
|
||||
store.upsert_role(item, sub)
|
||||
stored += 1
|
||||
return {"roles": stored}
|
||||
return {"roles": stored, "run": run}
|
||||
|
||||
|
||||
def _run_of(tally: Tally, context: RunContext) -> str:
|
||||
"""The marker this settle's own batches were queued with: off their
|
||||
payloads, which a batch that raised still has, else off their results
|
||||
(the only thing a settle called directly is given)."""
|
||||
for batch in context.waited_for:
|
||||
run = (getattr(batch, "payload", None) or {}).get("run")
|
||||
if run:
|
||||
return str(run)
|
||||
return next((str(r["run"]) for r in tally.results if r.get("run")), "")
|
||||
|
||||
|
||||
def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||
@@ -124,38 +201,70 @@ def close_boards(config: Config, store: Store, tally: Tally, *,
|
||||
`pull_batch` job that raised partway through still leaves the boards it
|
||||
had already parsed stamped — see the module docstring. A board whose
|
||||
`ats:ok:<target>` was not written during *this* run — it failed, or its
|
||||
whole batch job never got to it — is never closed. A search closes by its
|
||||
list only when that list was complete, and otherwise by `STALE_DAYS`.
|
||||
whole batch job never got to it — is never closed by its list, and a board
|
||||
that answered incompletely is not either. A search closes by its list only
|
||||
when that list was complete, and otherwise by `STALE_DAYS`. A board that
|
||||
has failed on every fetch since `STALE_DAYS` ago closes all its roles.
|
||||
|
||||
The stamps are compared with this run's own marker, the one its batches
|
||||
carried, never with the stored `pull:run`, which a later run may have
|
||||
replaced. A settle with no batch behind it (a refused pull, nothing to
|
||||
read) closes nothing by list or by age. Last, the run is marked settled,
|
||||
which lets the next pull start.
|
||||
|
||||
A board or search switched off while its batch was in flight had its
|
||||
roles reopened by that batch, after the switch closed them, so every
|
||||
switched-off one is closed again here.
|
||||
"""
|
||||
answered = searched = failed = closed = 0
|
||||
answered = searched = failed = incomplete = closed = 0
|
||||
stale = (datetime.now(UTC) - timedelta(days=STALE_DAYS)).isoformat(timespec="seconds")
|
||||
failing = failing_boards(store)
|
||||
run_marker = _run_of(tally, context)
|
||||
for source in SOURCES:
|
||||
for sub in store.subscriptions(source=source):
|
||||
if not (sub.subscribed and sub.fetch):
|
||||
closed += store.close_subscription(sub.id)
|
||||
run_marker = store.state.get(f"{source}:run", "")
|
||||
if not run_marker:
|
||||
continue
|
||||
for sub in store.subscriptions(source=source, fetchable=True):
|
||||
stamp = store.state.get(f"{source}:ok:{sub.target}", "")
|
||||
if not (stamp and stamp == run_marker):
|
||||
failed += 1
|
||||
since = failing.get(sub.target, ("", ""))[0] if source == "ats" else ""
|
||||
if since and since < stale:
|
||||
closed += store.close_subscription(sub.id)
|
||||
continue
|
||||
if source == "search":
|
||||
searched += 1
|
||||
if not store.state.get(f"search:complete:{sub.target}", ""):
|
||||
continue
|
||||
else:
|
||||
answered += 1
|
||||
if not store.state.get(f"{source}:complete:{sub.target}", ""):
|
||||
if source == "ats":
|
||||
incomplete += 1
|
||||
continue
|
||||
seen_ids = json.loads(store.state.get(f"{source}:ids:{sub.target}", "[]"))
|
||||
closed += store.mark_closed(source, seen_ids, sub.id)
|
||||
if searched:
|
||||
stale = datetime.now(UTC) - timedelta(days=STALE_DAYS)
|
||||
closed += store.close_unseen("search", stale.isoformat(timespec="seconds"))
|
||||
closed += store.close_unseen("search", stale)
|
||||
if run_marker:
|
||||
release_run(store, run_marker)
|
||||
roles = sum(int(result.get("roles", 0)) for result in tally.results)
|
||||
return {"boards": answered, "searches": searched, "failed": failed, "roles": roles,
|
||||
"closed": closed}
|
||||
return {"boards": answered, "searches": searched, "failed": failed,
|
||||
"incomplete": incomplete, "roles": roles, "closed": closed}
|
||||
|
||||
|
||||
def failing_boards(store: Store) -> dict[str, tuple[str, str]]:
|
||||
"""`target -> (since, reason)` for every `ats` board whose last fetch
|
||||
failed: when the failures began, and what the latest one said."""
|
||||
out: dict[str, tuple[str, str]] = {}
|
||||
for key, value in store.state_items(FAILING).items():
|
||||
try:
|
||||
record = json.loads(value)
|
||||
except ValueError:
|
||||
continue
|
||||
out[key[len(FAILING):]] = (str(record.get("since") or ""),
|
||||
str(record.get("reason") or ""))
|
||||
return out
|
||||
|
||||
|
||||
def search_target(provider: str, name: str) -> str:
|
||||
@@ -184,8 +293,9 @@ def seed_searches(config: Config, store: Store) -> list[str]:
|
||||
return added
|
||||
|
||||
|
||||
def pull_task() -> Task:
|
||||
def pull_task(run: str = "") -> Task:
|
||||
"""The pull on its own. `app.jobs.cascade` makes it the first stage
|
||||
of the chain the `pull` job starts."""
|
||||
return Task(kind="pull_batch", finder=_boards, name="pull",
|
||||
of the chain the `pull` job starts. `run` is a marker `claim_run` already
|
||||
gave the caller; without one the finder claims its own."""
|
||||
return Task(kind="pull_batch", finder=partial(_boards, run=run), name="pull",
|
||||
label="Reading every board", settle=SETTLE)
|
||||
+1
-1
@@ -68,7 +68,7 @@ def set_board(store: Store, key: str, board: str, *, name: str = "",
|
||||
rows: list[Any] | None = None
|
||||
if check:
|
||||
try:
|
||||
rows = fetch_board(board)
|
||||
rows, _complete = fetch_board(board)
|
||||
except Exception as exc:
|
||||
raise ValueError(f"{board} did not answer: {exc}") from exc
|
||||
host, _, slug = board.partition(":")
|
||||
|
||||
@@ -1211,6 +1211,7 @@ table.applications td[data-label="Saved"] { white-space: nowrap; color: var(--di
|
||||
border-radius: .3rem; padding: .35rem .5rem;
|
||||
}
|
||||
.crow .board-edit input { flex: 1 1 8rem; min-width: 0; }
|
||||
.crow .sub .failing { color: var(--warn); }
|
||||
|
||||
.linky {
|
||||
padding: 0; border: 0; background: none; font-size: .8rem;
|
||||
|
||||
+12
-1
@@ -85,7 +85,18 @@
|
||||
sub.append(" · ");
|
||||
sub.appendChild(link);
|
||||
}
|
||||
sub.append(` · ${count(company.open_roles, "open role")} `);
|
||||
sub.append(` · ${count(company.open_roles, "open role")}`);
|
||||
if (company.failing_since) {
|
||||
// The board has failed on every pull since then; after 21 days its
|
||||
// roles close.
|
||||
const failing = document.createElement("span");
|
||||
failing.className = "failing";
|
||||
failing.textContent = `not answering since ${company.failing_since.slice(0, 10)}`
|
||||
+ (company.failing_reason ? `: ${company.failing_reason}` : "");
|
||||
sub.append(" · ");
|
||||
sub.appendChild(failing);
|
||||
}
|
||||
sub.append(" ");
|
||||
const edit = document.createElement("button");
|
||||
edit.type = "button";
|
||||
edit.className = "linky";
|
||||
|
||||
@@ -1319,6 +1319,26 @@ class Store:
|
||||
`self.state`: it can keep a counter without being able to read roles."""
|
||||
return _StateMap(self)
|
||||
|
||||
def state_items(self, prefix: str) -> dict[str, str]:
|
||||
"""Every `state` key that starts with `prefix`, with its value, in one
|
||||
query, for a reader that wants one kind of key across every target."""
|
||||
return {str(r["key"]): str(r["value"]) for r in self._rows(
|
||||
"SELECT key, value FROM state WHERE substr(key, 1, ?) = ? ORDER BY key",
|
||||
(len(prefix), prefix))}
|
||||
|
||||
def claim_state(self, key: str, value: str, *, released: str, stale: str) -> bool:
|
||||
"""Set `key` to `value` only while it is free: unset, equal to the
|
||||
value under `released` (the claim last let go of), or no later than
|
||||
`stale`. Returns whether it was set. One statement, so of two
|
||||
processes claiming at once exactly one wins."""
|
||||
return self._write(
|
||||
"""INSERT INTO state (key, value) VALUES (?, ?)
|
||||
ON CONFLICT(key) DO UPDATE SET value = excluded.value
|
||||
WHERE state.value <= ?
|
||||
OR state.value = COALESCE(
|
||||
(SELECT r.value FROM state AS r WHERE r.key = ?), '')""",
|
||||
(key, value, stale, released)) > 0
|
||||
|
||||
|
||||
def _check_status(status: str) -> None:
|
||||
if status not in STATUSES:
|
||||
|
||||
+15
-8
@@ -265,7 +265,9 @@ class CompanyOut(BaseModel):
|
||||
still open. `ats` is the host its careers site runs on whether or not the
|
||||
app reads it, and `ats_slug` its name there, so a followed company with
|
||||
no `board` and an `ats` is one the app cannot read yet; `careers_url` is
|
||||
where a person can."""
|
||||
where a person can. `failing_since` and `failing_reason` say since when
|
||||
the board has failed on every fetch and why it last did, both empty
|
||||
while it answers."""
|
||||
|
||||
key: str
|
||||
name: str
|
||||
@@ -278,20 +280,22 @@ class CompanyOut(BaseModel):
|
||||
ats_slug: str = ""
|
||||
careers_url: str = ""
|
||||
country: str = ""
|
||||
failing_since: str = ""
|
||||
failing_reason: str = ""
|
||||
|
||||
|
||||
def _company_boards(store: Store) -> dict[str, tuple[int, str, bool]]:
|
||||
"""`company_key -> (subscription id, ats host, on)`, read off each
|
||||
def _company_boards(store: Store) -> dict[str, tuple[int, str, bool, str]]:
|
||||
"""`company_key -> (subscription id, ats host, on, target)`, read off each
|
||||
company's own subscription. Same join as `Store.company_boards`, with the
|
||||
id and switch state a company row needs to draw its own on/off control
|
||||
rather than fetching `/api/sources` as well. A detached board (`show`
|
||||
0) is no board."""
|
||||
boards: dict[str, tuple[int, str, bool]] = {}
|
||||
boards: dict[str, tuple[int, str, bool, str]] = {}
|
||||
for sub in store.subscriptions(source="ats"):
|
||||
if not sub.show:
|
||||
continue
|
||||
boards.setdefault(company_key(sub.company), (sub.id, sub.board,
|
||||
bool(sub.subscribed and sub.fetch)))
|
||||
boards.setdefault(company_key(sub.company), (
|
||||
sub.id, sub.board, bool(sub.subscribed and sub.fetch), sub.target))
|
||||
return boards
|
||||
|
||||
|
||||
@@ -301,14 +305,17 @@ def list_companies(store: StoreDep, agent: int | None = None) -> list[CompanyOut
|
||||
absent) has it; 0 throughout when there is no agent."""
|
||||
boards = _company_boards(store)
|
||||
counts = store.open_role_counts()
|
||||
failing = jobs.pull.failing_boards(store)
|
||||
out = []
|
||||
for c in store.companies(_agent_id(store, agent)):
|
||||
sub_id, host, on = boards.get(c.key, (None, "", None))
|
||||
sub_id, host, on, target = boards.get(c.key, (None, "", None, ""))
|
||||
since, reason = failing.get(target, ("", ""))
|
||||
out.append(CompanyOut(key=c.key, name=c.name, board=host,
|
||||
subscription_id=sub_id, on=on, affinity=c.affinity,
|
||||
open_roles=counts.get(c.key, 0), ats=c.ats,
|
||||
ats_slug=c.ats_slug, careers_url=c.careers_url,
|
||||
country=c.country))
|
||||
country=c.country,
|
||||
failing_since=since, failing_reason=reason))
|
||||
return out
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user