148 lines
6.1 KiB
Bash
Executable File
148 lines
6.1 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Clusev restart-sentinel watcher — runs on the HOST (never in a container).
|
|
#
|
|
# Watches for the restart sentinel file the dashboard writes (via a shared bind
|
|
# mount, see docker-compose.prod.yml: ./run -> storage/app/restart-signal) and,
|
|
# when it appears, restarts the prod stack, then deletes the sentinel.
|
|
#
|
|
# SECURITY: the panel container gets NO Docker socket. It only writes a marker
|
|
# file. This scoped host unit is the ONLY thing that talks to Docker — so a panel
|
|
# compromise cannot drive arbitrary `docker` commands.
|
|
#
|
|
# Two ways to run it:
|
|
# 1. systemd .path unit (preferred) — clusev-restart.path triggers
|
|
# clusev-restart.service, which runs this script ONCE per sentinel.
|
|
# Invoke as: watch.sh once
|
|
# 2. Standalone poll/inotify loop (fallback, no systemd) — runs forever:
|
|
# Invoke as: watch.sh loop
|
|
#
|
|
# Config via env (with sane defaults):
|
|
# CLUSEV_DIR project dir holding docker-compose.prod.yml (default: script's repo root)
|
|
# CLUSEV_SIGNAL absolute path of the sentinel file (default: $CLUSEV_DIR/run/restart.request)
|
|
# CLUSEV_POLL poll interval seconds for the loop fallback (default: 5)
|
|
set -euo pipefail
|
|
|
|
# Resolve the repo root: docker/restart-sentinel/ -> ../../
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
DEFAULT_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)"
|
|
|
|
CLUSEV_DIR="${CLUSEV_DIR:-$DEFAULT_DIR}"
|
|
CLUSEV_SIGNAL="${CLUSEV_SIGNAL:-${CLUSEV_DIR}/run/restart.request}"
|
|
CLUSEV_UPDATE_SIGNAL="${CLUSEV_UPDATE_SIGNAL:-${CLUSEV_DIR}/run/update.request}"
|
|
CLUSEV_POLL="${CLUSEV_POLL:-5}"
|
|
COMPOSE_FILE="${CLUSEV_DIR}/docker-compose.prod.yml"
|
|
|
|
TAG="clusev-restart"
|
|
log() { printf '%s %s: %s\n' "$(date -Is)" "$TAG" "$*"; }
|
|
|
|
# Publish an "error" stage to the update-progress feed (run/update-phase.json, bind-mounted into the
|
|
# app container) so the browser shows the failure AT ONCE instead of spinning to its 10-minute
|
|
# timeout. Best-effort — a write failure must never change the update's outcome.
|
|
write_update_error() {
|
|
local f="${CLUSEV_DIR}/run/update-phase.json"
|
|
mkdir -p "${CLUSEV_DIR}/run" 2>/dev/null || true
|
|
printf '{"stage":"error","at":%s}\n' "$(date +%s 2>/dev/null || echo 0)" > "$f" 2>/dev/null || true
|
|
chmod 644 "$f" 2>/dev/null || true
|
|
return 0
|
|
}
|
|
|
|
# Restart the stack and consume the sentinel. `up -d` (not bare `restart`) so a
|
|
# changed image/config is also applied and any down service is brought back.
|
|
do_restart() {
|
|
[ -f "$COMPOSE_FILE" ] || { log "compose file not found: $COMPOSE_FILE — skipping"; return 1; }
|
|
|
|
log "restart requested — applying (docker compose up -d) in ${CLUSEV_DIR}"
|
|
if docker compose -f "$COMPOSE_FILE" up -d; then
|
|
log "stack restarted"
|
|
else
|
|
log "docker compose up -d FAILED — leaving sentinel so it retries"
|
|
return 1
|
|
fi
|
|
|
|
rm -f "$CLUSEV_SIGNAL" && log "sentinel cleared: $CLUSEV_SIGNAL"
|
|
}
|
|
|
|
# One shot: only act if the sentinel is actually present (the .path unit guarantees this,
|
|
# but a manual invocation might not).
|
|
run_once() {
|
|
if [ -f "$CLUSEV_SIGNAL" ]; then
|
|
do_restart
|
|
else
|
|
log "no sentinel at $CLUSEV_SIGNAL — nothing to do"
|
|
fi
|
|
}
|
|
|
|
# Run update.sh (git pull + idempotent re-install). Unlike restart, the sentinel is consumed
|
|
# BEFORE running: a persistent failure (e.g. a non-fast-forwardable pull) must NOT loop the
|
|
# .path unit forever — the failure is in the journal and the operator re-triggers from the
|
|
# dashboard. Runs as root (the service unit sets User=root) because update.sh -> install.sh
|
|
# needs root for docker/systemd/apt.
|
|
do_update() {
|
|
TAG="clusev-update"
|
|
# Every failure path below publishes the error stage too, so a page opened by the dashboard never
|
|
# spins to the timeout when the update can't even start.
|
|
[ -f "$COMPOSE_FILE" ] || { log "compose file not found: $COMPOSE_FILE — skipping"; write_update_error; return 1; }
|
|
local updater="${CLUSEV_DIR}/update.sh"
|
|
if [ ! -f "$updater" ]; then
|
|
log "update.sh not found at $updater — clearing sentinel, nothing to do"
|
|
rm -f "$CLUSEV_UPDATE_SIGNAL"
|
|
write_update_error
|
|
return 1
|
|
fi
|
|
rm -f "$CLUSEV_UPDATE_SIGNAL" && log "update sentinel consumed: $CLUSEV_UPDATE_SIGNAL"
|
|
log "update requested — running update.sh in ${CLUSEV_DIR}"
|
|
# Tee the whole run to a host log the operator can inspect, and cap it with a generous timeout so
|
|
# a truly-stuck step (e.g. a hung build) fails LOUDLY rather than pinning the progress page. On ANY
|
|
# non-zero exit (update.sh or its exec'd install.sh — the exit code propagates through exec) publish
|
|
# the error stage so the browser stops spinning and shows the failure. `timeout` exit 124 = capped.
|
|
local logf="${CLUSEV_DIR}/run/update.log" rc=0
|
|
mkdir -p "${CLUSEV_DIR}/run" 2>/dev/null || true
|
|
timeout -k 30 1800 bash "$updater" 2>&1 | tee "$logf" || rc=${PIPESTATUS[0]}
|
|
if [ "$rc" = 0 ]; then
|
|
log "update applied"
|
|
else
|
|
if [ "$rc" = 124 ]; then
|
|
log "update.sh TIMED OUT after 30m — see $logf; re-trigger from the dashboard to retry"
|
|
else
|
|
log "update.sh FAILED (exit $rc) — see $logf and the journal; re-trigger from the dashboard to retry"
|
|
fi
|
|
write_update_error
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
run_once_update() {
|
|
if [ -f "$CLUSEV_UPDATE_SIGNAL" ]; then
|
|
do_update
|
|
else
|
|
TAG="clusev-update"; log "no sentinel at $CLUSEV_UPDATE_SIGNAL — nothing to do"
|
|
fi
|
|
}
|
|
|
|
# Fallback loop for hosts without systemd: prefer inotifywait, else poll.
|
|
run_loop() {
|
|
log "watching $CLUSEV_SIGNAL (loop mode)"
|
|
if command -v inotifywait >/dev/null 2>&1; then
|
|
# Act once on startup in case the sentinel already exists, then block on events.
|
|
[ -f "$CLUSEV_SIGNAL" ] && do_restart || true
|
|
while true; do
|
|
# Watch the directory (the file is created/moved into it) for create/move events.
|
|
inotifywait -q -e create -e moved_to -e close_write "$(dirname "$CLUSEV_SIGNAL")" >/dev/null 2>&1 || true
|
|
[ -f "$CLUSEV_SIGNAL" ] && do_restart || true
|
|
done
|
|
else
|
|
log "inotifywait not found — polling every ${CLUSEV_POLL}s"
|
|
while true; do
|
|
[ -f "$CLUSEV_SIGNAL" ] && do_restart || true
|
|
sleep "$CLUSEV_POLL"
|
|
done
|
|
fi
|
|
}
|
|
|
|
case "${1:-once}" in
|
|
once) run_once ;;
|
|
update) run_once_update ;;
|
|
loop) run_loop ;;
|
|
*) log "usage: $0 [once|update|loop]"; exit 64 ;;
|
|
esac
|