191 lines
9.0 KiB
Bash
Executable File
191 lines
9.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Clusev restart-sentinel watcher — runs on the HOST (never in a container).
|
|
#
|
|
# Watches for the restart sentinel file the dashboard writes (via a shared bind
|
|
# mount, see docker-compose.prod.yml: ./run -> storage/app/restart-signal) and,
|
|
# when it appears, restarts the prod stack, then deletes the sentinel.
|
|
#
|
|
# SECURITY: the panel container gets NO Docker socket. It only writes a marker
|
|
# file. This scoped host unit is the ONLY thing that talks to Docker — so a panel
|
|
# compromise cannot drive arbitrary `docker` commands.
|
|
#
|
|
# Two ways to run it:
|
|
# 1. systemd .path unit (preferred) — clusev-restart.path triggers
|
|
# clusev-restart.service, which runs this script ONCE per sentinel.
|
|
# Invoke as: watch.sh once
|
|
# 2. Standalone poll/inotify loop (fallback, no systemd) — runs forever:
|
|
# Invoke as: watch.sh loop
|
|
#
|
|
# Config via env (with sane defaults):
|
|
# CLUSEV_DIR project dir holding docker-compose.prod.yml (default: script's repo root)
|
|
# CLUSEV_SIGNAL absolute path of the sentinel file (default: $CLUSEV_DIR/run/restart.request)
|
|
# CLUSEV_POLL poll interval seconds for the loop fallback (default: 5)
|
|
set -euo pipefail
|
|
|
|
# Resolve the repo root: docker/restart-sentinel/ -> ../../
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
DEFAULT_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)"
|
|
|
|
CLUSEV_DIR="${CLUSEV_DIR:-$DEFAULT_DIR}"
|
|
CLUSEV_SIGNAL="${CLUSEV_SIGNAL:-${CLUSEV_DIR}/run/restart.request}"
|
|
CLUSEV_UPDATE_SIGNAL="${CLUSEV_UPDATE_SIGNAL:-${CLUSEV_DIR}/run/update.request}"
|
|
CLUSEV_POLL="${CLUSEV_POLL:-5}"
|
|
COMPOSE_FILE="${CLUSEV_DIR}/docker-compose.prod.yml"
|
|
|
|
TAG="clusev-restart"
|
|
log() { printf '%s %s: %s\n' "$(date -Is)" "$TAG" "$*"; }
|
|
|
|
# run_write_json PAYLOAD — write PAYLOAD to run/update-phase.json SYMLINK-SAFELY as root. ./run is
|
|
# the container-writable bind mount, so a compromised container could pre-plant update-phase.json as
|
|
# a symlink to an arbitrary host path; a plain `> file` would follow it and let this root write escape
|
|
# ./run. We build the temp at the REPO ROOT — same filesystem as ./run but NOT mounted into any
|
|
# container, so the container cannot unlink/replace it mid-write (no TOCTOU) — then rename(2) it over
|
|
# the target: an atomic same-fs rename replaces the path itself (link included) with our regular file
|
|
# and NEVER follows the link. Best-effort — a write failure must never change the update's outcome.
|
|
run_write_json() {
|
|
local d="${CLUSEV_DIR}/run" tmp
|
|
mkdir -p "$d" 2>/dev/null || true
|
|
tmp="$(mktemp "${CLUSEV_DIR}/.phase.XXXXXX" 2>/dev/null)" || return 0
|
|
printf '%s\n' "$1" > "$tmp" 2>/dev/null || { rm -f "$tmp" 2>/dev/null; return 0; }
|
|
chmod 644 "$tmp" 2>/dev/null || true
|
|
mv -f "$tmp" "${d}/update-phase.json" 2>/dev/null || rm -f "$tmp" 2>/dev/null
|
|
return 0
|
|
}
|
|
|
|
# Publish an "error" stage to the update-progress feed so the browser shows the failure AT ONCE
|
|
# instead of spinning to its 10-minute timeout.
|
|
write_update_error() {
|
|
run_write_json "$(printf '{"stage":"error","at":%s}' "$(date +%s 2>/dev/null || echo 0)")"
|
|
}
|
|
|
|
# Authenticate an update request marker. The app (DeploymentService) signs it with UPDATE_HMAC_KEY
|
|
# (line 1 = ISO8601 timestamp, line 2 = HMAC-SHA256(timestamp)); we recompute and compare. Refusing
|
|
# an unsigned/forged marker stops a stray or limited write to ./run from driving a root update.
|
|
# (A full app-container compromise holds the key, so this raises the bar rather than being absolute —
|
|
# the standing guarantee is that a triggered update only re-installs the TRUSTED remote's code.)
|
|
verify_update_request() {
|
|
local f="$1" key ts mac expect
|
|
key="$(grep -m1 '^UPDATE_HMAC_KEY=' "${CLUSEV_DIR}/.env" 2>/dev/null | cut -d= -f2-)"
|
|
# Fail CLOSED: with no key the marker cannot be authenticated, so refuse rather than run a
|
|
# ROOT update on an unauthenticated request. install.sh always backfills UPDATE_HMAC_KEY
|
|
# (set_kv), so a real install has one; a missing key means a broken/hand-edited .env — the
|
|
# operator re-runs install.sh (or sets the key) to restore dashboard-triggered updates.
|
|
[ -n "$key" ] || { log "UPDATE_HMAC_KEY unset — REFUSING update request (re-run install.sh to backfill the key)"; return 1; }
|
|
ts="$(sed -n '1p' "$f" 2>/dev/null)"
|
|
mac="$(sed -n '2p' "$f" 2>/dev/null)"
|
|
[ -n "$ts" ] && [ -n "$mac" ] || { log "update request unsigned/malformed — refusing"; return 1; }
|
|
expect="$(printf '%s' "$ts" | openssl dgst -sha256 -hmac "$key" 2>/dev/null | sed 's/^.*= *//')"
|
|
[ -n "$expect" ] && [ "$expect" = "$mac" ] || { log "update request HMAC mismatch — refusing"; return 1; }
|
|
return 0
|
|
}
|
|
|
|
# Restart the stack and consume the sentinel. `up -d` (not bare `restart`) so a
|
|
# changed image/config is also applied and any down service is brought back.
|
|
do_restart() {
|
|
[ -f "$COMPOSE_FILE" ] || { log "compose file not found: $COMPOSE_FILE — skipping"; return 1; }
|
|
|
|
log "restart requested — applying (docker compose up -d) in ${CLUSEV_DIR}"
|
|
if docker compose -f "$COMPOSE_FILE" up -d; then
|
|
log "stack restarted"
|
|
else
|
|
log "docker compose up -d FAILED — leaving sentinel so it retries"
|
|
return 1
|
|
fi
|
|
|
|
rm -f "$CLUSEV_SIGNAL" && log "sentinel cleared: $CLUSEV_SIGNAL"
|
|
}
|
|
|
|
# One shot: only act if the sentinel is actually present (the .path unit guarantees this,
|
|
# but a manual invocation might not).
|
|
run_once() {
|
|
if [ -f "$CLUSEV_SIGNAL" ]; then
|
|
do_restart
|
|
else
|
|
log "no sentinel at $CLUSEV_SIGNAL — nothing to do"
|
|
fi
|
|
}
|
|
|
|
# Run update.sh (git pull + idempotent re-install). Unlike restart, the sentinel is consumed
|
|
# BEFORE running: a persistent failure (e.g. a non-fast-forwardable pull) must NOT loop the
|
|
# .path unit forever — the failure is in the journal and the operator re-triggers from the
|
|
# dashboard. Runs as root (the service unit sets User=root) because update.sh -> install.sh
|
|
# needs root for docker/systemd/apt.
|
|
do_update() {
|
|
TAG="clusev-update"
|
|
# Every failure path below publishes the error stage too, so a page opened by the dashboard never
|
|
# spins to the timeout when the update can't even start.
|
|
[ -f "$COMPOSE_FILE" ] || { log "compose file not found: $COMPOSE_FILE — skipping"; write_update_error; return 1; }
|
|
local updater="${CLUSEV_DIR}/update.sh"
|
|
if [ ! -f "$updater" ]; then
|
|
log "update.sh not found at $updater — clearing sentinel, nothing to do"
|
|
rm -f "$CLUSEV_UPDATE_SIGNAL"
|
|
write_update_error
|
|
return 1
|
|
fi
|
|
# Authenticate BEFORE consuming/running: refuse (and clear, so it does not loop the .path unit) a
|
|
# marker that carries no valid HMAC signature.
|
|
if ! verify_update_request "$CLUSEV_UPDATE_SIGNAL"; then
|
|
rm -f "$CLUSEV_UPDATE_SIGNAL"
|
|
write_update_error
|
|
return 1
|
|
fi
|
|
rm -f "$CLUSEV_UPDATE_SIGNAL" && log "update sentinel consumed: $CLUSEV_UPDATE_SIGNAL"
|
|
log "update requested — running update.sh in ${CLUSEV_DIR}"
|
|
# Tee the whole run to a host log the operator can inspect, and cap it with a generous timeout so
|
|
# a truly-stuck step (e.g. a hung build) fails LOUDLY rather than pinning the progress page. On ANY
|
|
# non-zero exit (update.sh or its exec'd install.sh — the exit code propagates through exec) publish
|
|
# the error stage so the browser stops spinning and shows the failure. `timeout` exit 124 = capped.
|
|
# The log lives at the repo ROOT, NOT under ./run: ./run is the container-writable bind mount, so a
|
|
# log there could be pre-planted as a symlink and this root `tee` would follow it; the repo root is
|
|
# not mounted into any container, so a container cannot redirect this write.
|
|
local logf="${CLUSEV_DIR}/update.log" rc=0
|
|
rm -f "$logf" 2>/dev/null || true
|
|
timeout -k 30 1800 bash "$updater" 2>&1 | tee "$logf" || rc=${PIPESTATUS[0]}
|
|
if [ "$rc" = 0 ]; then
|
|
log "update applied"
|
|
else
|
|
if [ "$rc" = 124 ]; then
|
|
log "update.sh TIMED OUT after 30m — see $logf; re-trigger from the dashboard to retry"
|
|
else
|
|
log "update.sh FAILED (exit $rc) — see $logf and the journal; re-trigger from the dashboard to retry"
|
|
fi
|
|
write_update_error
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
run_once_update() {
|
|
if [ -f "$CLUSEV_UPDATE_SIGNAL" ]; then
|
|
do_update
|
|
else
|
|
TAG="clusev-update"; log "no sentinel at $CLUSEV_UPDATE_SIGNAL — nothing to do"
|
|
fi
|
|
}
|
|
|
|
# Fallback loop for hosts without systemd: prefer inotifywait, else poll.
|
|
run_loop() {
|
|
log "watching $CLUSEV_SIGNAL (loop mode)"
|
|
if command -v inotifywait >/dev/null 2>&1; then
|
|
# Act once on startup in case the sentinel already exists, then block on events.
|
|
[ -f "$CLUSEV_SIGNAL" ] && do_restart || true
|
|
while true; do
|
|
# Watch the directory (the file is created/moved into it) for create/move events.
|
|
inotifywait -q -e create -e moved_to -e close_write "$(dirname "$CLUSEV_SIGNAL")" >/dev/null 2>&1 || true
|
|
[ -f "$CLUSEV_SIGNAL" ] && do_restart || true
|
|
done
|
|
else
|
|
log "inotifywait not found — polling every ${CLUSEV_POLL}s"
|
|
while true; do
|
|
[ -f "$CLUSEV_SIGNAL" ] && do_restart || true
|
|
sleep "$CLUSEV_POLL"
|
|
done
|
|
fi
|
|
}
|
|
|
|
case "${1:-once}" in
|
|
once) run_once ;;
|
|
update) run_once_update ;;
|
|
loop) run_loop ;;
|
|
*) log "usage: $0 [once|update|loop]"; exit 64 ;;
|
|
esac
|