#!/usr/bin/env bash # Clusev restart-sentinel watcher — runs on the HOST (never in a container). # # Watches for the restart sentinel file the dashboard writes (via a shared bind # mount, see docker-compose.prod.yml: ./run -> storage/app/restart-signal) and, # when it appears, restarts the prod stack, then deletes the sentinel. # # SECURITY: the panel container gets NO Docker socket. It only writes a marker # file. This scoped host unit is the ONLY thing that talks to Docker — so a panel # compromise cannot drive arbitrary `docker` commands. # # Two ways to run it: # 1. systemd .path unit (preferred) — clusev-restart.path triggers # clusev-restart.service, which runs this script ONCE per sentinel. # Invoke as: watch.sh once # 2. Standalone poll/inotify loop (fallback, no systemd) — runs forever: # Invoke as: watch.sh loop # # Config via env (with sane defaults): # CLUSEV_DIR project dir holding docker-compose.prod.yml (default: script's repo root) # CLUSEV_SIGNAL absolute path of the sentinel file (default: $CLUSEV_DIR/run/restart.request) # CLUSEV_POLL poll interval seconds for the loop fallback (default: 5) set -euo pipefail # Resolve the repo root: docker/restart-sentinel/ -> ../../ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DEFAULT_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)" CLUSEV_DIR="${CLUSEV_DIR:-$DEFAULT_DIR}" CLUSEV_SIGNAL="${CLUSEV_SIGNAL:-${CLUSEV_DIR}/run/restart.request}" CLUSEV_UPDATE_SIGNAL="${CLUSEV_UPDATE_SIGNAL:-${CLUSEV_DIR}/run/update.request}" CLUSEV_POLL="${CLUSEV_POLL:-5}" COMPOSE_FILE="${CLUSEV_DIR}/docker-compose.prod.yml" TAG="clusev-restart" log() { printf '%s %s: %s\n' "$(date -Is)" "$TAG" "$*"; } # run_write_json PAYLOAD — write PAYLOAD to run/update-phase.json SYMLINK-SAFELY as root. ./run is # the container-writable bind mount, so a compromised container could pre-plant update-phase.json as # a symlink to an arbitrary host path; a plain `> file` would follow it and let this root write escape # ./run. We build the temp at the REPO ROOT — same filesystem as ./run but NOT mounted into any # container, so the container cannot unlink/replace it mid-write (no TOCTOU) — then rename(2) it over # the target: an atomic same-fs rename replaces the path itself (link included) with our regular file # and NEVER follows the link. Best-effort — a write failure must never change the update's outcome. run_write_json() { local d="${CLUSEV_DIR}/run" tmp mkdir -p "$d" 2>/dev/null || true tmp="$(mktemp "${CLUSEV_DIR}/.phase.XXXXXX" 2>/dev/null)" || return 0 printf '%s\n' "$1" > "$tmp" 2>/dev/null || { rm -f "$tmp" 2>/dev/null; return 0; } chmod 644 "$tmp" 2>/dev/null || true mv -f "$tmp" "${d}/update-phase.json" 2>/dev/null || rm -f "$tmp" 2>/dev/null return 0 } # Publish an "error" stage to the update-progress feed so the browser shows the failure AT ONCE # instead of spinning to its 10-minute timeout. write_update_error() { run_write_json "$(printf '{"stage":"error","at":%s}' "$(date +%s 2>/dev/null || echo 0)")" } # Authenticate an update request marker. The app (DeploymentService) signs it with UPDATE_HMAC_KEY # (line 1 = ISO8601 timestamp, line 2 = HMAC-SHA256(timestamp)); we recompute and compare. Refusing # an unsigned/forged marker stops a stray or limited write to ./run from driving a root update. # (A full app-container compromise holds the key, so this raises the bar rather than being absolute — # the standing guarantee is that a triggered update only re-installs the TRUSTED remote's code.) verify_update_request() { local f="$1" key ts mac expect key="$(grep -m1 '^UPDATE_HMAC_KEY=' "${CLUSEV_DIR}/.env" 2>/dev/null | cut -d= -f2-)" # Fail CLOSED: with no key the marker cannot be authenticated, so refuse rather than run a # ROOT update on an unauthenticated request. install.sh always backfills UPDATE_HMAC_KEY # (set_kv), so a real install has one; a missing key means a broken/hand-edited .env — the # operator re-runs install.sh (or sets the key) to restore dashboard-triggered updates. [ -n "$key" ] || { log "UPDATE_HMAC_KEY unset — REFUSING update request (re-run install.sh to backfill the key)"; return 1; } ts="$(sed -n '1p' "$f" 2>/dev/null)" mac="$(sed -n '2p' "$f" 2>/dev/null)" [ -n "$ts" ] && [ -n "$mac" ] || { log "update request unsigned/malformed — refusing"; return 1; } expect="$(printf '%s' "$ts" | openssl dgst -sha256 -hmac "$key" 2>/dev/null | sed 's/^.*= *//')" [ -n "$expect" ] && [ "$expect" = "$mac" ] || { log "update request HMAC mismatch — refusing"; return 1; } return 0 } # Restart the stack and consume the sentinel. `up -d` (not bare `restart`) so a # changed image/config is also applied and any down service is brought back. do_restart() { [ -f "$COMPOSE_FILE" ] || { log "compose file not found: $COMPOSE_FILE — skipping"; return 1; } log "restart requested — applying (docker compose up -d) in ${CLUSEV_DIR}" if docker compose -f "$COMPOSE_FILE" up -d; then log "stack restarted" else log "docker compose up -d FAILED — leaving sentinel so it retries" return 1 fi rm -f "$CLUSEV_SIGNAL" && log "sentinel cleared: $CLUSEV_SIGNAL" } # One shot: only act if the sentinel is actually present (the .path unit guarantees this, # but a manual invocation might not). run_once() { if [ -f "$CLUSEV_SIGNAL" ]; then do_restart else log "no sentinel at $CLUSEV_SIGNAL — nothing to do" fi } # Run update.sh (git pull + idempotent re-install). Unlike restart, the sentinel is consumed # BEFORE running: a persistent failure (e.g. a non-fast-forwardable pull) must NOT loop the # .path unit forever — the failure is in the journal and the operator re-triggers from the # dashboard. Runs as root (the service unit sets User=root) because update.sh -> install.sh # needs root for docker/systemd/apt. do_update() { TAG="clusev-update" # Every failure path below publishes the error stage too, so a page opened by the dashboard never # spins to the timeout when the update can't even start. [ -f "$COMPOSE_FILE" ] || { log "compose file not found: $COMPOSE_FILE — skipping"; write_update_error; return 1; } local updater="${CLUSEV_DIR}/update.sh" if [ ! -f "$updater" ]; then log "update.sh not found at $updater — clearing sentinel, nothing to do" rm -f "$CLUSEV_UPDATE_SIGNAL" write_update_error return 1 fi # Authenticate BEFORE consuming/running: refuse (and clear, so it does not loop the .path unit) a # marker that carries no valid HMAC signature. if ! verify_update_request "$CLUSEV_UPDATE_SIGNAL"; then rm -f "$CLUSEV_UPDATE_SIGNAL" write_update_error return 1 fi rm -f "$CLUSEV_UPDATE_SIGNAL" && log "update sentinel consumed: $CLUSEV_UPDATE_SIGNAL" log "update requested — running update.sh in ${CLUSEV_DIR}" # Tee the whole run to a host log the operator can inspect, and cap it with a generous timeout so # a truly-stuck step (e.g. a hung build) fails LOUDLY rather than pinning the progress page. On ANY # non-zero exit (update.sh or its exec'd install.sh — the exit code propagates through exec) publish # the error stage so the browser stops spinning and shows the failure. `timeout` exit 124 = capped. # The log lives at the repo ROOT, NOT under ./run: ./run is the container-writable bind mount, so a # log there could be pre-planted as a symlink and this root `tee` would follow it; the repo root is # not mounted into any container, so a container cannot redirect this write. local logf="${CLUSEV_DIR}/update.log" rc=0 rm -f "$logf" 2>/dev/null || true timeout -k 30 1800 bash "$updater" 2>&1 | tee "$logf" || rc=${PIPESTATUS[0]} if [ "$rc" = 0 ]; then log "update applied" else if [ "$rc" = 124 ]; then log "update.sh TIMED OUT after 30m — see $logf; re-trigger from the dashboard to retry" else log "update.sh FAILED (exit $rc) — see $logf and the journal; re-trigger from the dashboard to retry" fi write_update_error return 1 fi } run_once_update() { if [ -f "$CLUSEV_UPDATE_SIGNAL" ]; then do_update else TAG="clusev-update"; log "no sentinel at $CLUSEV_UPDATE_SIGNAL — nothing to do" fi } # Fallback loop for hosts without systemd: prefer inotifywait, else poll. run_loop() { log "watching $CLUSEV_SIGNAL (loop mode)" if command -v inotifywait >/dev/null 2>&1; then # Act once on startup in case the sentinel already exists, then block on events. [ -f "$CLUSEV_SIGNAL" ] && do_restart || true while true; do # Watch the directory (the file is created/moved into it) for create/move events. inotifywait -q -e create -e moved_to -e close_write "$(dirname "$CLUSEV_SIGNAL")" >/dev/null 2>&1 || true [ -f "$CLUSEV_SIGNAL" ] && do_restart || true done else log "inotifywait not found — polling every ${CLUSEV_POLL}s" while true; do [ -f "$CLUSEV_SIGNAL" ] && do_restart || true sleep "$CLUSEV_POLL" done fi } case "${1:-once}" in once) run_once ;; update) run_once_update ;; loop) run_loop ;; *) log "usage: $0 [once|update|loop]"; exit 64 ;; esac