diff --git a/app/Services/DeploymentService.php b/app/Services/DeploymentService.php index 12ba19e..de90114 100644 --- a/app/Services/DeploymentService.php +++ b/app/Services/DeploymentService.php @@ -260,7 +260,7 @@ class DeploymentService return null; } $data = json_decode((string) @file_get_contents($path), true); - if (! is_array($data) || ! in_array($data['stage'] ?? null, ['fetch', 'build', 'restart', 'migrate', 'done'], true)) { + if (! is_array($data) || ! in_array($data['stage'] ?? null, ['fetch', 'build', 'restart', 'migrate', 'done', 'error'], true)) { return null; } // Require a real timestamp: the progress page's freshness logic ties a stage to THIS update diff --git a/docker/restart-sentinel/watch.sh b/docker/restart-sentinel/watch.sh index a032fc4..caa58f7 100755 --- a/docker/restart-sentinel/watch.sh +++ b/docker/restart-sentinel/watch.sh @@ -35,6 +35,17 @@ COMPOSE_FILE="${CLUSEV_DIR}/docker-compose.prod.yml" TAG="clusev-restart" log() { printf '%s %s: %s\n' "$(date -Is)" "$TAG" "$*"; } +# Publish an "error" stage to the update-progress feed (run/update-phase.json, bind-mounted into the +# app container) so the browser shows the failure AT ONCE instead of spinning to its 10-minute +# timeout. Best-effort — a write failure must never change the update's outcome. +write_update_error() { + local f="${CLUSEV_DIR}/run/update-phase.json" + mkdir -p "${CLUSEV_DIR}/run" 2>/dev/null || true + printf '{"stage":"error","at":%s}\n' "$(date +%s 2>/dev/null || echo 0)" > "$f" 2>/dev/null || true + chmod 644 "$f" 2>/dev/null || true + return 0 +} + # Restart the stack and consume the sentinel. `up -d` (not bare `restart`) so a # changed image/config is also applied and any down service is brought back. do_restart() { @@ -68,19 +79,34 @@ run_once() { # needs root for docker/systemd/apt. do_update() { TAG="clusev-update" - [ -f "$COMPOSE_FILE" ] || { log "compose file not found: $COMPOSE_FILE — skipping"; return 1; } + # Every failure path below publishes the error stage too, so a page opened by the dashboard never + # spins to the timeout when the update can't even start. + [ -f "$COMPOSE_FILE" ] || { log "compose file not found: $COMPOSE_FILE — skipping"; write_update_error; return 1; } local updater="${CLUSEV_DIR}/update.sh" if [ ! -f "$updater" ]; then log "update.sh not found at $updater — clearing sentinel, nothing to do" rm -f "$CLUSEV_UPDATE_SIGNAL" + write_update_error return 1 fi rm -f "$CLUSEV_UPDATE_SIGNAL" && log "update sentinel consumed: $CLUSEV_UPDATE_SIGNAL" log "update requested — running update.sh in ${CLUSEV_DIR}" - if bash "$updater"; then + # Tee the whole run to a host log the operator can inspect, and cap it with a generous timeout so + # a truly-stuck step (e.g. a hung build) fails LOUDLY rather than pinning the progress page. On ANY + # non-zero exit (update.sh or its exec'd install.sh — the exit code propagates through exec) publish + # the error stage so the browser stops spinning and shows the failure. `timeout` exit 124 = capped. + local logf="${CLUSEV_DIR}/run/update.log" rc=0 + mkdir -p "${CLUSEV_DIR}/run" 2>/dev/null || true + timeout -k 30 1800 bash "$updater" 2>&1 | tee "$logf" || rc=${PIPESTATUS[0]} + if [ "$rc" = 0 ]; then log "update applied" else - log "update.sh FAILED — see output above; re-trigger from the dashboard to retry" + if [ "$rc" = 124 ]; then + log "update.sh TIMED OUT after 30m — see $logf; re-trigger from the dashboard to retry" + else + log "update.sh FAILED (exit $rc) — see $logf and the journal; re-trigger from the dashboard to retry" + fi + write_update_error return 1 fi } diff --git a/lang/de/update.php b/lang/de/update.php index 3090422..d98d0ff 100644 --- a/lang/de/update.php +++ b/lang/de/update.php @@ -18,4 +18,9 @@ return [ 'timeout_heading' => 'Update dauert ungewöhnlich lange', 'timeout_hint' => 'Bitte die Seite manuell neu laden oder die Logs prüfen (journalctl -u clusev-update.service).', 'reload_button' => 'Neu laden', + + // Fehler — der Host-Updater ist mit Fehler beendet (git pull, Build oder Migration) oder im Timeout. + 'error_heading' => 'Update fehlgeschlagen', + 'error_hint' => 'Der Stack konnte nicht aktualisiert werden — die bisherige Version läuft weiter. Ursache im Host-Log prüfen (journalctl -u clusev-update.service, oder run/update.log), beheben und das Update erneut über das Dashboard auslösen.', + 'back_button' => 'Zurück', ]; diff --git a/lang/en/update.php b/lang/en/update.php index cba1c9c..2f664ed 100644 --- a/lang/en/update.php +++ b/lang/en/update.php @@ -18,4 +18,9 @@ return [ 'timeout_heading' => 'Update is taking unusually long', 'timeout_hint' => 'Please reload the page manually or check the logs (journalctl -u clusev-update.service).', 'reload_button' => 'Reload', + + // Error — the host updater exited non-zero (git pull, build or migrate failed) or timed out. + 'error_heading' => 'Update failed', + 'error_hint' => 'The stack could not be updated — your previous version is still running. Check the host log for the cause (journalctl -u clusev-update.service, or run/update.log), fix it, then re-trigger the update from the dashboard.', + 'back_button' => 'Back', ]; diff --git a/resources/views/update-progress.blade.php b/resources/views/update-progress.blade.php index d24f01b..7d7fc59 100644 --- a/resources/views/update-progress.blade.php +++ b/resources/views/update-progress.blade.php @@ -63,6 +63,23 @@ {{ __('update.reload_button') }} + + {{-- Error: the host updater exited non-zero (or timed out). Surfaced by the + update-status feed reporting stage=error, so the page stops at once instead + of spinning to the 10-minute timeout. The previous version keeps running. --}} + {{-- Phase checklist --}} @@ -199,6 +216,7 @@ var runningEl = document.getElementById('js-status-running'); var doneEl = document.getElementById('js-status-done'); var timeoutEl = document.getElementById('js-status-timeout'); + var errorEl = document.getElementById('js-status-error'); // ── Polling ──────────────────────────────────────────────────────────── var stableNew = 0; // consecutive polls that have seen the NEW app live @@ -236,6 +254,28 @@ timeoutEl.classList.remove('hidden'); } + // The host updater reported stage=error (exited non-zero or was killed by its timeout). + // Stop everything and show the failure at once — mark the phase that was active as failed. + function showError() { + if (done) return; + done = true; + clearInterval(pollInterval); + clearInterval(statusInterval); + clearInterval(tickInterval); + if (currentPhase >= 0 && currentPhase < PHASE_KEYS.length) { + var key = PHASE_KEYS[currentPhase]; + var dot = document.getElementById('js-phase-dot-' + key); + var label = document.getElementById('js-phase-label-' + key); + var li = document.getElementById('js-phase-' + key); + if (li) li.classList.remove('border-accent/30', 'bg-accent/5'); + if (dot) dot.style.backgroundColor = 'var(--color-offline)'; + if (label) label.style.color = 'var(--color-offline)'; + } + spinnerEl.classList.add('hidden'); + runningEl.classList.add('hidden'); + errorEl.classList.remove('hidden'); + } + function tick() { var elapsed = baseElapsedMs + (Date.now() - clientLoad); @@ -315,6 +355,7 @@ // previous run triggering a premature completion on refresh. if (!startedAt || typeof data.at !== 'number' || data.at < startedAt - 5) return; feedActive = true; + if (data.stage === 'error') { showError(); return; } if (data.stage === 'done') { showDone(); return; } var idx = STAGE_INDEX[data.stage]; // The live feed is authoritative — the host writes stages monotonically forward, diff --git a/tests/Feature/UpdateProgressTest.php b/tests/Feature/UpdateProgressTest.php index caf0a77..7819e9c 100644 --- a/tests/Feature/UpdateProgressTest.php +++ b/tests/Feature/UpdateProgressTest.php @@ -288,6 +288,34 @@ class UpdateProgressTest extends TestCase } } + public function test_update_status_surfaces_the_error_stage(): void + { + // The host updater (watch.sh) writes stage=error on ANY failure so the progress page can stop + // and show the failure at once instead of spinning to its 10-minute timeout. + $dir = storage_path('app/restart-signal'); + @mkdir($dir, 0775, true); + file_put_contents($dir.'/update-phase.json', json_encode(['stage' => 'error', 'at' => 123])); + + try { + $this->getJson('/update-status.json') + ->assertOk() + ->assertExactJson(['stage' => 'error', 'at' => 123]); + } finally { + @unlink($dir.'/update-phase.json'); + } + } + + public function test_update_progress_page_renders_an_error_state(): void + { + $body = $this->get(route('update.progress'))->content(); + + // A failed update must surface — the page needs an error branch driven by the feed's + // stage=error, not just the running/done/timeout states. + $this->assertStringContainsString('js-status-error', $body); + $this->assertStringContainsString('function showError', $body); + $this->assertStringContainsString("data.stage === 'error'", $body); + } + // ── (c) resumable progress (survives a browser refresh) ─────────────────── public function test_request_update_writes_a_recent_start_marker(): void diff --git a/update.sh b/update.sh index bb450a4..54906d0 100755 --- a/update.sh +++ b/update.sh @@ -19,6 +19,14 @@ set -euo pipefail cd "$(dirname "$0")" REPO_DIR="$(pwd)" +# Never let a private-repo credential miss HANG the update: git must fail fast instead of blocking on +# an interactive username/password prompt (the systemd updater has no TTY, so a prompt would wait +# forever — the classic "update spins with no error"). Also abort a stalled transfer instead of +# hanging on it. The pull itself is additionally wrapped in `timeout` below. +export GIT_TERMINAL_PROMPT=0 +export GIT_HTTP_LOW_SPEED_LIMIT="${GIT_HTTP_LOW_SPEED_LIMIT:-1000}" +export GIT_HTTP_LOW_SPEED_TIME="${GIT_HTTP_LOW_SPEED_TIME:-30}" + # set_stage: publish a coarse update stage for the in-browser progress screen (best-effort — the # app container serves run/update-phase.json to it). A write failure must never fail the update. # Written first as "fetch" so the screen leaves the time-guess behind the moment the pull begins, @@ -51,8 +59,8 @@ if [ "${CLUSEV_UPDATE_REEXEC:-0}" != 1 ]; then # 2. fast-forward pull only — never auto-merge or throw away local edits before_hash="$(sha256sum -- "$0" | cut -d' ' -f1)" info "Hole Aktualisierungen (git pull --ff-only) ..." - git pull --ff-only \ - || die "git pull fehlgeschlagen (lokale Aenderungen oder divergierter Branch). Mit 'git status' pruefen und erneut versuchen." + timeout 300 git pull --ff-only \ + || die "git pull fehlgeschlagen — Timeout, fehlende Zugangsdaten zum privaten Repo, lokale Aenderungen oder divergierter Branch. Mit 'git status' und dem Repo-Zugriff (Token im origin-Remote) pruefen und erneut versuchen." after_hash="$(sha256sum -- "$0" | cut -d' ' -f1)" # 3. self-update: relaunch the new update.sh if it changed