From 72b82df7a1e893a7908826b83f8bd71d9e113bf8 Mon Sep 17 00:00:00 2001 From: boban Date: Fri, 3 Jul 2026 18:58:32 +0200 Subject: [PATCH] fix(update): surface update failures instead of spinning forever MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A dashboard-triggered update that failed on the host left the /update-progress page spinning to a vague 10-minute timeout — no error was ever shown, because nothing wrote a failure marker for the page to read. Fix: make failures visible and prevent the credential-prompt hang class. - docker/restart-sentinel/watch.sh: run update.sh under `timeout -k 30 1800` with its output tee'd to run/update.log; on ANY non-zero exit (update.sh or its exec'd install.sh) — and on the early compose-missing / updater-missing paths — write {"stage":"error"} to run/update-phase.json via write_update_error(). - update.sh: export GIT_TERMINAL_PROMPT=0 + GIT_HTTP_LOW_SPEED_* and wrap the pull in `timeout 300`, so a private-repo credential miss fails fast instead of hanging on a non-interactive prompt. - DeploymentService::updatePhase(): whitelist the 'error' stage. - update-progress.blade.php: add an error state (#js-status-error) + showError(); the status-feed poll now stops and shows it on stage=error, marking the active phase red — instead of looping. - lang/{en,de}/update.php: error_heading / error_hint / back_button. - UpdateProgressTest: feed surfaces stage=error; page renders the error branch. Co-Authored-By: Claude Opus 4.8 --- app/Services/DeploymentService.php | 2 +- docker/restart-sentinel/watch.sh | 32 ++++++++++++++++-- lang/de/update.php | 5 +++ lang/en/update.php | 5 +++ resources/views/update-progress.blade.php | 41 +++++++++++++++++++++++ tests/Feature/UpdateProgressTest.php | 28 ++++++++++++++++ update.sh | 12 +++++-- 7 files changed, 119 insertions(+), 6 deletions(-) diff --git a/app/Services/DeploymentService.php b/app/Services/DeploymentService.php index 12ba19e..de90114 100644 --- a/app/Services/DeploymentService.php +++ b/app/Services/DeploymentService.php @@ -260,7 +260,7 @@ class DeploymentService return null; } $data = json_decode((string) @file_get_contents($path), true); - if (! is_array($data) || ! in_array($data['stage'] ?? null, ['fetch', 'build', 'restart', 'migrate', 'done'], true)) { + if (! is_array($data) || ! in_array($data['stage'] ?? null, ['fetch', 'build', 'restart', 'migrate', 'done', 'error'], true)) { return null; } // Require a real timestamp: the progress page's freshness logic ties a stage to THIS update diff --git a/docker/restart-sentinel/watch.sh b/docker/restart-sentinel/watch.sh index a032fc4..caa58f7 100755 --- a/docker/restart-sentinel/watch.sh +++ b/docker/restart-sentinel/watch.sh @@ -35,6 +35,17 @@ COMPOSE_FILE="${CLUSEV_DIR}/docker-compose.prod.yml" TAG="clusev-restart" log() { printf '%s %s: %s\n' "$(date -Is)" "$TAG" "$*"; } +# Publish an "error" stage to the update-progress feed (run/update-phase.json, bind-mounted into the +# app container) so the browser shows the failure AT ONCE instead of spinning to its 10-minute +# timeout. Best-effort — a write failure must never change the update's outcome. +write_update_error() { + local f="${CLUSEV_DIR}/run/update-phase.json" + mkdir -p "${CLUSEV_DIR}/run" 2>/dev/null || true + printf '{"stage":"error","at":%s}\n' "$(date +%s 2>/dev/null || echo 0)" > "$f" 2>/dev/null || true + chmod 644 "$f" 2>/dev/null || true + return 0 +} + # Restart the stack and consume the sentinel. `up -d` (not bare `restart`) so a # changed image/config is also applied and any down service is brought back. do_restart() { @@ -68,19 +79,34 @@ run_once() { # needs root for docker/systemd/apt. do_update() { TAG="clusev-update" - [ -f "$COMPOSE_FILE" ] || { log "compose file not found: $COMPOSE_FILE — skipping"; return 1; } + # Every failure path below publishes the error stage too, so a page opened by the dashboard never + # spins to the timeout when the update can't even start. + [ -f "$COMPOSE_FILE" ] || { log "compose file not found: $COMPOSE_FILE — skipping"; write_update_error; return 1; } local updater="${CLUSEV_DIR}/update.sh" if [ ! -f "$updater" ]; then log "update.sh not found at $updater — clearing sentinel, nothing to do" rm -f "$CLUSEV_UPDATE_SIGNAL" + write_update_error return 1 fi rm -f "$CLUSEV_UPDATE_SIGNAL" && log "update sentinel consumed: $CLUSEV_UPDATE_SIGNAL" log "update requested — running update.sh in ${CLUSEV_DIR}" - if bash "$updater"; then + # Tee the whole run to a host log the operator can inspect, and cap it with a generous timeout so + # a truly-stuck step (e.g. a hung build) fails LOUDLY rather than pinning the progress page. On ANY + # non-zero exit (update.sh or its exec'd install.sh — the exit code propagates through exec) publish + # the error stage so the browser stops spinning and shows the failure. `timeout` exit 124 = capped. + local logf="${CLUSEV_DIR}/run/update.log" rc=0 + mkdir -p "${CLUSEV_DIR}/run" 2>/dev/null || true + timeout -k 30 1800 bash "$updater" 2>&1 | tee "$logf" || rc=${PIPESTATUS[0]} + if [ "$rc" = 0 ]; then log "update applied" else - log "update.sh FAILED — see output above; re-trigger from the dashboard to retry" + if [ "$rc" = 124 ]; then + log "update.sh TIMED OUT after 30m — see $logf; re-trigger from the dashboard to retry" + else + log "update.sh FAILED (exit $rc) — see $logf and the journal; re-trigger from the dashboard to retry" + fi + write_update_error return 1 fi } diff --git a/lang/de/update.php b/lang/de/update.php index 3090422..d98d0ff 100644 --- a/lang/de/update.php +++ b/lang/de/update.php @@ -18,4 +18,9 @@ return [ 'timeout_heading' => 'Update dauert ungewöhnlich lange', 'timeout_hint' => 'Bitte die Seite manuell neu laden oder die Logs prüfen (journalctl -u clusev-update.service).', 'reload_button' => 'Neu laden', + + // Fehler — der Host-Updater ist mit Fehler beendet (git pull, Build oder Migration) oder im Timeout. + 'error_heading' => 'Update fehlgeschlagen', + 'error_hint' => 'Der Stack konnte nicht aktualisiert werden — die bisherige Version läuft weiter. Ursache im Host-Log prüfen (journalctl -u clusev-update.service, oder run/update.log), beheben und das Update erneut über das Dashboard auslösen.', + 'back_button' => 'Zurück', ]; diff --git a/lang/en/update.php b/lang/en/update.php index cba1c9c..2f664ed 100644 --- a/lang/en/update.php +++ b/lang/en/update.php @@ -18,4 +18,9 @@ return [ 'timeout_heading' => 'Update is taking unusually long', 'timeout_hint' => 'Please reload the page manually or check the logs (journalctl -u clusev-update.service).', 'reload_button' => 'Reload', + + // Error — the host updater exited non-zero (git pull, build or migrate failed) or timed out. + 'error_heading' => 'Update failed', + 'error_hint' => 'The stack could not be updated — your previous version is still running. Check the host log for the cause (journalctl -u clusev-update.service, or run/update.log), fix it, then re-trigger the update from the dashboard.', + 'back_button' => 'Back', ]; diff --git a/resources/views/update-progress.blade.php b/resources/views/update-progress.blade.php index d24f01b..7d7fc59 100644 --- a/resources/views/update-progress.blade.php +++ b/resources/views/update-progress.blade.php @@ -63,6 +63,23 @@ {{ __('update.reload_button') }} + + {{-- Error: the host updater exited non-zero (or timed out). Surfaced by the + update-status feed reporting stage=error, so the page stops at once instead + of spinning to the 10-minute timeout. The previous version keeps running. --}} + {{-- Phase checklist --}} @@ -199,6 +216,7 @@ var runningEl = document.getElementById('js-status-running'); var doneEl = document.getElementById('js-status-done'); var timeoutEl = document.getElementById('js-status-timeout'); + var errorEl = document.getElementById('js-status-error'); // ── Polling ──────────────────────────────────────────────────────────── var stableNew = 0; // consecutive polls that have seen the NEW app live @@ -236,6 +254,28 @@ timeoutEl.classList.remove('hidden'); } + // The host updater reported stage=error (exited non-zero or was killed by its timeout). + // Stop everything and show the failure at once — mark the phase that was active as failed. + function showError() { + if (done) return; + done = true; + clearInterval(pollInterval); + clearInterval(statusInterval); + clearInterval(tickInterval); + if (currentPhase >= 0 && currentPhase < PHASE_KEYS.length) { + var key = PHASE_KEYS[currentPhase]; + var dot = document.getElementById('js-phase-dot-' + key); + var label = document.getElementById('js-phase-label-' + key); + var li = document.getElementById('js-phase-' + key); + if (li) li.classList.remove('border-accent/30', 'bg-accent/5'); + if (dot) dot.style.backgroundColor = 'var(--color-offline)'; + if (label) label.style.color = 'var(--color-offline)'; + } + spinnerEl.classList.add('hidden'); + runningEl.classList.add('hidden'); + errorEl.classList.remove('hidden'); + } + function tick() { var elapsed = baseElapsedMs + (Date.now() - clientLoad); @@ -315,6 +355,7 @@ // previous run triggering a premature completion on refresh. if (!startedAt || typeof data.at !== 'number' || data.at < startedAt - 5) return; feedActive = true; + if (data.stage === 'error') { showError(); return; } if (data.stage === 'done') { showDone(); return; } var idx = STAGE_INDEX[data.stage]; // The live feed is authoritative — the host writes stages monotonically forward, diff --git a/tests/Feature/UpdateProgressTest.php b/tests/Feature/UpdateProgressTest.php index caf0a77..7819e9c 100644 --- a/tests/Feature/UpdateProgressTest.php +++ b/tests/Feature/UpdateProgressTest.php @@ -288,6 +288,34 @@ class UpdateProgressTest extends TestCase } } + public function test_update_status_surfaces_the_error_stage(): void + { + // The host updater (watch.sh) writes stage=error on ANY failure so the progress page can stop + // and show the failure at once instead of spinning to its 10-minute timeout. + $dir = storage_path('app/restart-signal'); + @mkdir($dir, 0775, true); + file_put_contents($dir.'/update-phase.json', json_encode(['stage' => 'error', 'at' => 123])); + + try { + $this->getJson('/update-status.json') + ->assertOk() + ->assertExactJson(['stage' => 'error', 'at' => 123]); + } finally { + @unlink($dir.'/update-phase.json'); + } + } + + public function test_update_progress_page_renders_an_error_state(): void + { + $body = $this->get(route('update.progress'))->content(); + + // A failed update must surface — the page needs an error branch driven by the feed's + // stage=error, not just the running/done/timeout states. + $this->assertStringContainsString('js-status-error', $body); + $this->assertStringContainsString('function showError', $body); + $this->assertStringContainsString("data.stage === 'error'", $body); + } + // ── (c) resumable progress (survives a browser refresh) ─────────────────── public function test_request_update_writes_a_recent_start_marker(): void diff --git a/update.sh b/update.sh index bb450a4..54906d0 100755 --- a/update.sh +++ b/update.sh @@ -19,6 +19,14 @@ set -euo pipefail cd "$(dirname "$0")" REPO_DIR="$(pwd)" +# Never let a private-repo credential miss HANG the update: git must fail fast instead of blocking on +# an interactive username/password prompt (the systemd updater has no TTY, so a prompt would wait +# forever — the classic "update spins with no error"). Also abort a stalled transfer instead of +# hanging on it. The pull itself is additionally wrapped in `timeout` below. +export GIT_TERMINAL_PROMPT=0 +export GIT_HTTP_LOW_SPEED_LIMIT="${GIT_HTTP_LOW_SPEED_LIMIT:-1000}" +export GIT_HTTP_LOW_SPEED_TIME="${GIT_HTTP_LOW_SPEED_TIME:-30}" + # set_stage: publish a coarse update stage for the in-browser progress screen (best-effort — the # app container serves run/update-phase.json to it). A write failure must never fail the update. # Written first as "fetch" so the screen leaves the time-guess behind the moment the pull begins, @@ -51,8 +59,8 @@ if [ "${CLUSEV_UPDATE_REEXEC:-0}" != 1 ]; then # 2. fast-forward pull only — never auto-merge or throw away local edits before_hash="$(sha256sum -- "$0" | cut -d' ' -f1)" info "Hole Aktualisierungen (git pull --ff-only) ..." - git pull --ff-only \ - || die "git pull fehlgeschlagen (lokale Aenderungen oder divergierter Branch). Mit 'git status' pruefen und erneut versuchen." + timeout 300 git pull --ff-only \ + || die "git pull fehlgeschlagen — Timeout, fehlende Zugangsdaten zum privaten Repo, lokale Aenderungen oder divergierter Branch. Mit 'git status' und dem Repo-Zugriff (Token im origin-Remote) pruefen und erneut versuchen." after_hash="$(sha256sum -- "$0" | cut -d' ' -f1)" # 3. self-update: relaunch the new update.sh if it changed