fix(update): surface update failures instead of spinning forever

A dashboard-triggered update that failed on the host left the /update-progress
page spinning to a vague 10-minute timeout — no error was ever shown, because
nothing wrote a failure marker for the page to read. Fix: make failures visible
and prevent the credential-prompt hang class.

- docker/restart-sentinel/watch.sh: run update.sh under `timeout -k 30 1800` with
  its output tee'd to run/update.log; on ANY non-zero exit (update.sh or its
  exec'd install.sh) — and on the early compose-missing / updater-missing paths —
  write {"stage":"error"} to run/update-phase.json via write_update_error().
- update.sh: export GIT_TERMINAL_PROMPT=0 + GIT_HTTP_LOW_SPEED_* and wrap the pull
  in `timeout 300`, so a private-repo credential miss fails fast instead of
  hanging on a non-interactive prompt.
- DeploymentService::updatePhase(): whitelist the 'error' stage.
- update-progress.blade.php: add an error state (#js-status-error) + showError();
  the status-feed poll now stops and shows it on stage=error, marking the active
  phase red — instead of looping.
- lang/{en,de}/update.php: error_heading / error_hint / back_button.
- UpdateProgressTest: feed surfaces stage=error; page renders the error branch.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

app/Services/DeploymentService.php

@@ -260,7 +260,7 @@ class DeploymentService
             return null;
         }
         $data = json_decode((string) @file_get_contents($path), true);
-        if (! is_array($data) || ! in_array($data['stage'] ?? null, ['fetch', 'build', 'restart', 'migrate', 'done'], true)) {
+        if (! is_array($data) || ! in_array($data['stage'] ?? null, ['fetch', 'build', 'restart', 'migrate', 'done', 'error'], true)) {
             return null;
         }
         // Require a real timestamp: the progress page's freshness logic ties a stage to THIS update

docker/restart-sentinel/watch.sh

@@ -35,6 +35,17 @@ COMPOSE_FILE="${CLUSEV_DIR}/docker-compose.prod.yml"
 TAG="clusev-restart"
 log() { printf '%s %s: %s\n' "$(date -Is)" "$TAG" "$*"; }
 
+# Publish an "error" stage to the update-progress feed (run/update-phase.json, bind-mounted into the
+# app container) so the browser shows the failure AT ONCE instead of spinning to its 10-minute
+# timeout. Best-effort — a write failure must never change the update's outcome.
+write_update_error() {
+  local f="${CLUSEV_DIR}/run/update-phase.json"
+  mkdir -p "${CLUSEV_DIR}/run" 2>/dev/null || true
+  printf '{"stage":"error","at":%s}\n' "$(date +%s 2>/dev/null || echo 0)" > "$f" 2>/dev/null || true
+  chmod 644 "$f" 2>/dev/null || true
+  return 0
+}
+
 # Restart the stack and consume the sentinel. `up -d` (not bare `restart`) so a
 # changed image/config is also applied and any down service is brought back.
 do_restart() {
@@ -68,19 +79,34 @@ run_once() {
 # needs root for docker/systemd/apt.
 do_update() {
   TAG="clusev-update"
-  [ -f "$COMPOSE_FILE" ] || { log "compose file not found: $COMPOSE_FILE — skipping"; return 1; }
+  # Every failure path below publishes the error stage too, so a page opened by the dashboard never
+  # spins to the timeout when the update can't even start.
+  [ -f "$COMPOSE_FILE" ] || { log "compose file not found: $COMPOSE_FILE — skipping"; write_update_error; return 1; }
   local updater="${CLUSEV_DIR}/update.sh"
   if [ ! -f "$updater" ]; then
     log "update.sh not found at $updater — clearing sentinel, nothing to do"
     rm -f "$CLUSEV_UPDATE_SIGNAL"
+    write_update_error
     return 1
   fi
   rm -f "$CLUSEV_UPDATE_SIGNAL" && log "update sentinel consumed: $CLUSEV_UPDATE_SIGNAL"
   log "update requested — running update.sh in ${CLUSEV_DIR}"
-  if bash "$updater"; then
+  # Tee the whole run to a host log the operator can inspect, and cap it with a generous timeout so
+  # a truly-stuck step (e.g. a hung build) fails LOUDLY rather than pinning the progress page. On ANY
+  # non-zero exit (update.sh or its exec'd install.sh — the exit code propagates through exec) publish
+  # the error stage so the browser stops spinning and shows the failure. `timeout` exit 124 = capped.
+  local logf="${CLUSEV_DIR}/run/update.log" rc=0
+  mkdir -p "${CLUSEV_DIR}/run" 2>/dev/null || true
+  timeout -k 30 1800 bash "$updater" 2>&1 | tee "$logf" || rc=${PIPESTATUS[0]}
+  if [ "$rc" = 0 ]; then
     log "update applied"
   else
-    log "update.sh FAILED — see output above; re-trigger from the dashboard to retry"
+    if [ "$rc" = 124 ]; then
+      log "update.sh TIMED OUT after 30m — see $logf; re-trigger from the dashboard to retry"
+    else
+      log "update.sh FAILED (exit $rc) — see $logf and the journal; re-trigger from the dashboard to retry"
+    fi
+    write_update_error
     return 1
   fi
 }

lang/de/update.php

@@ -18,4 +18,9 @@ return [
     'timeout_heading' => 'Update dauert ungewöhnlich lange',
     'timeout_hint' => 'Bitte die Seite manuell neu laden oder die Logs prüfen (journalctl -u clusev-update.service).',
     'reload_button' => 'Neu laden',
+
+    // Fehler — der Host-Updater ist mit Fehler beendet (git pull, Build oder Migration) oder im Timeout.
+    'error_heading' => 'Update fehlgeschlagen',
+    'error_hint' => 'Der Stack konnte nicht aktualisiert werden — die bisherige Version läuft weiter. Ursache im Host-Log prüfen (journalctl -u clusev-update.service, oder run/update.log), beheben und das Update erneut über das Dashboard auslösen.',
+    'back_button' => 'Zurück',
 ];

lang/en/update.php

@@ -18,4 +18,9 @@ return [
     'timeout_heading' => 'Update is taking unusually long',
     'timeout_hint' => 'Please reload the page manually or check the logs (journalctl -u clusev-update.service).',
     'reload_button' => 'Reload',
+
+    // Error — the host updater exited non-zero (git pull, build or migrate failed) or timed out.
+    'error_heading' => 'Update failed',
+    'error_hint' => 'The stack could not be updated — your previous version is still running. Check the host log for the cause (journalctl -u clusev-update.service, or run/update.log), fix it, then re-trigger the update from the dashboard.',
+    'back_button' => 'Back',
 ];

resources/views/update-progress.blade.php

@@ -63,6 +63,23 @@
                             {{ __('update.reload_button') }}
                         </button>
                     </div>
+
+                    {{-- Error: the host updater exited non-zero (or timed out). Surfaced by the
+                         update-status feed reporting stage=error, so the page stops at once instead
+                         of spinning to the 10-minute timeout. The previous version keeps running. --}}
+                    <div id="js-status-error" class="hidden">
+                        <svg class="mx-auto mb-3 h-12 w-12 text-offline" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="1.75" aria-hidden="true"><path stroke-linecap="round" stroke-linejoin="round" d="M12 9v2m0 4h.01M10.29 3.86L1.82 18a2 2 0 001.71 3h16.94a2 2 0 001.71-3L13.71 3.86a2 2 0 00-3.42 0z"/></svg>
+                        <h1 class="font-display text-xl font-semibold text-offline">{{ __('update.error_heading') }}</h1>
+                        <p class="mt-2 text-sm leading-relaxed text-ink-2">{{ __('update.error_hint') }}</p>
+                        <div class="mt-4 flex items-center justify-center gap-2">
+                            <a href="{{ $returnPath }}" class="inline-flex items-center gap-2 rounded-md border border-line bg-raised px-4 py-2 font-mono text-sm text-ink transition-colors hover:border-accent/40 hover:text-accent">
+                                {{ __('update.back_button') }}
+                            </a>
+                            <button onclick="window.location.reload()" class="inline-flex items-center gap-2 rounded-md border border-line bg-raised px-4 py-2 font-mono text-sm text-ink-2 transition-colors hover:border-accent/40 hover:text-accent">
+                                {{ __('update.reload_button') }}
+                            </button>
+                        </div>
+                    </div>
                 </div>
 
                 {{-- Phase checklist --}}
@@ -199,6 +216,7 @@
         var runningEl       = document.getElementById('js-status-running');
         var doneEl          = document.getElementById('js-status-done');
         var timeoutEl       = document.getElementById('js-status-timeout');
+        var errorEl         = document.getElementById('js-status-error');
 
         // ── Polling ────────────────────────────────────────────────────────────
         var stableNew        = 0;    // consecutive polls that have seen the NEW app live
@@ -236,6 +254,28 @@
             timeoutEl.classList.remove('hidden');
         }
 
+        // The host updater reported stage=error (exited non-zero or was killed by its timeout).
+        // Stop everything and show the failure at once — mark the phase that was active as failed.
+        function showError() {
+            if (done) return;
+            done = true;
+            clearInterval(pollInterval);
+            clearInterval(statusInterval);
+            clearInterval(tickInterval);
+            if (currentPhase >= 0 && currentPhase < PHASE_KEYS.length) {
+                var key   = PHASE_KEYS[currentPhase];
+                var dot   = document.getElementById('js-phase-dot-' + key);
+                var label = document.getElementById('js-phase-label-' + key);
+                var li    = document.getElementById('js-phase-' + key);
+                if (li)    li.classList.remove('border-accent/30', 'bg-accent/5');
+                if (dot)   dot.style.backgroundColor = 'var(--color-offline)';
+                if (label) label.style.color = 'var(--color-offline)';
+            }
+            spinnerEl.classList.add('hidden');
+            runningEl.classList.add('hidden');
+            errorEl.classList.remove('hidden');
+        }
+
         function tick() {
             var elapsed = baseElapsedMs + (Date.now() - clientLoad);
 
@@ -315,6 +355,7 @@
                     // previous run triggering a premature completion on refresh.
                     if (!startedAt || typeof data.at !== 'number' || data.at < startedAt - 5) return;
                     feedActive = true;
+                    if (data.stage === 'error') { showError(); return; }
                     if (data.stage === 'done') { showDone(); return; }
                     var idx = STAGE_INDEX[data.stage];
                     // The live feed is authoritative — the host writes stages monotonically forward,

tests/Feature/UpdateProgressTest.php

@@ -288,6 +288,34 @@ class UpdateProgressTest extends TestCase
         }
     }
 
+    public function test_update_status_surfaces_the_error_stage(): void
+    {
+        // The host updater (watch.sh) writes stage=error on ANY failure so the progress page can stop
+        // and show the failure at once instead of spinning to its 10-minute timeout.
+        $dir = storage_path('app/restart-signal');
+        @mkdir($dir, 0775, true);
+        file_put_contents($dir.'/update-phase.json', json_encode(['stage' => 'error', 'at' => 123]));
+
+        try {
+            $this->getJson('/update-status.json')
+                ->assertOk()
+                ->assertExactJson(['stage' => 'error', 'at' => 123]);
+        } finally {
+            @unlink($dir.'/update-phase.json');
+        }
+    }
+
+    public function test_update_progress_page_renders_an_error_state(): void
+    {
+        $body = $this->get(route('update.progress'))->content();
+
+        // A failed update must surface — the page needs an error branch driven by the feed's
+        // stage=error, not just the running/done/timeout states.
+        $this->assertStringContainsString('js-status-error', $body);
+        $this->assertStringContainsString('function showError', $body);
+        $this->assertStringContainsString("data.stage === 'error'", $body);
+    }
+
     // ── (c) resumable progress (survives a browser refresh) ───────────────────
 
     public function test_request_update_writes_a_recent_start_marker(): void

update.sh

@@ -19,6 +19,14 @@ set -euo pipefail
 cd "$(dirname "$0")"
 REPO_DIR="$(pwd)"
 
+# Never let a private-repo credential miss HANG the update: git must fail fast instead of blocking on
+# an interactive username/password prompt (the systemd updater has no TTY, so a prompt would wait
+# forever — the classic "update spins with no error"). Also abort a stalled transfer instead of
+# hanging on it. The pull itself is additionally wrapped in `timeout` below.
+export GIT_TERMINAL_PROMPT=0
+export GIT_HTTP_LOW_SPEED_LIMIT="${GIT_HTTP_LOW_SPEED_LIMIT:-1000}"
+export GIT_HTTP_LOW_SPEED_TIME="${GIT_HTTP_LOW_SPEED_TIME:-30}"
+
 # set_stage: publish a coarse update stage for the in-browser progress screen (best-effort — the
 # app container serves run/update-phase.json to it). A write failure must never fail the update.
 # Written first as "fetch" so the screen leaves the time-guess behind the moment the pull begins,
@@ -51,8 +59,8 @@ if [ "${CLUSEV_UPDATE_REEXEC:-0}" != 1 ]; then
   # 2. fast-forward pull only — never auto-merge or throw away local edits
   before_hash="$(sha256sum -- "$0" | cut -d' ' -f1)"
   info "Hole Aktualisierungen (git pull --ff-only) ..."
-  git pull --ff-only \
+  timeout 300 git pull --ff-only \
-    || die "git pull fehlgeschlagen (lokale Aenderungen oder divergierter Branch). Mit 'git status' pruefen und erneut versuchen."
+    || die "git pull fehlgeschlagen — Timeout, fehlende Zugangsdaten zum privaten Repo, lokale Aenderungen oder divergierter Branch. Mit 'git status' und dem Repo-Zugriff (Token im origin-Remote) pruefen und erneut versuchen."
   after_hash="$(sha256sum -- "$0" | cut -d' ' -f1)"
 
   # 3. self-update: relaunch the new update.sh if it changed