fix(update): surface update failures instead of spinning forever
A dashboard-triggered update that failed on the host left the /update-progress
page spinning to a vague 10-minute timeout — no error was ever shown, because
nothing wrote a failure marker for the page to read. Fix: make failures visible
and prevent the credential-prompt hang class.
- docker/restart-sentinel/watch.sh: run update.sh under `timeout -k 30 1800` with
its output tee'd to run/update.log; on ANY non-zero exit (update.sh or its
exec'd install.sh) — and on the early compose-missing / updater-missing paths —
write {"stage":"error"} to run/update-phase.json via write_update_error().
- update.sh: export GIT_TERMINAL_PROMPT=0 + GIT_HTTP_LOW_SPEED_* and wrap the pull
in `timeout 300`, so a private-repo credential miss fails fast instead of
hanging on a non-interactive prompt.
- DeploymentService::updatePhase(): whitelist the 'error' stage.
- update-progress.blade.php: add an error state (#js-status-error) + showError();
the status-feed poll now stops and shows it on stage=error, marking the active
phase red — instead of looping.
- lang/{en,de}/update.php: error_heading / error_hint / back_button.
- UpdateProgressTest: feed surfaces stage=error; page renders the error branch.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
feat/v1-foundation
boban
parent
6348d269a5
commit
72b82df7a1
|
|
@ -260,7 +260,7 @@ class DeploymentService
|
|||
return null;
|
||||
}
|
||||
$data = json_decode((string) @file_get_contents($path), true);
|
||||
if (! is_array($data) || ! in_array($data['stage'] ?? null, ['fetch', 'build', 'restart', 'migrate', 'done'], true)) {
|
||||
if (! is_array($data) || ! in_array($data['stage'] ?? null, ['fetch', 'build', 'restart', 'migrate', 'done', 'error'], true)) {
|
||||
return null;
|
||||
}
|
||||
// Require a real timestamp: the progress page's freshness logic ties a stage to THIS update
|
||||
|
|
|
|||
|
|
@ -35,6 +35,17 @@ COMPOSE_FILE="${CLUSEV_DIR}/docker-compose.prod.yml"
|
|||
TAG="clusev-restart"
|
||||
log() { printf '%s %s: %s\n' "$(date -Is)" "$TAG" "$*"; }
|
||||
|
||||
# Publish an "error" stage to the update-progress feed (run/update-phase.json, bind-mounted into the
|
||||
# app container) so the browser shows the failure AT ONCE instead of spinning to its 10-minute
|
||||
# timeout. Best-effort — a write failure must never change the update's outcome.
|
||||
write_update_error() {
|
||||
local f="${CLUSEV_DIR}/run/update-phase.json"
|
||||
mkdir -p "${CLUSEV_DIR}/run" 2>/dev/null || true
|
||||
printf '{"stage":"error","at":%s}\n' "$(date +%s 2>/dev/null || echo 0)" > "$f" 2>/dev/null || true
|
||||
chmod 644 "$f" 2>/dev/null || true
|
||||
return 0
|
||||
}
|
||||
|
||||
# Restart the stack and consume the sentinel. `up -d` (not bare `restart`) so a
|
||||
# changed image/config is also applied and any down service is brought back.
|
||||
do_restart() {
|
||||
|
|
@ -68,19 +79,34 @@ run_once() {
|
|||
# needs root for docker/systemd/apt.
|
||||
do_update() {
|
||||
TAG="clusev-update"
|
||||
[ -f "$COMPOSE_FILE" ] || { log "compose file not found: $COMPOSE_FILE — skipping"; return 1; }
|
||||
# Every failure path below publishes the error stage too, so a page opened by the dashboard never
|
||||
# spins to the timeout when the update can't even start.
|
||||
[ -f "$COMPOSE_FILE" ] || { log "compose file not found: $COMPOSE_FILE — skipping"; write_update_error; return 1; }
|
||||
local updater="${CLUSEV_DIR}/update.sh"
|
||||
if [ ! -f "$updater" ]; then
|
||||
log "update.sh not found at $updater — clearing sentinel, nothing to do"
|
||||
rm -f "$CLUSEV_UPDATE_SIGNAL"
|
||||
write_update_error
|
||||
return 1
|
||||
fi
|
||||
rm -f "$CLUSEV_UPDATE_SIGNAL" && log "update sentinel consumed: $CLUSEV_UPDATE_SIGNAL"
|
||||
log "update requested — running update.sh in ${CLUSEV_DIR}"
|
||||
if bash "$updater"; then
|
||||
# Tee the whole run to a host log the operator can inspect, and cap it with a generous timeout so
|
||||
# a truly-stuck step (e.g. a hung build) fails LOUDLY rather than pinning the progress page. On ANY
|
||||
# non-zero exit (update.sh or its exec'd install.sh — the exit code propagates through exec) publish
|
||||
# the error stage so the browser stops spinning and shows the failure. `timeout` exit 124 = capped.
|
||||
local logf="${CLUSEV_DIR}/run/update.log" rc=0
|
||||
mkdir -p "${CLUSEV_DIR}/run" 2>/dev/null || true
|
||||
timeout -k 30 1800 bash "$updater" 2>&1 | tee "$logf" || rc=${PIPESTATUS[0]}
|
||||
if [ "$rc" = 0 ]; then
|
||||
log "update applied"
|
||||
else
|
||||
log "update.sh FAILED — see output above; re-trigger from the dashboard to retry"
|
||||
if [ "$rc" = 124 ]; then
|
||||
log "update.sh TIMED OUT after 30m — see $logf; re-trigger from the dashboard to retry"
|
||||
else
|
||||
log "update.sh FAILED (exit $rc) — see $logf and the journal; re-trigger from the dashboard to retry"
|
||||
fi
|
||||
write_update_error
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
|
|
|||
|
|
@ -18,4 +18,9 @@ return [
|
|||
'timeout_heading' => 'Update dauert ungewöhnlich lange',
|
||||
'timeout_hint' => 'Bitte die Seite manuell neu laden oder die Logs prüfen (journalctl -u clusev-update.service).',
|
||||
'reload_button' => 'Neu laden',
|
||||
|
||||
// Fehler — der Host-Updater ist mit Fehler beendet (git pull, Build oder Migration) oder im Timeout.
|
||||
'error_heading' => 'Update fehlgeschlagen',
|
||||
'error_hint' => 'Der Stack konnte nicht aktualisiert werden — die bisherige Version läuft weiter. Ursache im Host-Log prüfen (journalctl -u clusev-update.service, oder run/update.log), beheben und das Update erneut über das Dashboard auslösen.',
|
||||
'back_button' => 'Zurück',
|
||||
];
|
||||
|
|
|
|||
|
|
@ -18,4 +18,9 @@ return [
|
|||
'timeout_heading' => 'Update is taking unusually long',
|
||||
'timeout_hint' => 'Please reload the page manually or check the logs (journalctl -u clusev-update.service).',
|
||||
'reload_button' => 'Reload',
|
||||
|
||||
// Error — the host updater exited non-zero (git pull, build or migrate failed) or timed out.
|
||||
'error_heading' => 'Update failed',
|
||||
'error_hint' => 'The stack could not be updated — your previous version is still running. Check the host log for the cause (journalctl -u clusev-update.service, or run/update.log), fix it, then re-trigger the update from the dashboard.',
|
||||
'back_button' => 'Back',
|
||||
];
|
||||
|
|
|
|||
|
|
@ -63,6 +63,23 @@
|
|||
{{ __('update.reload_button') }}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{{-- Error: the host updater exited non-zero (or timed out). Surfaced by the
|
||||
update-status feed reporting stage=error, so the page stops at once instead
|
||||
of spinning to the 10-minute timeout. The previous version keeps running. --}}
|
||||
<div id="js-status-error" class="hidden">
|
||||
<svg class="mx-auto mb-3 h-12 w-12 text-offline" xmlns="http://www.w3.org/2000/svg" fill="none" viewBox="0 0 24 24" stroke="currentColor" stroke-width="1.75" aria-hidden="true"><path stroke-linecap="round" stroke-linejoin="round" d="M12 9v2m0 4h.01M10.29 3.86L1.82 18a2 2 0 001.71 3h16.94a2 2 0 001.71-3L13.71 3.86a2 2 0 00-3.42 0z"/></svg>
|
||||
<h1 class="font-display text-xl font-semibold text-offline">{{ __('update.error_heading') }}</h1>
|
||||
<p class="mt-2 text-sm leading-relaxed text-ink-2">{{ __('update.error_hint') }}</p>
|
||||
<div class="mt-4 flex items-center justify-center gap-2">
|
||||
<a href="{{ $returnPath }}" class="inline-flex items-center gap-2 rounded-md border border-line bg-raised px-4 py-2 font-mono text-sm text-ink transition-colors hover:border-accent/40 hover:text-accent">
|
||||
{{ __('update.back_button') }}
|
||||
</a>
|
||||
<button onclick="window.location.reload()" class="inline-flex items-center gap-2 rounded-md border border-line bg-raised px-4 py-2 font-mono text-sm text-ink-2 transition-colors hover:border-accent/40 hover:text-accent">
|
||||
{{ __('update.reload_button') }}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{{-- Phase checklist --}}
|
||||
|
|
@ -199,6 +216,7 @@
|
|||
var runningEl = document.getElementById('js-status-running');
|
||||
var doneEl = document.getElementById('js-status-done');
|
||||
var timeoutEl = document.getElementById('js-status-timeout');
|
||||
var errorEl = document.getElementById('js-status-error');
|
||||
|
||||
// ── Polling ────────────────────────────────────────────────────────────
|
||||
var stableNew = 0; // consecutive polls that have seen the NEW app live
|
||||
|
|
@ -236,6 +254,28 @@
|
|||
timeoutEl.classList.remove('hidden');
|
||||
}
|
||||
|
||||
// The host updater reported stage=error (exited non-zero or was killed by its timeout).
|
||||
// Stop everything and show the failure at once — mark the phase that was active as failed.
|
||||
function showError() {
|
||||
if (done) return;
|
||||
done = true;
|
||||
clearInterval(pollInterval);
|
||||
clearInterval(statusInterval);
|
||||
clearInterval(tickInterval);
|
||||
if (currentPhase >= 0 && currentPhase < PHASE_KEYS.length) {
|
||||
var key = PHASE_KEYS[currentPhase];
|
||||
var dot = document.getElementById('js-phase-dot-' + key);
|
||||
var label = document.getElementById('js-phase-label-' + key);
|
||||
var li = document.getElementById('js-phase-' + key);
|
||||
if (li) li.classList.remove('border-accent/30', 'bg-accent/5');
|
||||
if (dot) dot.style.backgroundColor = 'var(--color-offline)';
|
||||
if (label) label.style.color = 'var(--color-offline)';
|
||||
}
|
||||
spinnerEl.classList.add('hidden');
|
||||
runningEl.classList.add('hidden');
|
||||
errorEl.classList.remove('hidden');
|
||||
}
|
||||
|
||||
function tick() {
|
||||
var elapsed = baseElapsedMs + (Date.now() - clientLoad);
|
||||
|
||||
|
|
@ -315,6 +355,7 @@
|
|||
// previous run triggering a premature completion on refresh.
|
||||
if (!startedAt || typeof data.at !== 'number' || data.at < startedAt - 5) return;
|
||||
feedActive = true;
|
||||
if (data.stage === 'error') { showError(); return; }
|
||||
if (data.stage === 'done') { showDone(); return; }
|
||||
var idx = STAGE_INDEX[data.stage];
|
||||
// The live feed is authoritative — the host writes stages monotonically forward,
|
||||
|
|
|
|||
|
|
@ -288,6 +288,34 @@ class UpdateProgressTest extends TestCase
|
|||
}
|
||||
}
|
||||
|
||||
public function test_update_status_surfaces_the_error_stage(): void
|
||||
{
|
||||
// The host updater (watch.sh) writes stage=error on ANY failure so the progress page can stop
|
||||
// and show the failure at once instead of spinning to its 10-minute timeout.
|
||||
$dir = storage_path('app/restart-signal');
|
||||
@mkdir($dir, 0775, true);
|
||||
file_put_contents($dir.'/update-phase.json', json_encode(['stage' => 'error', 'at' => 123]));
|
||||
|
||||
try {
|
||||
$this->getJson('/update-status.json')
|
||||
->assertOk()
|
||||
->assertExactJson(['stage' => 'error', 'at' => 123]);
|
||||
} finally {
|
||||
@unlink($dir.'/update-phase.json');
|
||||
}
|
||||
}
|
||||
|
||||
public function test_update_progress_page_renders_an_error_state(): void
|
||||
{
|
||||
$body = $this->get(route('update.progress'))->content();
|
||||
|
||||
// A failed update must surface — the page needs an error branch driven by the feed's
|
||||
// stage=error, not just the running/done/timeout states.
|
||||
$this->assertStringContainsString('js-status-error', $body);
|
||||
$this->assertStringContainsString('function showError', $body);
|
||||
$this->assertStringContainsString("data.stage === 'error'", $body);
|
||||
}
|
||||
|
||||
// ── (c) resumable progress (survives a browser refresh) ───────────────────
|
||||
|
||||
public function test_request_update_writes_a_recent_start_marker(): void
|
||||
|
|
|
|||
12
update.sh
12
update.sh
|
|
@ -19,6 +19,14 @@ set -euo pipefail
|
|||
cd "$(dirname "$0")"
|
||||
REPO_DIR="$(pwd)"
|
||||
|
||||
# Never let a private-repo credential miss HANG the update: git must fail fast instead of blocking on
|
||||
# an interactive username/password prompt (the systemd updater has no TTY, so a prompt would wait
|
||||
# forever — the classic "update spins with no error"). Also abort a stalled transfer instead of
|
||||
# hanging on it. The pull itself is additionally wrapped in `timeout` below.
|
||||
export GIT_TERMINAL_PROMPT=0
|
||||
export GIT_HTTP_LOW_SPEED_LIMIT="${GIT_HTTP_LOW_SPEED_LIMIT:-1000}"
|
||||
export GIT_HTTP_LOW_SPEED_TIME="${GIT_HTTP_LOW_SPEED_TIME:-30}"
|
||||
|
||||
# set_stage: publish a coarse update stage for the in-browser progress screen (best-effort — the
|
||||
# app container serves run/update-phase.json to it). A write failure must never fail the update.
|
||||
# Written first as "fetch" so the screen leaves the time-guess behind the moment the pull begins,
|
||||
|
|
@ -51,8 +59,8 @@ if [ "${CLUSEV_UPDATE_REEXEC:-0}" != 1 ]; then
|
|||
# 2. fast-forward pull only — never auto-merge or throw away local edits
|
||||
before_hash="$(sha256sum -- "$0" | cut -d' ' -f1)"
|
||||
info "Hole Aktualisierungen (git pull --ff-only) ..."
|
||||
git pull --ff-only \
|
||||
|| die "git pull fehlgeschlagen (lokale Aenderungen oder divergierter Branch). Mit 'git status' pruefen und erneut versuchen."
|
||||
timeout 300 git pull --ff-only \
|
||||
|| die "git pull fehlgeschlagen — Timeout, fehlende Zugangsdaten zum privaten Repo, lokale Aenderungen oder divergierter Branch. Mit 'git status' und dem Repo-Zugriff (Token im origin-Remote) pruefen und erneut versuchen."
|
||||
after_hash="$(sha256sum -- "$0" | cut -d' ' -f1)"
|
||||
|
||||
# 3. self-update: relaunch the new update.sh if it changed
|
||||
|
|
|
|||
Loading…
Reference in New Issue