authorize('provisioning.retry'); // Atomically claim the run: only the request that transitions it out of // FAILED proceeds, so two concurrent retries can't double-dispatch. $claimed = ProvisioningRun::query() ->where('uuid', $uuid) ->where('status', ProvisioningRun::STATUS_FAILED) ->update([ 'status' => ProvisioningRun::STATUS_RUNNING, 'attempt' => 0, 'next_attempt_at' => now(), 'started_at' => now(), // reset the step timer so it doesn't re-time-out instantly 'error' => null, ]); if ($claimed === 0) { return; // already retried by a concurrent request } $run = ProvisioningRun::query()->where('uuid', $uuid)->first(); if ($run === null) { return; } // Move the subject out of its error state so the console reflects the retry. $subject = $run->subject; if ($subject instanceof Host) { $subject->update(['status' => 'onboarding']); } elseif ($subject instanceof Order) { $subject->update(['status' => 'provisioning']); Instance::query()->where('order_id', $subject->id)->where('status', 'failed')->update(['status' => 'provisioning']); } AdvanceRunJob::dispatch($run->uuid); $this->dispatch('notify', message: __('admin.run_retried')); } /** * A run that claims to be in progress but hasn't advanced in a while looks * stuck — unless it is in a legitimate scheduled backoff (next_attempt_at in * the future, e.g. RunRunner's retry delay of up to 300 s). */ private function isStale(ProvisioningRun $run): bool { if (! in_array($run->status, ['running', 'waiting'], true)) { return false; } if ($run->next_attempt_at !== null && $run->next_attempt_at->isFuture()) { return false; // waiting out a scheduled retry/poll — not stuck } // Compare against the CURRENT step's own allowed duration — some steps // (VM clone, Proxmox install) legitimately run for many minutes — plus a // small grace. Only past its own deadline without advancing is it stuck. $pipeline = config('provisioning.pipelines.'.$run->pipeline, []); $class = $pipeline[$run->current_step] ?? null; $maxSeconds = $class !== null ? app($class)->maxDuration() : 120; return $run->updated_at !== null && $run->updated_at->lt(now()->subSeconds($maxSeconds + 30)); } private function subjectLabel(ProvisioningRun $run): string { $subject = $run->subject; if ($subject instanceof Order) { return $subject->customer?->name ?? 'Order'; } if ($subject instanceof Host) { return $subject->name; } return class_basename($run->subject_type); } public function render() { $runs = ProvisioningRun::query()->latest('id')->limit(30)->get(); $active = $runs->first(fn (ProvisioningRun $r) => in_array( $r->status, ['pending', 'running', 'waiting'], true )) ?? $runs->first(); return view('livewire.admin.provisioning', [ 'hasActive' => $active !== null && in_array($active->status, ['pending', 'running', 'waiting'], true), 'rows' => $runs->map(function (ProvisioningRun $r) { $total = max(count(config('provisioning.pipelines.'.$r->pipeline, [])), 1); $done = $r->status === ProvisioningRun::STATUS_COMPLETED ? $total : $r->current_step; return [ 'uuid' => $r->uuid, 'customer' => $this->subjectLabel($r), 'pipeline' => $r->pipeline, 'step' => $r->status === 'completed' ? '—' : $this->currentStepLabel($r), 'n' => ($r->current_step + 1).'/'.$total, 'percent' => (int) round($done / $total * 100), 'attempt' => $r->attempt, 'state' => $this->runState($r), 'failed' => $r->status === ProvisioningRun::STATUS_FAILED, 'activity' => $r->updated_at?->diffForHumans() ?? '—', 'stale' => $this->isStale($r), ]; })->all(), 'panel' => $active ? $this->panelFor($active) : null, ]); } /** Compact "current run" readout for the right column. */ private function panelFor(ProvisioningRun $run): array { $pipeline = config('provisioning.pipelines.'.$run->pipeline, []); $total = max(count($pipeline), 1); $current = min($run->current_step, $total - 1); $completed = $run->status === ProvisioningRun::STATUS_COMPLETED; $done = $completed ? $total : $current; return [ 'uuid' => $run->uuid, 'subject' => $this->subjectLabel($run), 'pipeline' => $run->pipeline, 'attempt' => $run->attempt, 'status' => $this->runState($run), // running|done|failed 'failed' => $run->status === ProvisioningRun::STATUS_FAILED, 'current' => $completed ? null : (isset($pipeline[$current]) ? __(app($pipeline[$current])->label()) : null), 'next' => isset($pipeline[$current + 1]) && ! $completed ? __(app($pipeline[$current + 1])->label()) : null, 'error' => $run->error, 'done' => $done, 'total' => $total, 'percent' => (int) round($done / $total * 100), 'activity' => $run->updated_at?->diffForHumans() ?? '—', 'started' => $run->started_at?->diffForHumans() ?? null, 'stale' => $this->isStale($run), ]; } }