diff --git a/app/Actions/StartHostOnboarding.php b/app/Actions/StartHostOnboarding.php index 30e73f8..9c9ca0a 100644 --- a/app/Actions/StartHostOnboarding.php +++ b/app/Actions/StartHostOnboarding.php @@ -6,6 +6,7 @@ use App\Models\Host; use App\Models\ProvisioningRun; use App\Provisioning\Jobs\AdvanceRunJob; use Illuminate\Support\Facades\Crypt; +use Illuminate\Support\Facades\DB; /** * Registers a fresh host and kicks off its onboarding run. The one-time root @@ -19,23 +20,29 @@ class StartHostOnboarding */ public function run(array $input): Host { - $host = Host::create([ - 'name' => $input['name'], - 'datacenter' => $input['datacenter'], - 'public_ip' => $input['public_ip'], - 'status' => 'pending', - ]); + // Host + run are created atomically; a partial insert would otherwise + // leave a permanently pending host with no run. + [$host, $run] = DB::transaction(function () use ($input) { + $host = Host::create([ + 'name' => $input['name'], + 'datacenter' => $input['datacenter'], + 'public_ip' => $input['public_ip'], + 'status' => 'pending', + ]); - $run = ProvisioningRun::create([ - 'subject_type' => Host::class, - 'subject_id' => $host->id, - 'pipeline' => 'host', - 'status' => ProvisioningRun::STATUS_PENDING, - 'current_step' => 0, - 'context' => ['root_password' => Crypt::encryptString($input['root_password'])], - ]); + $run = ProvisioningRun::create([ + 'subject_type' => Host::class, + 'subject_id' => $host->id, + 'pipeline' => 'host', + 'status' => ProvisioningRun::STATUS_PENDING, + 'current_step' => 0, + 'context' => ['root_password' => Crypt::encryptString($input['root_password'])], + ]); - AdvanceRunJob::dispatch($run->uuid); + return [$host, $run]; + }); + + AdvanceRunJob::dispatch($run->uuid); // after commit — the run definitely exists return $host; } diff --git a/app/Provisioning/Jobs/AdvanceRunJob.php b/app/Provisioning/Jobs/AdvanceRunJob.php index 1ac5f63..1c8444a 100644 --- a/app/Provisioning/Jobs/AdvanceRunJob.php +++ b/app/Provisioning/Jobs/AdvanceRunJob.php @@ -14,8 +14,15 @@ class AdvanceRunJob implements ShouldQueue { use Dispatchable, InteractsWithQueue, Queueable, SerializesModels; + /** The run state machine owns retries — the job itself runs once. */ + public $tries = 1; + + /** Above the longest step (InstallProxmoxVe: 1800s) so the worker doesn't kill it. */ + public $timeout = 2100; + public function __construct(public string $runUuid) { + $this->onConnection('provisioning'); $this->onQueue('provisioning'); } diff --git a/app/Provisioning/RunRunner.php b/app/Provisioning/RunRunner.php index 692716e..0a5725c 100644 --- a/app/Provisioning/RunRunner.php +++ b/app/Provisioning/RunRunner.php @@ -20,7 +20,9 @@ class RunRunner public function advance(ProvisioningRun $run): void { - $lock = Cache::lock('run:'.$run->uuid, 120); + // Lock must outlive the longest step so a duplicate job (dispatched by + // the minutely tick while this one still runs) can't execute concurrently. + $lock = Cache::lock('run:'.$run->uuid, 2100); if (! $lock->get()) { return; // another worker already holds this run diff --git a/config/queue.php b/config/queue.php index 79c2c0a..1a61c7f 100644 --- a/config/queue.php +++ b/config/queue.php @@ -73,6 +73,18 @@ return [ 'after_commit' => false, ], + // Long-running host/customer provisioning steps. retry_after must exceed + // the longest step timeout (AdvanceRunJob::$timeout) so a running job is + // never re-queued underneath itself. + 'provisioning' => [ + 'driver' => 'redis', + 'connection' => env('REDIS_QUEUE_CONNECTION', 'default'), + 'queue' => 'provisioning', + 'retry_after' => (int) env('PROVISIONING_QUEUE_RETRY_AFTER', 2400), + 'block_for' => null, + 'after_commit' => false, + ], + 'deferred' => [ 'driver' => 'deferred', ], diff --git a/docker-compose.yml b/docker-compose.yml index 8c33b17..3f01d88 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,6 +52,19 @@ services: - redis - mariadb + # Dedicated worker for long-running provisioning steps (own timeout, single try; + # the DB state machine owns retries). Separate from the fast default queue. + queue-provisioning: + image: clupilot-app:dev + restart: unless-stopped + command: php artisan queue:work provisioning --queue=provisioning --tries=1 --timeout=2100 --sleep=3 + volumes: + - .:/var/www/html + depends_on: + - app + - redis + - mariadb + scheduler: image: clupilot-app:dev restart: unless-stopped diff --git a/docs/superpowers/specs/2026-07-25-host-onboarding-design.md b/docs/superpowers/specs/2026-07-25-host-onboarding-design.md index ec5d60e..610a62d 100644 --- a/docs/superpowers/specs/2026-07-25-host-onboarding-design.md +++ b/docs/superpowers/specs/2026-07-25-host-onboarding-design.md @@ -220,6 +220,7 @@ Routen englisch (R13). Alle Texte DE+EN identische Keys (R16). Nur Token-Utiliti 4. **`apt full-upgrade`-Dauer** → großzügiges Step-Timeout. 5. **Token-Idempotenz:** Secret nur bei Erzeugung sichtbar → Crash-vor-Persistenz muss Token löschen+neu. 6. **Regeln-Gotchas** (State-Handoff §6): `@js()` in Blade-Komponenten, Test-Isolation (sqlite `:memory:`), CSRF in Tests, `@verbatim`, Locale, Vite-Reload. +7. **Langlaufende Schritte:** `InstallProxmoxVe` blockiert einen Worker bis zu 1800 s synchron. v1.0 löst das über eine dedizierte `provisioning`-Queue (eigener Worker, `--timeout=2100`, `retry_after=2400`, Run-Lock 2100 s). v1.1-Verbesserung: lange Kommandos in resumierbare Teil-Jobs zerlegen statt synchron zu blockieren. ---