fix(engine): durable queueing for long provisioning steps

- Dedicated 'provisioning' queue connection (retry_after 2400) + worker
  (--timeout=2100); AdvanceRunJob tries=1, timeout=2100 on that queue.
- Run lock TTL raised to 2100s so a long step can't be run concurrently by a
  duplicate tick-dispatched job.
- StartHostOnboarding creates host+run in a transaction, dispatches after commit.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

app/Actions/StartHostOnboarding.php

@@ -6,6 +6,7 @@ use App\Models\Host;
 use App\Models\ProvisioningRun;
 use App\Provisioning\Jobs\AdvanceRunJob;
 use Illuminate\Support\Facades\Crypt;
+use Illuminate\Support\Facades\DB;
 
 /**
  * Registers a fresh host and kicks off its onboarding run. The one-time root
@@ -19,6 +20,9 @@ class StartHostOnboarding
      */
     public function run(array $input): Host
     {
+        // Host + run are created atomically; a partial insert would otherwise
+        // leave a permanently pending host with no run.
+        [$host, $run] = DB::transaction(function () use ($input) {
             $host = Host::create([
                 'name' => $input['name'],
                 'datacenter' => $input['datacenter'],
@@ -35,7 +39,10 @@ class StartHostOnboarding
                 'context' => ['root_password' => Crypt::encryptString($input['root_password'])],
             ]);
 
-        AdvanceRunJob::dispatch($run->uuid);
+            return [$host, $run];
+        });
+
+        AdvanceRunJob::dispatch($run->uuid); // after commit — the run definitely exists
 
         return $host;
     }

app/Provisioning/Jobs/AdvanceRunJob.php

@@ -14,8 +14,15 @@ class AdvanceRunJob implements ShouldQueue
 {
     use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
 
+    /** The run state machine owns retries — the job itself runs once. */
+    public $tries = 1;
+
+    /** Above the longest step (InstallProxmoxVe: 1800s) so the worker doesn't kill it. */
+    public $timeout = 2100;
+
     public function __construct(public string $runUuid)
     {
+        $this->onConnection('provisioning');
         $this->onQueue('provisioning');
     }
 

app/Provisioning/RunRunner.php

@@ -20,7 +20,9 @@ class RunRunner
 
     public function advance(ProvisioningRun $run): void
     {
-        $lock = Cache::lock('run:'.$run->uuid, 120);
+        // Lock must outlive the longest step so a duplicate job (dispatched by
+        // the minutely tick while this one still runs) can't execute concurrently.
+        $lock = Cache::lock('run:'.$run->uuid, 2100);
 
         if (! $lock->get()) {
             return; // another worker already holds this run

config/queue.php

@@ -73,6 +73,18 @@ return [
             'after_commit' => false,
         ],
 
+        // Long-running host/customer provisioning steps. retry_after must exceed
+        // the longest step timeout (AdvanceRunJob::$timeout) so a running job is
+        // never re-queued underneath itself.
+        'provisioning' => [
+            'driver' => 'redis',
+            'connection' => env('REDIS_QUEUE_CONNECTION', 'default'),
+            'queue' => 'provisioning',
+            'retry_after' => (int) env('PROVISIONING_QUEUE_RETRY_AFTER', 2400),
+            'block_for' => null,
+            'after_commit' => false,
+        ],
+
         'deferred' => [
             'driver' => 'deferred',
         ],

docker-compose.yml

@@ -52,6 +52,19 @@ services:
       - redis
       - mariadb
 
+  # Dedicated worker for long-running provisioning steps (own timeout, single try;
+  # the DB state machine owns retries). Separate from the fast default queue.
+  queue-provisioning:
+    image: clupilot-app:dev
+    restart: unless-stopped
+    command: php artisan queue:work provisioning --queue=provisioning --tries=1 --timeout=2100 --sleep=3
+    volumes:
+      - .:/var/www/html
+    depends_on:
+      - app
+      - redis
+      - mariadb
+
   scheduler:
     image: clupilot-app:dev
     restart: unless-stopped

docs/superpowers/specs/2026-07-25-host-onboarding-design.md

@@ -220,6 +220,7 @@ Routen englisch (R13). Alle Texte DE+EN identische Keys (R16). Nur Token-Utiliti
 4. **`apt full-upgrade`-Dauer** → großzügiges Step-Timeout.
 5. **Token-Idempotenz:** Secret nur bei Erzeugung sichtbar → Crash-vor-Persistenz muss Token löschen+neu.
 6. **Regeln-Gotchas** (State-Handoff §6): `@js()` in Blade-Komponenten, Test-Isolation (sqlite `:memory:`), CSRF in Tests, `@verbatim`, Locale, Vite-Reload.
+7. **Langlaufende Schritte:** `InstallProxmoxVe` blockiert einen Worker bis zu 1800 s synchron. v1.0 löst das über eine dedizierte `provisioning`-Queue (eigener Worker, `--timeout=2100`, `retry_after=2400`, Run-Lock 2100 s). v1.1-Verbesserung: lange Kommandos in resumierbare Teil-Jobs zerlegen statt synchron zu blockieren.
 
 ---