fix(engine): record wg peer only after verified handshake; async host purge

- ConfigureWireguard checks setup command results and only records the wg_peer
  resource after the handshake verifies, so a transient setup failure re-runs
  the full setup instead of getting stuck on the idempotent path.
- Host removal now deactivates immediately and queues PurgeHost, which deletes
  runs under the runner lock (waiting out a long step) — no 15s LockTimeout error.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

app/Livewire/Admin/ConfirmRemoveHost.php

@@ -3,13 +3,14 @@
 namespace App\Livewire\Admin;
 
 use App\Models\Host;
-use App\Provisioning\Jobs\RemoveWireguardPeer;
+use App\Provisioning\Jobs\PurgeHost;
-use Illuminate\Support\Facades\Cache;
 use LivewireUI\Modal\ModalComponent;
 
 /**
  * Confirmation for the destructive "remove host" action (R5). Deregisters the
- * CluPilot record only — it does not touch the physical server.
+ * CluPilot record only — it does not touch the physical server. The host is
+ * deactivated immediately and purged asynchronously so removal never blocks on
+ * (or races) an in-flight provisioning step.
  */
 class ConfirmRemoveHost extends ModalComponent
 {
@@ -28,21 +29,10 @@ class ConfirmRemoveHost extends ModalComponent
         $host = Host::query()->where('uuid', $this->uuid)->first();
 
         if ($host !== null) {
-            // Delete each run under its runner lock so an in-flight worker can't
+            // Deactivate now (out of placement, visibly removed), finalize on the
-            // keep mutating the server or write an event against a deleted run.
+            // provisioning worker which waits for the runner lock.
-            foreach ($host->runs()->get() as $run) {
+            $host->update(['status' => 'disabled']);
-                Cache::lock('run:'.$run->uuid, 30)->block(15, function () use ($run) {
+            PurgeHost::dispatch($host->uuid);
-                    $run->delete(); // cascades run_resources + step_events
-                });
-            }
-
-            // Drop the hub peer on the privileged provisioning worker (which owns
-            // wg0), so a freed wg_ip can't route to the removed server via a stale peer.
-            if (filled($host->wg_pubkey)) {
-                RemoveWireguardPeer::dispatch($host->wg_pubkey);
-            }
-
-            $host->delete();
         }
 
         return $this->redirectRoute('admin.hosts', navigate: true);

app/Provisioning/Jobs/PurgeHost.php

@@ -0,0 +1,52 @@
+<?php
+
+namespace App\Provisioning\Jobs;
+
+use App\Models\Host;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Illuminate\Support\Facades\Cache;
+
+/**
+ * Finalizes host removal off the web request: deletes each run under its runner
+ * lock (waiting for any in-flight step to finish rather than racing it), then
+ * removes the WireGuard peer and the host record. Runs on the provisioning queue.
+ */
+class PurgeHost implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public $tries = 1;
+
+    public $timeout = 2200;
+
+    public function __construct(public string $uuid)
+    {
+        $this->onConnection('provisioning');
+        $this->onQueue('provisioning');
+    }
+
+    public function handle(): void
+    {
+        $host = Host::query()->where('uuid', $this->uuid)->first();
+
+        if ($host === null) {
+            return;
+        }
+
+        foreach ($host->runs()->get() as $run) {
+            // Block until the runner lock is free (a long step may hold it for up
+            // to the step timeout), then delete — cascades events + resources.
+            Cache::lock('run:'.$run->uuid, 2200)->block(2100, fn () => $run->delete());
+        }
+
+        if (filled($host->wg_pubkey)) {
+            RemoveWireguardPeer::dispatch($host->wg_pubkey);
+        }
+
+        $host->delete();
+    }
+}

app/Provisioning/Steps/Host/ConfigureWireguard.php

@@ -26,52 +26,65 @@ class ConfigureWireguard extends HostStep
     public function execute(ProvisioningRun $run): StepResult
     {
         $host = $this->host($run);
-        $alreadyProvisioned = filled($host->wg_ip) && $this->hasResource($run, 'wg_peer');
-
         $this->keyLogin($this->shell, $host);
 
-        if (! $alreadyProvisioned) {
+        // Idempotent replay: tunnel already fully provisioned — just re-verify it.
-            $this->shell->run('export DEBIAN_FRONTEND=noninteractive; apt-get install -y wireguard');
+        if (filled($host->wg_ip) && $this->hasResource($run, 'wg_peer')) {
-            $this->shell->run('test -f /etc/wireguard/privatekey || (umask 077; wg genkey > /etc/wireguard/privatekey)');
+            return $this->verifyHandshake();
-
-            $publicKey = trim($this->shell->run('wg pubkey < /etc/wireguard/privatekey')->stdout);
-            if (blank($publicKey)) {
-                return StepResult::retry(20, 'could not read host WireGuard public key');
-            }
-
-            // Allocate + reserve the management IP atomically so concurrent
-            // onboarding runs can never receive the same address.
-            $wgIp = Cache::lock('wireguard:allocate', 30)->block(10, function () use ($host) {
-                $ip = $host->wg_ip ?: $this->hub->allocateIp();
-                if (blank($host->wg_ip)) {
-                    $host->update(['wg_ip' => $ip]);
-                }
-
-                return $ip;
-            });
-
-            $privateKey = trim($this->shell->run('cat /etc/wireguard/privatekey')->stdout);
-
-            $this->shell->putFile('/etc/wireguard/wg0.conf', $this->renderConfig($wgIp, $privateKey));
-            $this->shell->run('systemctl enable --now wg-quick@wg0 || wg-quick up wg0');
-
-            $this->hub->addPeer($publicKey, $wgIp);
-
-            // Persist external identity BEFORE verifying/advancing (crash-safe).
-            $host->update(['wg_pubkey' => $publicKey]);
-            $this->recordResource($run, $host, 'wg_peer', $publicKey);
         }
 
-        // Always verify the tunnel is up — including the idempotent replay path,
+        if (! $this->shell->run('export DEBIAN_FRONTEND=noninteractive; apt-get install -y wireguard')->ok()) {
-        // so we never advance onto Proxmox API calls over a dead tunnel.
+            return StepResult::retry(30, 'installing wireguard failed');
-        $hubIp = (string) config('provisioning.wireguard.hub_ip');
+        }
-        if (! $this->shell->run('ping -c1 -W2 '.escapeshellarg($hubIp))->ok()) {
+        $this->shell->run('test -f /etc/wireguard/privatekey || (umask 077; wg genkey > /etc/wireguard/privatekey)');
+
+        $publicKey = trim($this->shell->run('wg pubkey < /etc/wireguard/privatekey')->stdout);
+        if (blank($publicKey)) {
+            return StepResult::retry(20, 'could not read host WireGuard public key');
+        }
+
+        // Allocate + reserve the management IP atomically so concurrent onboarding
+        // runs can never receive the same address.
+        $wgIp = Cache::lock('wireguard:allocate', 30)->block(10, function () use ($host) {
+            $ip = $host->wg_ip ?: $this->hub->allocateIp();
+            if (blank($host->wg_ip)) {
+                $host->update(['wg_ip' => $ip]);
+            }
+
+            return $ip;
+        });
+
+        $privateKey = trim($this->shell->run('cat /etc/wireguard/privatekey')->stdout);
+        $this->shell->putFile('/etc/wireguard/wg0.conf', $this->renderConfig($wgIp, $privateKey));
+
+        if (! $this->shell->run('systemctl enable --now wg-quick@wg0 || wg-quick up wg0')->ok()) {
+            return StepResult::retry(20, 'bringing up wg0 failed');
+        }
+
+        $this->hub->addPeer($publicKey, $wgIp);
+
+        // Verify the tunnel BEFORE recording the peer as provisioned. If the
+        // handshake isn't up we retry and re-run the full setup next time (no
+        // wg_peer resource yet), instead of skipping a broken configuration.
+        if ($this->verifyHandshake()->type !== StepResult::ADVANCE) {
             return StepResult::retry(15, 'WireGuard handshake not up yet');
         }
 
+        $host->update(['wg_pubkey' => $publicKey]);
+        $this->recordResource($run, $host, 'wg_peer', $publicKey);
+
         return StepResult::advance();
     }
 
+    private function verifyHandshake(): StepResult
+    {
+        $hubIp = (string) config('provisioning.wireguard.hub_ip');
+
+        return $this->shell->run('ping -c1 -W2 '.escapeshellarg($hubIp))->ok()
+            ? StepResult::advance()
+            : StepResult::retry(15, 'WireGuard handshake not up yet');
+    }
+
     private function renderConfig(string $wgIp, string $privateKey): string
     {
         $subnet = (string) config('provisioning.wireguard.subnet', '10.66.0.0/24');

tests/Feature/Admin/HostManagementTest.php

@@ -7,6 +7,7 @@ use App\Models\Host;
 use App\Models\ProvisioningRun;
 use App\Models\User;
 use App\Provisioning\Jobs\AdvanceRunJob;
+use App\Provisioning\Jobs\PurgeHost;
 use App\Provisioning\Jobs\RemoveWireguardPeer;
 use Illuminate\Support\Facades\Crypt;
 use Illuminate\Support\Facades\Queue;
@@ -100,7 +101,7 @@ it('retries a failed run from the detail page', function () {
     Queue::assertPushed(AdvanceRunJob::class);
 });
 
-it('removes the host record and dispatches wireguard peer cleanup', function () {
+it('deactivates the host and queues an async purge', function () {
     Queue::fake();
     $host = Host::factory()->active()->create(['wg_pubkey' => 'HOSTPUB=']);
     ProvisioningRun::factory()->forHost($host)->create();
@@ -110,6 +111,17 @@ it('removes the host record and dispatches wireguard peer cleanup', function ()
         ->call('remove')
         ->assertRedirect(route('admin.hosts'));
 
+    expect($host->fresh()->status)->toBe('disabled');
+    Queue::assertPushed(PurgeHost::class, fn ($job) => $job->uuid === $host->uuid);
+});
+
+it('purges a host: deletes its runs and record and queues peer removal', function () {
+    Queue::fake([RemoveWireguardPeer::class]);
+    $host = Host::factory()->active()->create(['wg_pubkey' => 'HOSTPUB=']);
+    ProvisioningRun::factory()->forHost($host)->create();
+
+    (new PurgeHost($host->uuid))->handle();
+
     expect(Host::query()->find($host->id))->toBeNull();
     Queue::assertPushed(RemoveWireguardPeer::class, fn ($job) => $job->publicKey === 'HOSTPUB=');
 });