fix(engine): long SSH command timeout; wg peer removal on privileged worker

- PhpseclibRemoteShell sets a command timeout (default 2000s) so apt
  full-upgrade/install don't hit phpseclib's ~10s default and fail.
- Host removal dispatches RemoveWireguardPeer to the provisioning queue, which
  runs in the worker that owns wg0 (the web container can't manage WireGuard).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

app/Livewire/Admin/ConfirmRemoveHost.php

@@ -3,7 +3,7 @@
 namespace App\Livewire\Admin;
 
 use App\Models\Host;
-use App\Services\Wireguard\WireguardHub;
+use App\Provisioning\Jobs\RemoveWireguardPeer;
 use Illuminate\Support\Facades\Cache;
 use LivewireUI\Modal\ModalComponent;
 
@@ -23,7 +23,7 @@ class ConfirmRemoveHost extends ModalComponent
         $this->name = Host::query()->where('uuid', $uuid)->value('name') ?? '';
     }
 
-    public function remove(WireguardHub $hub)
+    public function remove()
     {
         $host = Host::query()->where('uuid', $this->uuid)->first();
 
@@ -36,10 +36,10 @@ class ConfirmRemoveHost extends ModalComponent
                 });
             }
 
-            // Drop the hub peer so its wg_ip can't later be reused while a stale
-            // peer still routes to the removed server.
+            // Drop the hub peer on the privileged provisioning worker (which owns
+            // wg0), so a freed wg_ip can't route to the removed server via a stale peer.
             if (filled($host->wg_pubkey)) {
-                $hub->removePeer($host->wg_pubkey);
+                RemoveWireguardPeer::dispatch($host->wg_pubkey);
             }
 
             $host->delete();

app/Provisioning/Jobs/RemoveWireguardPeer.php

@@ -0,0 +1,35 @@
+<?php
+
+namespace App\Provisioning\Jobs;
+
+use App\Services\Wireguard\WireguardHub;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+
+/**
+ * Removes a host's WireGuard peer. Runs on the provisioning queue so it executes
+ * in the privileged worker that owns the wg0 interface (the web container can't
+ * manage WireGuard).
+ */
+class RemoveWireguardPeer implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public $tries = 3;
+
+    public $timeout = 120;
+
+    public function __construct(public string $publicKey)
+    {
+        $this->onConnection('provisioning');
+        $this->onQueue('provisioning');
+    }
+
+    public function handle(WireguardHub $hub): void
+    {
+        $hub->removePeer($this->publicKey);
+    }
+}

app/Services/Ssh/PhpseclibRemoteShell.php

@@ -18,6 +18,7 @@ class PhpseclibRemoteShell implements RemoteShell
     public function connectWithPassword(string $host, string $user, string $password): void
     {
         $ssh = new SSH2($host);
+        $ssh->setTimeout($this->commandTimeout());
 
         if (! $ssh->login($user, $password)) {
             throw new RuntimeException("SSH password login failed for {$user}@{$host}");
@@ -30,6 +31,7 @@ class PhpseclibRemoteShell implements RemoteShell
     {
         $key = PublicKeyLoader::load($privateKey);
         $ssh = new SSH2($host);
+        $ssh->setTimeout($this->commandTimeout());
 
         if (! $ssh->login($user, $key)) {
             throw new RuntimeException("SSH key login failed for {$user}@{$host}");
@@ -73,6 +75,12 @@ class PhpseclibRemoteShell implements RemoteShell
         return $key === false ? '' : 'SHA256:'.base64_encode(hash('sha256', (string) $key, true));
     }
 
+    /** Long provisioning commands (apt full-upgrade …) far exceed the ~10s default. */
+    private function commandTimeout(): int
+    {
+        return (int) config('provisioning.ssh.command_timeout', 2000);
+    }
+
     private function ssh(): SSH2
     {
         if ($this->ssh === null) {

config/provisioning.php

@@ -41,6 +41,9 @@ return [
     'ssh' => [
         'public_key' => env('CLUPILOT_SSH_PUBLIC_KEY', ''),
         'private_key' => env('CLUPILOT_SSH_PRIVATE_KEY', ''),
+        // Per-command SSH timeout; below the job timeout (2100s) but far above
+        // the phpseclib ~10s default so apt full-upgrade etc. can finish.
+        'command_timeout' => (int) env('CLUPILOT_SSH_COMMAND_TIMEOUT', 2000),
     ],
 
     // Proxmox automation role/user created on each host.

tests/Feature/Admin/HostManagementTest.php

@@ -7,6 +7,7 @@ use App\Models\Host;
 use App\Models\ProvisioningRun;
 use App\Models\User;
 use App\Provisioning\Jobs\AdvanceRunJob;
+use App\Provisioning\Jobs\RemoveWireguardPeer;
 use Illuminate\Support\Facades\Crypt;
 use Illuminate\Support\Facades\Queue;
 use Livewire\Livewire;
@@ -99,10 +100,9 @@ it('retries a failed run from the detail page', function () {
     Queue::assertPushed(AdvanceRunJob::class);
 });
 
-it('removes the host record and its wireguard peer without wiping the server', function () {
-    $services = fakeServices();
+it('removes the host record and dispatches wireguard peer cleanup', function () {
+    Queue::fake();
     $host = Host::factory()->active()->create(['wg_pubkey' => 'HOSTPUB=']);
-    $services['hub']->addPeer('HOSTPUB=', $host->wg_ip);
     ProvisioningRun::factory()->forHost($host)->create();
 
     Livewire::actingAs(admin())
@@ -110,8 +110,8 @@ it('removes the host record and its wireguard peer without wiping the server', f
         ->call('remove')
         ->assertRedirect(route('admin.hosts'));
 
-    expect(Host::query()->find($host->id))->toBeNull()
-        ->and($services['hub']->peers())->not->toHaveKey('HOSTPUB=');
+    expect(Host::query()->find($host->id))->toBeNull();
+    Queue::assertPushed(RemoveWireguardPeer::class, fn ($job) => $job->publicKey === 'HOSTPUB=');
 });
 
 it('renders the live stepper for a running host', function () {

tests/Feature/Provisioning/ServicesTest.php

@@ -21,6 +21,15 @@ it('scripts remote command output and records calls (FakeRemoteShell)', function
         ->and($shell->connectionsWith('password'))->toHaveCount(1);
 });
 
+it('removes a peer via the RemoveWireguardPeer job', function () {
+    $hub = new FakeWireguardHub;
+    $hub->addPeer('PK', '10.66.0.9');
+
+    (new App\Provisioning\Jobs\RemoveWireguardPeer('PK'))->handle($hub);
+
+    expect($hub->peers())->toBe([]);
+});
+
 it('allocates ips and tracks peers (FakeWireguardHub)', function () {
     $hub = new FakeWireguardHub;