From a6a41719bb511730120a2268caeb8dd8fdb781eb Mon Sep 17 00:00:00 2001 From: nexxo Date: Sat, 25 Jul 2026 11:01:02 +0200 Subject: [PATCH] fix(engine): long SSH command timeout; wg peer removal on privileged worker - PhpseclibRemoteShell sets a command timeout (default 2000s) so apt full-upgrade/install don't hit phpseclib's ~10s default and fail. - Host removal dispatches RemoveWireguardPeer to the provisioning queue, which runs in the worker that owns wg0 (the web container can't manage WireGuard). Co-Authored-By: Claude Opus 4.8 --- app/Livewire/Admin/ConfirmRemoveHost.php | 10 +++--- app/Provisioning/Jobs/RemoveWireguardPeer.php | 35 +++++++++++++++++++ app/Services/Ssh/PhpseclibRemoteShell.php | 8 +++++ config/provisioning.php | 3 ++ tests/Feature/Admin/HostManagementTest.php | 10 +++--- tests/Feature/Provisioning/ServicesTest.php | 9 +++++ 6 files changed, 65 insertions(+), 10 deletions(-) create mode 100644 app/Provisioning/Jobs/RemoveWireguardPeer.php diff --git a/app/Livewire/Admin/ConfirmRemoveHost.php b/app/Livewire/Admin/ConfirmRemoveHost.php index ef2c8c3..1e62996 100644 --- a/app/Livewire/Admin/ConfirmRemoveHost.php +++ b/app/Livewire/Admin/ConfirmRemoveHost.php @@ -3,7 +3,7 @@ namespace App\Livewire\Admin; use App\Models\Host; -use App\Services\Wireguard\WireguardHub; +use App\Provisioning\Jobs\RemoveWireguardPeer; use Illuminate\Support\Facades\Cache; use LivewireUI\Modal\ModalComponent; @@ -23,7 +23,7 @@ class ConfirmRemoveHost extends ModalComponent $this->name = Host::query()->where('uuid', $uuid)->value('name') ?? ''; } - public function remove(WireguardHub $hub) + public function remove() { $host = Host::query()->where('uuid', $this->uuid)->first(); @@ -36,10 +36,10 @@ class ConfirmRemoveHost extends ModalComponent }); } - // Drop the hub peer so its wg_ip can't later be reused while a stale - // peer still routes to the removed server. + // Drop the hub peer on the privileged provisioning worker (which owns + // wg0), so a freed wg_ip can't route to the removed server via a stale peer. if (filled($host->wg_pubkey)) { - $hub->removePeer($host->wg_pubkey); + RemoveWireguardPeer::dispatch($host->wg_pubkey); } $host->delete(); diff --git a/app/Provisioning/Jobs/RemoveWireguardPeer.php b/app/Provisioning/Jobs/RemoveWireguardPeer.php new file mode 100644 index 0000000..f2abf49 --- /dev/null +++ b/app/Provisioning/Jobs/RemoveWireguardPeer.php @@ -0,0 +1,35 @@ +onConnection('provisioning'); + $this->onQueue('provisioning'); + } + + public function handle(WireguardHub $hub): void + { + $hub->removePeer($this->publicKey); + } +} diff --git a/app/Services/Ssh/PhpseclibRemoteShell.php b/app/Services/Ssh/PhpseclibRemoteShell.php index 2065722..de418d0 100644 --- a/app/Services/Ssh/PhpseclibRemoteShell.php +++ b/app/Services/Ssh/PhpseclibRemoteShell.php @@ -18,6 +18,7 @@ class PhpseclibRemoteShell implements RemoteShell public function connectWithPassword(string $host, string $user, string $password): void { $ssh = new SSH2($host); + $ssh->setTimeout($this->commandTimeout()); if (! $ssh->login($user, $password)) { throw new RuntimeException("SSH password login failed for {$user}@{$host}"); @@ -30,6 +31,7 @@ class PhpseclibRemoteShell implements RemoteShell { $key = PublicKeyLoader::load($privateKey); $ssh = new SSH2($host); + $ssh->setTimeout($this->commandTimeout()); if (! $ssh->login($user, $key)) { throw new RuntimeException("SSH key login failed for {$user}@{$host}"); @@ -73,6 +75,12 @@ class PhpseclibRemoteShell implements RemoteShell return $key === false ? '' : 'SHA256:'.base64_encode(hash('sha256', (string) $key, true)); } + /** Long provisioning commands (apt full-upgrade …) far exceed the ~10s default. */ + private function commandTimeout(): int + { + return (int) config('provisioning.ssh.command_timeout', 2000); + } + private function ssh(): SSH2 { if ($this->ssh === null) { diff --git a/config/provisioning.php b/config/provisioning.php index 2fd11e5..42c3bf3 100644 --- a/config/provisioning.php +++ b/config/provisioning.php @@ -41,6 +41,9 @@ return [ 'ssh' => [ 'public_key' => env('CLUPILOT_SSH_PUBLIC_KEY', ''), 'private_key' => env('CLUPILOT_SSH_PRIVATE_KEY', ''), + // Per-command SSH timeout; below the job timeout (2100s) but far above + // the phpseclib ~10s default so apt full-upgrade etc. can finish. + 'command_timeout' => (int) env('CLUPILOT_SSH_COMMAND_TIMEOUT', 2000), ], // Proxmox automation role/user created on each host. diff --git a/tests/Feature/Admin/HostManagementTest.php b/tests/Feature/Admin/HostManagementTest.php index abb33c9..905e3c1 100644 --- a/tests/Feature/Admin/HostManagementTest.php +++ b/tests/Feature/Admin/HostManagementTest.php @@ -7,6 +7,7 @@ use App\Models\Host; use App\Models\ProvisioningRun; use App\Models\User; use App\Provisioning\Jobs\AdvanceRunJob; +use App\Provisioning\Jobs\RemoveWireguardPeer; use Illuminate\Support\Facades\Crypt; use Illuminate\Support\Facades\Queue; use Livewire\Livewire; @@ -99,10 +100,9 @@ it('retries a failed run from the detail page', function () { Queue::assertPushed(AdvanceRunJob::class); }); -it('removes the host record and its wireguard peer without wiping the server', function () { - $services = fakeServices(); +it('removes the host record and dispatches wireguard peer cleanup', function () { + Queue::fake(); $host = Host::factory()->active()->create(['wg_pubkey' => 'HOSTPUB=']); - $services['hub']->addPeer('HOSTPUB=', $host->wg_ip); ProvisioningRun::factory()->forHost($host)->create(); Livewire::actingAs(admin()) @@ -110,8 +110,8 @@ it('removes the host record and its wireguard peer without wiping the server', f ->call('remove') ->assertRedirect(route('admin.hosts')); - expect(Host::query()->find($host->id))->toBeNull() - ->and($services['hub']->peers())->not->toHaveKey('HOSTPUB='); + expect(Host::query()->find($host->id))->toBeNull(); + Queue::assertPushed(RemoveWireguardPeer::class, fn ($job) => $job->publicKey === 'HOSTPUB='); }); it('renders the live stepper for a running host', function () { diff --git a/tests/Feature/Provisioning/ServicesTest.php b/tests/Feature/Provisioning/ServicesTest.php index 60e12f7..7a5992a 100644 --- a/tests/Feature/Provisioning/ServicesTest.php +++ b/tests/Feature/Provisioning/ServicesTest.php @@ -21,6 +21,15 @@ it('scripts remote command output and records calls (FakeRemoteShell)', function ->and($shell->connectionsWith('password'))->toHaveCount(1); }); +it('removes a peer via the RemoveWireguardPeer job', function () { + $hub = new FakeWireguardHub; + $hub->addPeer('PK', '10.66.0.9'); + + (new App\Provisioning\Jobs\RemoveWireguardPeer('PK'))->handle($hub); + + expect($hub->peers())->toBe([]); +}); + it('allocates ips and tracks peers (FakeWireguardHub)', function () { $hub = new FakeWireguardHub;