diff --git a/app/Livewire/Admin/ConfirmRemoveHost.php b/app/Livewire/Admin/ConfirmRemoveHost.php index 1e62996..e8e0ba7 100644 --- a/app/Livewire/Admin/ConfirmRemoveHost.php +++ b/app/Livewire/Admin/ConfirmRemoveHost.php @@ -3,13 +3,14 @@ namespace App\Livewire\Admin; use App\Models\Host; -use App\Provisioning\Jobs\RemoveWireguardPeer; -use Illuminate\Support\Facades\Cache; +use App\Provisioning\Jobs\PurgeHost; use LivewireUI\Modal\ModalComponent; /** * Confirmation for the destructive "remove host" action (R5). Deregisters the - * CluPilot record only — it does not touch the physical server. + * CluPilot record only — it does not touch the physical server. The host is + * deactivated immediately and purged asynchronously so removal never blocks on + * (or races) an in-flight provisioning step. */ class ConfirmRemoveHost extends ModalComponent { @@ -28,21 +29,10 @@ class ConfirmRemoveHost extends ModalComponent $host = Host::query()->where('uuid', $this->uuid)->first(); if ($host !== null) { - // Delete each run under its runner lock so an in-flight worker can't - // keep mutating the server or write an event against a deleted run. - foreach ($host->runs()->get() as $run) { - Cache::lock('run:'.$run->uuid, 30)->block(15, function () use ($run) { - $run->delete(); // cascades run_resources + step_events - }); - } - - // Drop the hub peer on the privileged provisioning worker (which owns - // wg0), so a freed wg_ip can't route to the removed server via a stale peer. - if (filled($host->wg_pubkey)) { - RemoveWireguardPeer::dispatch($host->wg_pubkey); - } - - $host->delete(); + // Deactivate now (out of placement, visibly removed), finalize on the + // provisioning worker which waits for the runner lock. + $host->update(['status' => 'disabled']); + PurgeHost::dispatch($host->uuid); } return $this->redirectRoute('admin.hosts', navigate: true); diff --git a/app/Provisioning/Jobs/PurgeHost.php b/app/Provisioning/Jobs/PurgeHost.php new file mode 100644 index 0000000..af8ed52 --- /dev/null +++ b/app/Provisioning/Jobs/PurgeHost.php @@ -0,0 +1,52 @@ +onConnection('provisioning'); + $this->onQueue('provisioning'); + } + + public function handle(): void + { + $host = Host::query()->where('uuid', $this->uuid)->first(); + + if ($host === null) { + return; + } + + foreach ($host->runs()->get() as $run) { + // Block until the runner lock is free (a long step may hold it for up + // to the step timeout), then delete — cascades events + resources. + Cache::lock('run:'.$run->uuid, 2200)->block(2100, fn () => $run->delete()); + } + + if (filled($host->wg_pubkey)) { + RemoveWireguardPeer::dispatch($host->wg_pubkey); + } + + $host->delete(); + } +} diff --git a/app/Provisioning/Steps/Host/ConfigureWireguard.php b/app/Provisioning/Steps/Host/ConfigureWireguard.php index 18647d8..959aa83 100644 --- a/app/Provisioning/Steps/Host/ConfigureWireguard.php +++ b/app/Provisioning/Steps/Host/ConfigureWireguard.php @@ -26,52 +26,65 @@ class ConfigureWireguard extends HostStep public function execute(ProvisioningRun $run): StepResult { $host = $this->host($run); - $alreadyProvisioned = filled($host->wg_ip) && $this->hasResource($run, 'wg_peer'); - $this->keyLogin($this->shell, $host); - if (! $alreadyProvisioned) { - $this->shell->run('export DEBIAN_FRONTEND=noninteractive; apt-get install -y wireguard'); - $this->shell->run('test -f /etc/wireguard/privatekey || (umask 077; wg genkey > /etc/wireguard/privatekey)'); - - $publicKey = trim($this->shell->run('wg pubkey < /etc/wireguard/privatekey')->stdout); - if (blank($publicKey)) { - return StepResult::retry(20, 'could not read host WireGuard public key'); - } - - // Allocate + reserve the management IP atomically so concurrent - // onboarding runs can never receive the same address. - $wgIp = Cache::lock('wireguard:allocate', 30)->block(10, function () use ($host) { - $ip = $host->wg_ip ?: $this->hub->allocateIp(); - if (blank($host->wg_ip)) { - $host->update(['wg_ip' => $ip]); - } - - return $ip; - }); - - $privateKey = trim($this->shell->run('cat /etc/wireguard/privatekey')->stdout); - - $this->shell->putFile('/etc/wireguard/wg0.conf', $this->renderConfig($wgIp, $privateKey)); - $this->shell->run('systemctl enable --now wg-quick@wg0 || wg-quick up wg0'); - - $this->hub->addPeer($publicKey, $wgIp); - - // Persist external identity BEFORE verifying/advancing (crash-safe). - $host->update(['wg_pubkey' => $publicKey]); - $this->recordResource($run, $host, 'wg_peer', $publicKey); + // Idempotent replay: tunnel already fully provisioned — just re-verify it. + if (filled($host->wg_ip) && $this->hasResource($run, 'wg_peer')) { + return $this->verifyHandshake(); } - // Always verify the tunnel is up — including the idempotent replay path, - // so we never advance onto Proxmox API calls over a dead tunnel. - $hubIp = (string) config('provisioning.wireguard.hub_ip'); - if (! $this->shell->run('ping -c1 -W2 '.escapeshellarg($hubIp))->ok()) { + if (! $this->shell->run('export DEBIAN_FRONTEND=noninteractive; apt-get install -y wireguard')->ok()) { + return StepResult::retry(30, 'installing wireguard failed'); + } + $this->shell->run('test -f /etc/wireguard/privatekey || (umask 077; wg genkey > /etc/wireguard/privatekey)'); + + $publicKey = trim($this->shell->run('wg pubkey < /etc/wireguard/privatekey')->stdout); + if (blank($publicKey)) { + return StepResult::retry(20, 'could not read host WireGuard public key'); + } + + // Allocate + reserve the management IP atomically so concurrent onboarding + // runs can never receive the same address. + $wgIp = Cache::lock('wireguard:allocate', 30)->block(10, function () use ($host) { + $ip = $host->wg_ip ?: $this->hub->allocateIp(); + if (blank($host->wg_ip)) { + $host->update(['wg_ip' => $ip]); + } + + return $ip; + }); + + $privateKey = trim($this->shell->run('cat /etc/wireguard/privatekey')->stdout); + $this->shell->putFile('/etc/wireguard/wg0.conf', $this->renderConfig($wgIp, $privateKey)); + + if (! $this->shell->run('systemctl enable --now wg-quick@wg0 || wg-quick up wg0')->ok()) { + return StepResult::retry(20, 'bringing up wg0 failed'); + } + + $this->hub->addPeer($publicKey, $wgIp); + + // Verify the tunnel BEFORE recording the peer as provisioned. If the + // handshake isn't up we retry and re-run the full setup next time (no + // wg_peer resource yet), instead of skipping a broken configuration. + if ($this->verifyHandshake()->type !== StepResult::ADVANCE) { return StepResult::retry(15, 'WireGuard handshake not up yet'); } + $host->update(['wg_pubkey' => $publicKey]); + $this->recordResource($run, $host, 'wg_peer', $publicKey); + return StepResult::advance(); } + private function verifyHandshake(): StepResult + { + $hubIp = (string) config('provisioning.wireguard.hub_ip'); + + return $this->shell->run('ping -c1 -W2 '.escapeshellarg($hubIp))->ok() + ? StepResult::advance() + : StepResult::retry(15, 'WireGuard handshake not up yet'); + } + private function renderConfig(string $wgIp, string $privateKey): string { $subnet = (string) config('provisioning.wireguard.subnet', '10.66.0.0/24'); diff --git a/tests/Feature/Admin/HostManagementTest.php b/tests/Feature/Admin/HostManagementTest.php index 905e3c1..fb46acd 100644 --- a/tests/Feature/Admin/HostManagementTest.php +++ b/tests/Feature/Admin/HostManagementTest.php @@ -7,6 +7,7 @@ use App\Models\Host; use App\Models\ProvisioningRun; use App\Models\User; use App\Provisioning\Jobs\AdvanceRunJob; +use App\Provisioning\Jobs\PurgeHost; use App\Provisioning\Jobs\RemoveWireguardPeer; use Illuminate\Support\Facades\Crypt; use Illuminate\Support\Facades\Queue; @@ -100,7 +101,7 @@ it('retries a failed run from the detail page', function () { Queue::assertPushed(AdvanceRunJob::class); }); -it('removes the host record and dispatches wireguard peer cleanup', function () { +it('deactivates the host and queues an async purge', function () { Queue::fake(); $host = Host::factory()->active()->create(['wg_pubkey' => 'HOSTPUB=']); ProvisioningRun::factory()->forHost($host)->create(); @@ -110,6 +111,17 @@ it('removes the host record and dispatches wireguard peer cleanup', function () ->call('remove') ->assertRedirect(route('admin.hosts')); + expect($host->fresh()->status)->toBe('disabled'); + Queue::assertPushed(PurgeHost::class, fn ($job) => $job->uuid === $host->uuid); +}); + +it('purges a host: deletes its runs and record and queues peer removal', function () { + Queue::fake([RemoveWireguardPeer::class]); + $host = Host::factory()->active()->create(['wg_pubkey' => 'HOSTPUB=']); + ProvisioningRun::factory()->forHost($host)->create(); + + (new PurgeHost($host->uuid))->handle(); + expect(Host::query()->find($host->id))->toBeNull(); Queue::assertPushed(RemoveWireguardPeer::class, fn ($job) => $job->publicKey === 'HOSTPUB='); });