boban
/
CluPilotCloud
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(engine): record wg peer only after verified handshake; async host purge 

- ConfigureWireguard checks setup command results and only records the wg_peer
  resource after the handshake verifies, so a transient setup failure re-runs
  the full setup instead of getting stuck on the idempotent path.
- Host removal now deactivates immediately and queues PurgeHost, which deletes
  runs under the runner lock (waiting out a long step) — no 15s LockTimeout error.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
feat/portal-design
nexxo 2026-07-25 11:06:13 +02:00
parent a6a41719bb
commit 9e664654a2
4 changed files with 122 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
app/Livewire/Admin/ConfirmRemoveHost.php
View File

@ -3,13 +3,14 @@
namespace App\Livewire\Admin;
use App\Models\Host;
use App\Provisioning\Jobs\RemoveWireguardPeer;
use Illuminate\Support\Facades\Cache;
use App\Provisioning\Jobs\PurgeHost;
use LivewireUI\Modal\ModalComponent;
/**
* Confirmation for the destructive "remove host" action (R5). Deregisters the
* CluPilot record only — it does not touch the physical server.
* CluPilot record only — it does not touch the physical server. The host is
* deactivated immediately and purged asynchronously so removal never blocks on
* (or races) an in-flight provisioning step.
*/
class ConfirmRemoveHost extends ModalComponent
{
@ -28,21 +29,10 @@ class ConfirmRemoveHost extends ModalComponent
$host = Host::query()->where('uuid', $this->uuid)->first();
if ($host !== null) {
// Delete each run under its runner lock so an in-flight worker can't
// keep mutating the server or write an event against a deleted run.
foreach ($host->runs()->get() as $run) {
Cache::lock('run:'.$run->uuid, 30)->block(15, function () use ($run) {
$run->delete(); // cascades run_resources + step_events
});
}
// Drop the hub peer on the privileged provisioning worker (which owns
// wg0), so a freed wg_ip can't route to the removed server via a stale peer.
if (filled($host->wg_pubkey)) {
RemoveWireguardPeer::dispatch($host->wg_pubkey);
}
$host->delete();
// Deactivate now (out of placement, visibly removed), finalize on the
// provisioning worker which waits for the runner lock.
$host->update(['status' => 'disabled']);
PurgeHost::dispatch($host->uuid);
}
return $this->redirectRoute('admin.hosts', navigate: true);

52
app/Provisioning/Jobs/PurgeHost.php Normal file
View File

@ -0,0 +1,52 @@
<?php
namespace App\Provisioning\Jobs;
use App\Models\Host;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Cache;
/**
* Finalizes host removal off the web request: deletes each run under its runner
* lock (waiting for any in-flight step to finish rather than racing it), then
* removes the WireGuard peer and the host record. Runs on the provisioning queue.
*/
class PurgeHost implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
public $tries = 1;
public $timeout = 2200;
public function __construct(public string $uuid)
{
$this->onConnection('provisioning');
$this->onQueue('provisioning');
}
public function handle(): void
{
$host = Host::query()->where('uuid', $this->uuid)->first();
if ($host === null) {
return;
}
foreach ($host->runs()->get() as $run) {
// Block until the runner lock is free (a long step may hold it for up
// to the step timeout), then delete — cascades events + resources.
Cache::lock('run:'.$run->uuid, 2200)->block(2100, fn () => $run->delete());
}
if (filled($host->wg_pubkey)) {
RemoveWireguardPeer::dispatch($host->wg_pubkey);
}
$host->delete();
}
}

85
app/Provisioning/Steps/Host/ConfigureWireguard.php
View File

@ -26,52 +26,65 @@ class ConfigureWireguard extends HostStep
public function execute(ProvisioningRun $run): StepResult
{
$host = $this->host($run);
$alreadyProvisioned = filled($host->wg_ip) && $this->hasResource($run, 'wg_peer');
$this->keyLogin($this->shell, $host);
if (! $alreadyProvisioned) {
$this->shell->run('export DEBIAN_FRONTEND=noninteractive; apt-get install -y wireguard');
$this->shell->run('test -f /etc/wireguard/privatekey || (umask 077; wg genkey > /etc/wireguard/privatekey)');
$publicKey = trim($this->shell->run('wg pubkey < /etc/wireguard/privatekey')->stdout);
if (blank($publicKey)) {
return StepResult::retry(20, 'could not read host WireGuard public key');
}
// Allocate + reserve the management IP atomically so concurrent
// onboarding runs can never receive the same address.
$wgIp = Cache::lock('wireguard:allocate', 30)->block(10, function () use ($host) {
$ip = $host->wg_ip ?: $this->hub->allocateIp();
if (blank($host->wg_ip)) {
$host->update(['wg_ip' => $ip]);
}
return $ip;
});
$privateKey = trim($this->shell->run('cat /etc/wireguard/privatekey')->stdout);
$this->shell->putFile('/etc/wireguard/wg0.conf', $this->renderConfig($wgIp, $privateKey));
$this->shell->run('systemctl enable --now wg-quick@wg0 || wg-quick up wg0');
$this->hub->addPeer($publicKey, $wgIp);
// Persist external identity BEFORE verifying/advancing (crash-safe).
$host->update(['wg_pubkey' => $publicKey]);
$this->recordResource($run, $host, 'wg_peer', $publicKey);
// Idempotent replay: tunnel already fully provisioned — just re-verify it.
if (filled($host->wg_ip) && $this->hasResource($run, 'wg_peer')) {
return $this->verifyHandshake();
}
// Always verify the tunnel is up — including the idempotent replay path,
// so we never advance onto Proxmox API calls over a dead tunnel.
$hubIp = (string) config('provisioning.wireguard.hub_ip');
if (! $this->shell->run('ping -c1 -W2 '.escapeshellarg($hubIp))->ok()) {
if (! $this->shell->run('export DEBIAN_FRONTEND=noninteractive; apt-get install -y wireguard')->ok()) {
return StepResult::retry(30, 'installing wireguard failed');
}
$this->shell->run('test -f /etc/wireguard/privatekey || (umask 077; wg genkey > /etc/wireguard/privatekey)');
$publicKey = trim($this->shell->run('wg pubkey < /etc/wireguard/privatekey')->stdout);
if (blank($publicKey)) {
return StepResult::retry(20, 'could not read host WireGuard public key');
}
// Allocate + reserve the management IP atomically so concurrent onboarding
// runs can never receive the same address.
$wgIp = Cache::lock('wireguard:allocate', 30)->block(10, function () use ($host) {
$ip = $host->wg_ip ?: $this->hub->allocateIp();
if (blank($host->wg_ip)) {
$host->update(['wg_ip' => $ip]);
}
return $ip;
});
$privateKey = trim($this->shell->run('cat /etc/wireguard/privatekey')->stdout);
$this->shell->putFile('/etc/wireguard/wg0.conf', $this->renderConfig($wgIp, $privateKey));
if (! $this->shell->run('systemctl enable --now wg-quick@wg0 || wg-quick up wg0')->ok()) {
return StepResult::retry(20, 'bringing up wg0 failed');
}
$this->hub->addPeer($publicKey, $wgIp);
// Verify the tunnel BEFORE recording the peer as provisioned. If the
// handshake isn't up we retry and re-run the full setup next time (no
// wg_peer resource yet), instead of skipping a broken configuration.
if ($this->verifyHandshake()->type !== StepResult::ADVANCE) {
return StepResult::retry(15, 'WireGuard handshake not up yet');
}
$host->update(['wg_pubkey' => $publicKey]);
$this->recordResource($run, $host, 'wg_peer', $publicKey);
return StepResult::advance();
}
private function verifyHandshake(): StepResult
{
$hubIp = (string) config('provisioning.wireguard.hub_ip');
return $this->shell->run('ping -c1 -W2 '.escapeshellarg($hubIp))->ok()
? StepResult::advance()
: StepResult::retry(15, 'WireGuard handshake not up yet');
}
private function renderConfig(string $wgIp, string $privateKey): string
{
$subnet = (string) config('provisioning.wireguard.subnet', '10.66.0.0/24');

14
tests/Feature/Admin/HostManagementTest.php
View File

@ -7,6 +7,7 @@ use App\Models\Host;
use App\Models\ProvisioningRun;
use App\Models\User;
use App\Provisioning\Jobs\AdvanceRunJob;
use App\Provisioning\Jobs\PurgeHost;
use App\Provisioning\Jobs\RemoveWireguardPeer;
use Illuminate\Support\Facades\Crypt;
use Illuminate\Support\Facades\Queue;
@ -100,7 +101,7 @@ it('retries a failed run from the detail page', function () {
Queue::assertPushed(AdvanceRunJob::class);
});
it('removes the host record and dispatches wireguard peer cleanup', function () {
it('deactivates the host and queues an async purge', function () {
Queue::fake();
$host = Host::factory()->active()->create(['wg_pubkey' => 'HOSTPUB=']);
ProvisioningRun::factory()->forHost($host)->create();
@ -110,6 +111,17 @@ it('removes the host record and dispatches wireguard peer cleanup', function ()
->call('remove')
->assertRedirect(route('admin.hosts'));
expect($host->fresh()->status)->toBe('disabled');
Queue::assertPushed(PurgeHost::class, fn ($job) => $job->uuid === $host->uuid);
});
it('purges a host: deletes its runs and record and queues peer removal', function () {
Queue::fake([RemoveWireguardPeer::class]);
$host = Host::factory()->active()->create(['wg_pubkey' => 'HOSTPUB=']);
ProvisioningRun::factory()->forHost($host)->create();
(new PurgeHost($host->uuid))->handle();
expect(Host::query()->find($host->id))->toBeNull();
Queue::assertPushed(RemoveWireguardPeer::class, fn ($job) => $job->publicKey === 'HOSTPUB=');
});