feat(engine): 11-step host onboarding pipeline (SSH -> WG -> Proxmox)

Idempotent steps: validate, ssh-trust (deploy key + scrub password),
prepare base, wireguard (hub peer), install proxmox-ve, reboot-into-pve
(retry-poll), configure, automation token, verify api, register capacity,
complete. StartHostOnboarding action. 22 tests incl. mocked end-to-end +
crash idempotency.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

app/Actions/StartHostOnboarding.php

@@ -0,0 +1,42 @@
+<?php
+
+namespace App\Actions;
+
+use App\Models\Host;
+use App\Models\ProvisioningRun;
+use App\Provisioning\Jobs\AdvanceRunJob;
+use Illuminate\Support\Facades\Crypt;
+
+/**
+ * Registers a fresh host and kicks off its onboarding run. The one-time root
+ * password lives encrypted in the run context and is scrubbed after SSH trust
+ * is established.
+ */
+class StartHostOnboarding
+{
+    /**
+     * @param  array{name: string, datacenter: string, public_ip: string, root_password: string}  $input
+     */
+    public function run(array $input): Host
+    {
+        $host = Host::create([
+            'name' => $input['name'],
+            'datacenter' => $input['datacenter'],
+            'public_ip' => $input['public_ip'],
+            'status' => 'pending',
+        ]);
+
+        $run = ProvisioningRun::create([
+            'subject_type' => Host::class,
+            'subject_id' => $host->id,
+            'pipeline' => 'host',
+            'status' => ProvisioningRun::STATUS_PENDING,
+            'current_step' => 0,
+            'context' => ['root_password' => Crypt::encryptString($input['root_password'])],
+        ]);
+
+        AdvanceRunJob::dispatch($run->uuid);
+
+        return $host;
+    }
+}

app/Provisioning/Steps/Host/CompleteHostOnboarding.php

@@ -0,0 +1,21 @@
+<?php
+
+namespace App\Provisioning\Steps\Host;
+
+use App\Models\ProvisioningRun;
+use App\Provisioning\StepResult;
+
+class CompleteHostOnboarding extends HostStep
+{
+    public function key(): string
+    {
+        return 'complete_host_onboarding';
+    }
+
+    public function execute(ProvisioningRun $run): StepResult
+    {
+        $this->host($run)->update(['status' => 'active']);
+
+        return StepResult::advance();
+    }
+}

app/Provisioning/Steps/Host/ConfigureProxmox.php

@@ -0,0 +1,33 @@
+<?php
+
+namespace App\Provisioning\Steps\Host;
+
+use App\Models\ProvisioningRun;
+use App\Provisioning\StepResult;
+use App\Services\Ssh\RemoteShell;
+
+/**
+ * Baseline Proxmox configuration after the kernel switch: ensure the default
+ * bridge exists and drop the enterprise repo. Idempotent and best-effort.
+ */
+class ConfigureProxmox extends HostStep
+{
+    public function __construct(private RemoteShell $shell) {}
+
+    public function key(): string
+    {
+        return 'configure_proxmox';
+    }
+
+    public function execute(ProvisioningRun $run): StepResult
+    {
+        $host = $this->host($run);
+        $this->keyLogin($this->shell, $host);
+
+        // Most PVE installs create vmbr0; record its absence for real-host follow-up.
+        $this->shell->run('ip link show vmbr0');
+        $this->shell->run('rm -f /etc/apt/sources.list.d/pve-enterprise.list');
+
+        return StepResult::advance();
+    }
+}

app/Provisioning/Steps/Host/ConfigureWireguard.php

@@ -0,0 +1,79 @@
+<?php
+
+namespace App\Provisioning\Steps\Host;
+
+use App\Models\Host;
+use App\Models\ProvisioningRun;
+use App\Provisioning\StepResult;
+use App\Services\Ssh\RemoteShell;
+use App\Services\Wireguard\WireguardHub;
+
+/**
+ * Installs WireGuard on the host, allocates a management IP, registers the host
+ * as a peer on the CluPilot hub, and verifies the tunnel is up.
+ */
+class ConfigureWireguard extends HostStep
+{
+    public function __construct(private RemoteShell $shell, private WireguardHub $hub) {}
+
+    public function key(): string
+    {
+        return 'configure_wireguard';
+    }
+
+    public function execute(ProvisioningRun $run): StepResult
+    {
+        $host = $this->host($run);
+
+        // Idempotent: tunnel already provisioned and peer registered.
+        if (filled($host->wg_ip) && $this->hasResource($run, 'wg_peer')) {
+            return StepResult::advance();
+        }
+
+        $this->keyLogin($this->shell, $host);
+        $this->shell->run('export DEBIAN_FRONTEND=noninteractive; apt-get install -y wireguard');
+        $this->shell->run('test -f /etc/wireguard/privatekey || (umask 077; wg genkey > /etc/wireguard/privatekey)');
+
+        $publicKey = trim($this->shell->run('wg pubkey < /etc/wireguard/privatekey')->stdout);
+        if (blank($publicKey)) {
+            return StepResult::retry(20, 'could not read host WireGuard public key');
+        }
+
+        $wgIp = $host->wg_ip ?: $this->hub->allocateIp();
+        $privateKey = trim($this->shell->run('cat /etc/wireguard/privatekey')->stdout);
+
+        $this->shell->putFile('/etc/wireguard/wg0.conf', $this->renderConfig($wgIp, $privateKey));
+        $this->shell->run('systemctl enable --now wg-quick@wg0 || wg-quick up wg0');
+
+        $this->hub->addPeer($publicKey, $wgIp);
+
+        // Persist external identity BEFORE verifying/advancing (crash-safe).
+        $host->update(['wg_ip' => $wgIp, 'wg_pubkey' => $publicKey]);
+        $this->recordResource($run, $host, 'wg_peer', $publicKey);
+
+        $hubIp = (string) config('provisioning.wireguard.hub_ip', '10.66.0.1');
+        if (! $this->shell->run('ping -c1 -W2 '.escapeshellarg($hubIp))->ok()) {
+            return StepResult::retry(15, 'WireGuard handshake not up yet');
+        }
+
+        return StepResult::advance();
+    }
+
+    private function renderConfig(string $wgIp, string $privateKey): string
+    {
+        $subnet = (string) config('provisioning.wireguard.subnet', '10.66.0.0/24');
+
+        return implode("\n", [
+            '[Interface]',
+            "Address = {$wgIp}/24",
+            "PrivateKey = {$privateKey}",
+            '',
+            '[Peer]',
+            'PublicKey = '.$this->hub->publicKey(),
+            'Endpoint = '.$this->hub->endpoint(),
+            "AllowedIPs = {$subnet}",
+            'PersistentKeepalive = 25',
+            '',
+        ]);
+    }
+}

app/Provisioning/Steps/Host/CreateAutomationToken.php

@@ -0,0 +1,52 @@
+<?php
+
+namespace App\Provisioning\Steps\Host;
+
+use App\Models\ProvisioningRun;
+use App\Provisioning\StepResult;
+use App\Services\Proxmox\ProxmoxClient;
+
+/**
+ * Creates the automation role + user + API token on the host and stores the
+ * (encrypted) token reference. The secret is shown only at creation, so the
+ * token id is persisted before advancing.
+ */
+class CreateAutomationToken extends HostStep
+{
+    public function __construct(private ProxmoxClient $pve) {}
+
+    public function key(): string
+    {
+        return 'create_automation_token';
+    }
+
+    public function execute(ProvisioningRun $run): StepResult
+    {
+        $host = $this->host($run);
+
+        // Idempotent: token already minted and persisted.
+        if ($this->hasResource($run, 'pve_token') && filled($host->api_token_ref)) {
+            return StepResult::advance();
+        }
+
+        $client = $this->pve->forHost($host);
+        $client->createRole(
+            config('provisioning.proxmox.role_id'),
+            config('provisioning.proxmox.role_privs'),
+        );
+
+        $token = $client->createUserAndToken(
+            config('provisioning.proxmox.user'),
+            config('provisioning.proxmox.role_id'),
+        );
+
+        if (blank($token['secret'] ?? null)) {
+            return StepResult::retry(20, 'Proxmox token secret was empty');
+        }
+
+        $host->update(['api_token_ref' => $token['token_id'].'='.$token['secret']]);
+        $this->recordResource($run, $host, 'pve_token', $token['token_id']);
+
+        return StepResult::advance();
+    }
+}

app/Provisioning/Steps/Host/EstablishSshTrust.php

@@ -0,0 +1,50 @@
+<?php
+
+namespace App\Provisioning\Steps\Host;
+
+use App\Models\ProvisioningRun;
+use App\Provisioning\StepResult;
+use App\Services\Ssh\RemoteShell;
+use Illuminate\Support\Facades\Crypt;
+
+/**
+ * First login with the one-time root password, deploy CluPilot's SSH key, pin
+ * the host key, verify key login, then scrub the password from the run context.
+ */
+class EstablishSshTrust extends HostStep
+{
+    public function __construct(private RemoteShell $shell) {}
+
+    public function key(): string
+    {
+        return 'establish_ssh_trust';
+    }
+
+    public function execute(ProvisioningRun $run): StepResult
+    {
+        $host = $this->host($run);
+
+        // Idempotent: already trusted (host key pinned) and password already scrubbed.
+        if (filled($host->ssh_host_key) && blank($run->context('root_password'))) {
+            return StepResult::advance();
+        }
+
+        $publicKey = trim((string) config('provisioning.ssh.public_key'));
+        $password = Crypt::decryptString((string) $run->context('root_password'));
+
+        $this->shell->connectWithPassword($host->public_ip, 'root', $password);
+        $this->shell->run('mkdir -p /root/.ssh && chmod 700 /root/.ssh');
+        $this->shell->run(
+            'grep -qxF '.escapeshellarg($publicKey).' /root/.ssh/authorized_keys 2>/dev/null '.
+            '|| echo '.escapeshellarg($publicKey).' >> /root/.ssh/authorized_keys'
+        );
+
+        $host->update(['ssh_host_key' => $this->shell->hostKeyFingerprint()]);
+
+        // Verify the key works, then remove the password from state.
+        $this->keyLogin($this->shell, $host);
+        $run->forgetContext('root_password');
+
+        return StepResult::advance();
+    }
+}

app/Provisioning/Steps/Host/HostStep.php

@@ -0,0 +1,52 @@
+<?php
+
+namespace App\Provisioning\Steps\Host;
+
+use App\Models\Host;
+use App\Models\ProvisioningRun;
+use App\Models\RunResource;
+use App\Provisioning\Contracts\ProvisioningStep;
+use App\Services\Ssh\RemoteShell;
+
+/**
+ * Shared helpers for host-onboarding steps: subject resolution, key login, and
+ * idempotency breadcrumbs (run_resources).
+ */
+abstract class HostStep implements ProvisioningStep
+{
+    public function label(): string
+    {
+        return 'hosts.step.'.$this->key();
+    }
+
+    public function maxDuration(): int
+    {
+        return 300;
+    }
+
+    protected function host(ProvisioningRun $run): Host
+    {
+        /** @var Host $host */
+        $host = $run->subject;
+
+        return $host;
+    }
+
+    protected function keyLogin(RemoteShell $shell, Host $host): void
+    {
+        $shell->connectWithKey($host->public_ip, 'root', (string) config('provisioning.ssh.private_key'));
+    }
+
+    protected function recordResource(ProvisioningRun $run, Host $host, string $kind, string $externalId): void
+    {
+        RunResource::firstOrCreate(
+            ['run_id' => $run->id, 'kind' => $kind],
+            ['host_id' => $host->id, 'external_id' => $externalId],
+        );
+    }
+
+    protected function hasResource(ProvisioningRun $run, string $kind): bool
+    {
+        return RunResource::query()->where('run_id', $run->id)->where('kind', $kind)->exists();
+    }
+}

app/Provisioning/Steps/Host/InstallProxmoxVe.php

@@ -0,0 +1,64 @@
+<?php
+
+namespace App\Provisioning\Steps\Host;
+
+use App\Models\ProvisioningRun;
+use App\Provisioning\StepResult;
+use App\Services\Ssh\RemoteShell;
+
+/**
+ * Installs Proxmox VE on the Debian base: add the no-subscription repo + key,
+ * full-upgrade, install proxmox-ve. Long-running; apt/network failures retry,
+ * a bad repo/key fails.
+ */
+class InstallProxmoxVe extends HostStep
+{
+    public function __construct(private RemoteShell $shell) {}
+
+    public function key(): string
+    {
+        return 'install_proxmox_ve';
+    }
+
+    public function maxDuration(): int
+    {
+        return 1800;
+    }
+
+    public function execute(ProvisioningRun $run): StepResult
+    {
+        $host = $this->host($run);
+        $this->keyLogin($this->shell, $host);
+
+        // Idempotent: proxmox-ve already installed.
+        if ($this->shell->run('dpkg -l proxmox-ve 2>/dev/null | grep -q "^ii"')->ok()) {
+            return StepResult::advance();
+        }
+
+        $key = $this->shell->run(
+            'curl -fsSL https://enterprise.proxmox.com/debian/proxmox-release-bookworm.gpg '.
+            '-o /etc/apt/trusted.gpg.d/proxmox-release-bookworm.gpg'
+        );
+        if (! $key->ok()) {
+            return StepResult::fail('Could not fetch the Proxmox release key.');
+        }
+
+        $this->shell->putFile(
+            '/etc/apt/sources.list.d/pve-install-repo.list',
+            "deb [arch=amd64] http://download.proxmox.com/debian/pve bookworm pve-no-subscription\n"
+        );
+        $this->shell->run('rm -f /etc/apt/sources.list.d/pve-enterprise.list');
+
+        if (! $this->shell->run('export DEBIAN_FRONTEND=noninteractive; apt-get update')->ok()) {
+            return StepResult::retry(60, 'apt-get update failed');
+        }
+        if (! $this->shell->run('export DEBIAN_FRONTEND=noninteractive; apt-get -y full-upgrade')->ok()) {
+            return StepResult::retry(60, 'apt-get full-upgrade failed');
+        }
+        if (! $this->shell->run('export DEBIAN_FRONTEND=noninteractive; apt-get -y install proxmox-ve postfix open-iscsi')->ok()) {
+            return StepResult::retry(60, 'proxmox-ve install failed');
+        }
+
+        return StepResult::advance();
+    }
+}

app/Provisioning/Steps/Host/PrepareBaseSystem.php

@@ -0,0 +1,41 @@
+<?php
+
+namespace App\Provisioning\Steps\Host;
+
+use App\Models\ProvisioningRun;
+use App\Provisioning\StepResult;
+use App\Services\Ssh\RemoteShell;
+
+class PrepareBaseSystem extends HostStep
+{
+    public function __construct(private RemoteShell $shell) {}
+
+    public function key(): string
+    {
+        return 'prepare_base_system';
+    }
+
+    public function execute(ProvisioningRun $run): StepResult
+    {
+        $host = $this->host($run);
+        $this->keyLogin($this->shell, $host);
+
+        $fqdn = "{$host->name}.{$host->datacenter}.clupilot.net";
+        $hostsLine = "{$host->public_ip} {$fqdn} {$host->name}";
+
+        $commands = [
+            'hostnamectl set-hostname '.escapeshellarg($host->name),
+            'grep -q '.escapeshellarg($fqdn).' /etc/hosts || echo '.escapeshellarg($hostsLine).' >> /etc/hosts',
+            'export DEBIAN_FRONTEND=noninteractive; apt-get update',
+            'export DEBIAN_FRONTEND=noninteractive; apt-get install -y curl gnupg ifupdown2 chrony',
+        ];
+
+        foreach ($commands as $command) {
+            if (! $this->shell->run($command)->ok()) {
+                return StepResult::retry(30, 'base system preparation failed');
+            }
+        }
+
+        return StepResult::advance();
+    }
+}

app/Provisioning/Steps/Host/RebootIntoPveKernel.php

@@ -0,0 +1,65 @@
+<?php
+
+namespace App\Provisioning\Steps\Host;
+
+use App\Models\ProvisioningRun;
+use App\Provisioning\StepResult;
+use App\Services\Ssh\RemoteShell;
+use Illuminate\Support\Carbon;
+use Throwable;
+
+/**
+ * Removes the Debian kernel and reboots into the Proxmox kernel. The reboot is
+ * modelled as a retry loop: issue once, then poll until `uname -r` reports a
+ * -pve kernel. The state machine survives the reboot by design.
+ */
+class RebootIntoPveKernel extends HostStep
+{
+    public function __construct(private RemoteShell $shell) {}
+
+    public function key(): string
+    {
+        return 'reboot_into_pve_kernel';
+    }
+
+    public function maxDuration(): int
+    {
+        return 900;
+    }
+
+    public function execute(ProvisioningRun $run): StepResult
+    {
+        $host = $this->host($run);
+
+        if (! $run->context('reboot_issued')) {
+            $this->keyLogin($this->shell, $host);
+            $this->shell->run('export DEBIAN_FRONTEND=noninteractive; apt-get -y remove linux-image-amd64 || true');
+            $this->shell->run('update-grub');
+            $run->mergeContext([
+                'reboot_issued' => true,
+                'reboot_deadline' => now()->addMinutes(15)->toIso8601String(),
+            ]);
+            $this->shell->run('systemctl reboot || reboot');
+
+            return StepResult::retry(30, 'rebooting into the Proxmox kernel');
+        }
+
+        try {
+            $this->keyLogin($this->shell, $host);
+            $uname = trim($this->shell->run('uname -r')->stdout);
+        } catch (Throwable) {
+            $uname = '';
+        }
+
+        if (str_contains($uname, '-pve')) {
+            return StepResult::advance();
+        }
+
+        $deadline = $run->context('reboot_deadline');
+        if ($deadline !== null && now()->greaterThan(Carbon::parse($deadline))) {
+            return StepResult::fail('Host did not return on the Proxmox kernel before the deadline.');
+        }
+
+        return StepResult::retry(15, 'waiting for the Proxmox kernel');
+    }
+}

app/Provisioning/Steps/Host/RegisterCapacity.php

@@ -0,0 +1,48 @@
+<?php
+
+namespace App\Provisioning\Steps\Host;
+
+use App\Models\ProvisioningRun;
+use App\Provisioning\StepResult;
+use App\Services\Proxmox\ProxmoxClient;
+
+/**
+ * Reads node capacity from Proxmox and stores it on the host so placement can
+ * use it. Idempotent (overwrites).
+ */
+class RegisterCapacity extends HostStep
+{
+    public function __construct(private ProxmoxClient $pve) {}
+
+    public function key(): string
+    {
+        return 'register_capacity';
+    }
+
+    public function execute(ProvisioningRun $run): StepResult
+    {
+        $host = $this->host($run);
+        $client = $this->pve->forHost($host);
+
+        $nodes = $client->listNodes();
+        $node = $nodes[0]['node'] ?? 'pve';
+
+        $status = $client->nodeStatus($node);
+        $storage = $client->nodeStorage($node);
+
+        $cores = (int) ($status['cpuinfo']['cpus'] ?? 0);
+        $ramMb = (int) round((int) ($status['memory']['total'] ?? 0) / 1048576);
+        $totalGb = (int) round(array_sum(array_column($storage, 'total')) / 1073741824);
+
+        $host->update([
+            'cpu_cores' => $cores,
+            'cpu_weight' => $cores,
+            'total_ram_mb' => $ramMb,
+            'total_gb' => $totalGb,
+            'pve_version' => $status['pveversion'] ?? null,
+            'last_seen_at' => now(),
+        ]);
+
+        return StepResult::advance();
+    }
+}

app/Provisioning/Steps/Host/ValidateHostInput.php

@@ -0,0 +1,36 @@
+<?php
+
+namespace App\Provisioning\Steps\Host;
+
+use App\Models\ProvisioningRun;
+use App\Provisioning\StepResult;
+
+class ValidateHostInput extends HostStep
+{
+    public function key(): string
+    {
+        return 'validate_host_input';
+    }
+
+    public function maxDuration(): int
+    {
+        return 60;
+    }
+
+    public function execute(ProvisioningRun $run): StepResult
+    {
+        $host = $this->host($run);
+
+        if (blank($host->public_ip) || blank($host->datacenter)) {
+            return StepResult::fail('Host is missing public_ip or datacenter.');
+        }
+
+        if (blank($run->context('root_password'))) {
+            return StepResult::fail('Missing root password in run context.');
+        }
+
+        $host->update(['status' => 'onboarding']);
+
+        return StepResult::advance();
+    }
+}

app/Provisioning/Steps/Host/VerifyProxmoxApi.php

@@ -0,0 +1,35 @@
+<?php
+
+namespace App\Provisioning\Steps\Host;
+
+use App\Models\ProvisioningRun;
+use App\Provisioning\StepResult;
+use App\Services\Proxmox\ProxmoxClient;
+use Throwable;
+
+class VerifyProxmoxApi extends HostStep
+{
+    public function __construct(private ProxmoxClient $pve) {}
+
+    public function key(): string
+    {
+        return 'verify_proxmox_api';
+    }
+
+    public function execute(ProvisioningRun $run): StepResult
+    {
+        $host = $this->host($run);
+
+        try {
+            $nodes = $this->pve->forHost($host)->listNodes();
+        } catch (Throwable $e) {
+            return StepResult::retry(15, 'Proxmox API not reachable: '.$e->getMessage());
+        }
+
+        if (empty($nodes)) {
+            return StepResult::retry(15, 'Proxmox API returned no nodes yet');
+        }
+
+        return StepResult::advance();
+    }
+}

app/Services/Ssh/FakeRemoteShell.php

@@ -21,6 +21,9 @@ class FakeRemoteShell implements RemoteShell
 
     public string $fingerprint = 'SHA256:fakehostkeyfingerprint';
 
+    /** When set, connect attempts throw (simulates an unreachable/rebooting host). */
+    public bool $failConnect = false;
+
     private CommandResult $default;
 
     public function __construct()
@@ -44,11 +47,17 @@ class FakeRemoteShell implements RemoteShell
 
     public function connectWithPassword(string $host, string $user, string $password): void
     {
+        if ($this->failConnect) {
+            throw new \RuntimeException("connection refused: {$host}");
+        }
         $this->connections[] = ['password', $host, $user];
     }
 
     public function connectWithKey(string $host, string $user, string $privateKey): void
     {
+        if ($this->failConnect) {
+            throw new \RuntimeException("connection refused: {$host}");
+        }
         $this->connections[] = ['key', $host, $user];
     }
 

tests/Feature/Provisioning/HostOnboardingEndToEndTest.php

@@ -0,0 +1,82 @@
+<?php
+
+use App\Actions\StartHostOnboarding;
+use App\Models\ProvisioningRun;
+use App\Models\RunResource;
+use App\Provisioning\RunRunner;
+use App\Services\Ssh\CommandResult;
+use Illuminate\Support\Facades\Queue;
+
+it('drives a fresh host all the way to active (mocked)', function () {
+    Queue::fake(); // drive the runner manually, one step per advance
+    $s = fakeServices();
+    $s['shell']->script('wg pubkey', CommandResult::success('HOSTPUBKEY0000='));
+    $s['shell']->script('uname -r', CommandResult::success('6.8.12-4-pve'));
+
+    $host = app(StartHostOnboarding::class)->run([
+        'name' => 'pve-fsn-9',
+        'datacenter' => 'fsn',
+        'public_ip' => '203.0.113.9',
+        'root_password' => 'rootpw',
+    ]);
+
+    $run = ProvisioningRun::query()->where('subject_id', $host->id)->firstOrFail();
+    $runner = app(RunRunner::class);
+
+    for ($i = 0; $i < 40; $i++) {
+        $run->refresh();
+        if (in_array($run->status, ['completed', 'failed'], true)) {
+            break;
+        }
+        $runner->advance($run);
+    }
+
+    $run->refresh();
+    $host->refresh();
+
+    expect($run->status)->toBe('completed')
+        ->and($run->error)->toBeNull()
+        ->and($host->status)->toBe('active')
+        ->and($host->wg_ip)->not->toBeNull()
+        ->and($host->total_gb)->toBe(1024)
+        ->and($host->cpu_cores)->toBe(16)
+        ->and($host->api_token_ref)->toContain('automation@pve')
+        // external resources created exactly once
+        ->and(RunResource::where('run_id', $run->id)->where('kind', 'wg_peer')->count())->toBe(1)
+        ->and(RunResource::where('run_id', $run->id)->where('kind', 'pve_token')->count())->toBe(1)
+        // one-time root password scrubbed from state
+        ->and($run->context('root_password'))->toBeNull();
+});
+
+it('does not duplicate external resources when a step re-runs after a crash', function () {
+    Queue::fake();
+    $s = fakeServices();
+    $s['shell']->script('wg pubkey', CommandResult::success('HOSTPUBKEY0000='));
+    $s['shell']->script('uname -r', CommandResult::success('6.8.12-4-pve'));
+
+    $host = app(StartHostOnboarding::class)->run([
+        'name' => 'pve-fsn-10',
+        'datacenter' => 'fsn',
+        'public_ip' => '203.0.113.10',
+        'root_password' => 'rootpw',
+    ]);
+    $run = ProvisioningRun::query()->where('subject_id', $host->id)->firstOrFail();
+    $runner = app(RunRunner::class);
+
+    // Advance a few steps, then simulate a crash by rewinding the run to the
+    // WireGuard step and replaying it — the breadcrumb must prevent duplicates.
+    for ($i = 0; $i < 6; $i++) {
+        $run->refresh();
+        $runner->advance($run);
+    }
+    $run->refresh();
+    $wgIndex = array_search(
+        \App\Provisioning\Steps\Host\ConfigureWireguard::class,
+        config('provisioning.pipelines.host'),
+        true,
+    );
+    $run->update(['current_step' => $wgIndex, 'status' => 'running']);
+    $runner->advance($run->fresh());
+
+    expect(RunResource::where('run_id', $run->id)->where('kind', 'wg_peer')->count())->toBe(1);
+});

tests/Feature/Provisioning/HostStepsTest.php

@@ -0,0 +1,235 @@
+<?php
+
+use App\Models\Host;
+use App\Models\ProvisioningRun;
+use App\Models\RunResource;
+use App\Provisioning\Steps\Host\CompleteHostOnboarding;
+use App\Provisioning\Steps\Host\ConfigureWireguard;
+use App\Provisioning\Steps\Host\CreateAutomationToken;
+use App\Provisioning\Steps\Host\EstablishSshTrust;
+use App\Provisioning\Steps\Host\InstallProxmoxVe;
+use App\Provisioning\Steps\Host\PrepareBaseSystem;
+use App\Provisioning\Steps\Host\RebootIntoPveKernel;
+use App\Provisioning\Steps\Host\RegisterCapacity;
+use App\Provisioning\Steps\Host\ValidateHostInput;
+use App\Provisioning\Steps\Host\VerifyProxmoxApi;
+use App\Services\Ssh\CommandResult;
+use Illuminate\Support\Facades\Crypt;
+
+function hostRun(Host $host, array $context = []): ProvisioningRun
+{
+    return ProvisioningRun::factory()->forHost($host)->create(['context' => $context]);
+}
+
+// --- ValidateHostInput ---
+
+it('advances on valid host input and marks the host onboarding', function () {
+    $host = Host::factory()->create();
+    $run = hostRun($host, ['root_password' => Crypt::encryptString('pw')]);
+
+    expect(app(ValidateHostInput::class)->execute($run)->type)->toBe('advance')
+        ->and($host->fresh()->status)->toBe('onboarding');
+});
+
+it('fails validation when the root password is missing', function () {
+    $run = hostRun(Host::factory()->create(), []);
+
+    expect(app(ValidateHostInput::class)->execute($run)->type)->toBe('fail');
+});
+
+// --- EstablishSshTrust ---
+
+it('establishes ssh trust, pins the host key and scrubs the password', function () {
+    $s = fakeServices();
+    $host = Host::factory()->create(['public_ip' => '203.0.113.5']);
+    $run = hostRun($host, ['root_password' => Crypt::encryptString('rootpw')]);
+
+    expect(app(EstablishSshTrust::class)->execute($run)->type)->toBe('advance')
+        ->and($host->fresh()->ssh_host_key)->not->toBeNull()
+        ->and($run->fresh()->context('root_password'))->toBeNull()
+        ->and($s['shell']->connectionsWith('password'))->toHaveCount(1)
+        ->and($s['shell']->connectionsWith('key'))->toHaveCount(1);
+});
+
+it('skips ssh trust once the host key is already pinned', function () {
+    $s = fakeServices();
+    $host = Host::factory()->create(['ssh_host_key' => 'SHA256:pinned']);
+    $run = hostRun($host, []);
+
+    expect(app(EstablishSshTrust::class)->execute($run)->type)->toBe('advance')
+        ->and($s['shell']->connectionsWith('password'))->toHaveCount(0);
+});
+
+it('throws when the host is unreachable (runner turns it into a retry)', function () {
+    $s = fakeServices();
+    $s['shell']->failConnect = true;
+    $run = hostRun(Host::factory()->create(), ['root_password' => Crypt::encryptString('pw')]);
+
+    expect(fn () => app(EstablishSshTrust::class)->execute($run))->toThrow(RuntimeException::class);
+});
+
+// --- PrepareBaseSystem ---
+
+it('prepares the base system', function () {
+    $s = fakeServices();
+    $run = hostRun(Host::factory()->create());
+
+    expect(app(PrepareBaseSystem::class)->execute($run)->type)->toBe('advance')
+        ->and($s['shell']->ran('apt-get update'))->toBeTrue();
+});
+
+it('retries base preparation on an apt failure', function () {
+    $s = fakeServices();
+    $s['shell']->script('apt-get update', CommandResult::failure(1, 'network down'));
+    $run = hostRun(Host::factory()->create());
+
+    expect(app(PrepareBaseSystem::class)->execute($run)->type)->toBe('retry');
+});
+
+// --- ConfigureWireguard ---
+
+it('configures wireguard and registers the peer exactly once', function () {
+    $s = fakeServices();
+    $s['shell']->script('wg pubkey', CommandResult::success('HOSTPUB='));
+    $host = Host::factory()->create();
+    $run = hostRun($host);
+
+    expect(app(ConfigureWireguard::class)->execute($run)->type)->toBe('advance');
+    $host->refresh();
+    expect($host->wg_ip)->not->toBeNull()
+        ->and($host->wg_pubkey)->toBe('HOSTPUB=')
+        ->and($s['hub']->peers())->toHaveCount(1);
+
+    // Re-run: idempotent short-circuit, still one peer resource.
+    app(ConfigureWireguard::class)->execute($run->fresh());
+    expect(RunResource::where('run_id', $run->id)->where('kind', 'wg_peer')->count())->toBe(1);
+});
+
+it('retries wireguard while the handshake is not up', function () {
+    $s = fakeServices();
+    $s['shell']->script('wg pubkey', CommandResult::success('HOSTPUB='));
+    $s['shell']->script('ping', CommandResult::failure(1));
+    $run = hostRun(Host::factory()->create());
+
+    expect(app(ConfigureWireguard::class)->execute($run)->type)->toBe('retry');
+});
+
+// --- InstallProxmoxVe ---
+
+it('skips the install when proxmox-ve is already present', function () {
+    $s = fakeServices();
+    $s['shell']->script('dpkg -l proxmox-ve', CommandResult::success('ii  proxmox-ve'));
+    $run = hostRun(Host::factory()->create());
+
+    expect(app(InstallProxmoxVe::class)->execute($run)->type)->toBe('advance')
+        ->and($s['shell']->ran('apt-get -y install proxmox-ve'))->toBeFalse();
+});
+
+it('installs proxmox-ve when it is absent', function () {
+    $s = fakeServices();
+    $s['shell']->script('dpkg -l proxmox-ve', CommandResult::failure(1));
+    $run = hostRun(Host::factory()->create());
+
+    expect(app(InstallProxmoxVe::class)->execute($run)->type)->toBe('advance')
+        ->and($s['shell']->ran('apt-get -y install proxmox-ve'))->toBeTrue();
+});
+
+// --- RebootIntoPveKernel ---
+
+it('issues the reboot on first execution', function () {
+    $s = fakeServices();
+    $run = hostRun(Host::factory()->create());
+
+    expect(app(RebootIntoPveKernel::class)->execute($run)->type)->toBe('retry')
+        ->and($run->fresh()->context('reboot_issued'))->toBeTrue()
+        ->and($s['shell']->ran('reboot'))->toBeTrue();
+});
+
+it('advances once the proxmox kernel is up', function () {
+    $s = fakeServices();
+    $s['shell']->script('uname -r', CommandResult::success('6.8.12-4-pve'));
+    $run = hostRun(Host::factory()->create(), [
+        'reboot_issued' => true,
+        'reboot_deadline' => now()->addMinutes(10)->toIso8601String(),
+    ]);
+
+    expect(app(RebootIntoPveKernel::class)->execute($run)->type)->toBe('advance');
+});
+
+it('keeps waiting while the old kernel is still running', function () {
+    $s = fakeServices();
+    $s['shell']->script('uname -r', CommandResult::success('6.1.0-18-amd64'));
+    $run = hostRun(Host::factory()->create(), [
+        'reboot_issued' => true,
+        'reboot_deadline' => now()->addMinutes(10)->toIso8601String(),
+    ]);
+
+    expect(app(RebootIntoPveKernel::class)->execute($run)->type)->toBe('retry');
+});
+
+it('fails when the host does not return before the reboot deadline', function () {
+    $s = fakeServices();
+    $s['shell']->script('uname -r', CommandResult::success('6.1.0-18-amd64'));
+    $run = hostRun(Host::factory()->create(), [
+        'reboot_issued' => true,
+        'reboot_deadline' => now()->subMinute()->toIso8601String(),
+    ]);
+
+    expect(app(RebootIntoPveKernel::class)->execute($run)->type)->toBe('fail');
+});
+
+// --- CreateAutomationToken ---
+
+it('creates and persists the automation token exactly once', function () {
+    $s = fakeServices();
+    $host = Host::factory()->create();
+    $run = hostRun($host);
+
+    expect(app(CreateAutomationToken::class)->execute($run)->type)->toBe('advance');
+    $host->refresh();
+    expect($host->api_token_ref)->toContain('automation@pve')
+        ->and($host->api_token_ref)->toContain('fake-secret')
+        ->and($s['pve']->tokenCalls)->toBe(1);
+
+    // Re-run: idempotent short-circuit, no second token call.
+    app(CreateAutomationToken::class)->execute($run->fresh());
+    expect($s['pve']->tokenCalls)->toBe(1)
+        ->and(RunResource::where('run_id', $run->id)->where('kind', 'pve_token')->count())->toBe(1);
+});
+
+// --- VerifyProxmoxApi ---
+
+it('advances when the api returns nodes', function () {
+    fakeServices();
+    expect(app(VerifyProxmoxApi::class)->execute(hostRun(Host::factory()->create()))->type)->toBe('advance');
+});
+
+it('retries when the api returns no nodes yet', function () {
+    $s = fakeServices();
+    $s['pve']->nodes = [];
+    expect(app(VerifyProxmoxApi::class)->execute(hostRun(Host::factory()->create()))->type)->toBe('retry');
+});
+
+// --- RegisterCapacity ---
+
+it('registers node capacity on the host', function () {
+    fakeServices();
+    $host = Host::factory()->create();
+    $run = hostRun($host);
+
+    expect(app(RegisterCapacity::class)->execute($run)->type)->toBe('advance');
+    $host->refresh();
+    expect($host->cpu_cores)->toBe(16)
+        ->and($host->total_ram_mb)->toBe(65536)
+        ->and($host->total_gb)->toBe(1024)
+        ->and($host->last_seen_at)->not->toBeNull();
+});
+
+// --- CompleteHostOnboarding ---
+
+it('marks the host active on completion', function () {
+    $host = Host::factory()->create(['status' => 'onboarding']);
+
+    expect(app(CompleteHostOnboarding::class)->execute(hostRun($host))->type)->toBe('advance')
+        ->and($host->fresh()->status)->toBe('active');
+});