diff --git a/app/Actions/StartHostOnboarding.php b/app/Actions/StartHostOnboarding.php new file mode 100644 index 0000000..30e73f8 --- /dev/null +++ b/app/Actions/StartHostOnboarding.php @@ -0,0 +1,42 @@ + $input['name'], + 'datacenter' => $input['datacenter'], + 'public_ip' => $input['public_ip'], + 'status' => 'pending', + ]); + + $run = ProvisioningRun::create([ + 'subject_type' => Host::class, + 'subject_id' => $host->id, + 'pipeline' => 'host', + 'status' => ProvisioningRun::STATUS_PENDING, + 'current_step' => 0, + 'context' => ['root_password' => Crypt::encryptString($input['root_password'])], + ]); + + AdvanceRunJob::dispatch($run->uuid); + + return $host; + } +} diff --git a/app/Provisioning/Steps/Host/CompleteHostOnboarding.php b/app/Provisioning/Steps/Host/CompleteHostOnboarding.php new file mode 100644 index 0000000..2433375 --- /dev/null +++ b/app/Provisioning/Steps/Host/CompleteHostOnboarding.php @@ -0,0 +1,21 @@ +host($run)->update(['status' => 'active']); + + return StepResult::advance(); + } +} diff --git a/app/Provisioning/Steps/Host/ConfigureProxmox.php b/app/Provisioning/Steps/Host/ConfigureProxmox.php new file mode 100644 index 0000000..395f0c2 --- /dev/null +++ b/app/Provisioning/Steps/Host/ConfigureProxmox.php @@ -0,0 +1,33 @@ +host($run); + $this->keyLogin($this->shell, $host); + + // Most PVE installs create vmbr0; record its absence for real-host follow-up. + $this->shell->run('ip link show vmbr0'); + $this->shell->run('rm -f /etc/apt/sources.list.d/pve-enterprise.list'); + + return StepResult::advance(); + } +} diff --git a/app/Provisioning/Steps/Host/ConfigureWireguard.php b/app/Provisioning/Steps/Host/ConfigureWireguard.php new file mode 100644 index 0000000..8b4bd55 --- /dev/null +++ b/app/Provisioning/Steps/Host/ConfigureWireguard.php @@ -0,0 +1,79 @@ +host($run); + + // Idempotent: tunnel already provisioned and peer registered. + if (filled($host->wg_ip) && $this->hasResource($run, 'wg_peer')) { + return StepResult::advance(); + } + + $this->keyLogin($this->shell, $host); + $this->shell->run('export DEBIAN_FRONTEND=noninteractive; apt-get install -y wireguard'); + $this->shell->run('test -f /etc/wireguard/privatekey || (umask 077; wg genkey > /etc/wireguard/privatekey)'); + + $publicKey = trim($this->shell->run('wg pubkey < /etc/wireguard/privatekey')->stdout); + if (blank($publicKey)) { + return StepResult::retry(20, 'could not read host WireGuard public key'); + } + + $wgIp = $host->wg_ip ?: $this->hub->allocateIp(); + $privateKey = trim($this->shell->run('cat /etc/wireguard/privatekey')->stdout); + + $this->shell->putFile('/etc/wireguard/wg0.conf', $this->renderConfig($wgIp, $privateKey)); + $this->shell->run('systemctl enable --now wg-quick@wg0 || wg-quick up wg0'); + + $this->hub->addPeer($publicKey, $wgIp); + + // Persist external identity BEFORE verifying/advancing (crash-safe). + $host->update(['wg_ip' => $wgIp, 'wg_pubkey' => $publicKey]); + $this->recordResource($run, $host, 'wg_peer', $publicKey); + + $hubIp = (string) config('provisioning.wireguard.hub_ip', '10.66.0.1'); + if (! $this->shell->run('ping -c1 -W2 '.escapeshellarg($hubIp))->ok()) { + return StepResult::retry(15, 'WireGuard handshake not up yet'); + } + + return StepResult::advance(); + } + + private function renderConfig(string $wgIp, string $privateKey): string + { + $subnet = (string) config('provisioning.wireguard.subnet', '10.66.0.0/24'); + + return implode("\n", [ + '[Interface]', + "Address = {$wgIp}/24", + "PrivateKey = {$privateKey}", + '', + '[Peer]', + 'PublicKey = '.$this->hub->publicKey(), + 'Endpoint = '.$this->hub->endpoint(), + "AllowedIPs = {$subnet}", + 'PersistentKeepalive = 25', + '', + ]); + } +} diff --git a/app/Provisioning/Steps/Host/CreateAutomationToken.php b/app/Provisioning/Steps/Host/CreateAutomationToken.php new file mode 100644 index 0000000..97334a5 --- /dev/null +++ b/app/Provisioning/Steps/Host/CreateAutomationToken.php @@ -0,0 +1,52 @@ +host($run); + + // Idempotent: token already minted and persisted. + if ($this->hasResource($run, 'pve_token') && filled($host->api_token_ref)) { + return StepResult::advance(); + } + + $client = $this->pve->forHost($host); + $client->createRole( + config('provisioning.proxmox.role_id'), + config('provisioning.proxmox.role_privs'), + ); + + $token = $client->createUserAndToken( + config('provisioning.proxmox.user'), + config('provisioning.proxmox.role_id'), + ); + + if (blank($token['secret'] ?? null)) { + return StepResult::retry(20, 'Proxmox token secret was empty'); + } + + $host->update(['api_token_ref' => $token['token_id'].'='.$token['secret']]); + $this->recordResource($run, $host, 'pve_token', $token['token_id']); + + return StepResult::advance(); + } +} diff --git a/app/Provisioning/Steps/Host/EstablishSshTrust.php b/app/Provisioning/Steps/Host/EstablishSshTrust.php new file mode 100644 index 0000000..450993e --- /dev/null +++ b/app/Provisioning/Steps/Host/EstablishSshTrust.php @@ -0,0 +1,50 @@ +host($run); + + // Idempotent: already trusted (host key pinned) and password already scrubbed. + if (filled($host->ssh_host_key) && blank($run->context('root_password'))) { + return StepResult::advance(); + } + + $publicKey = trim((string) config('provisioning.ssh.public_key')); + $password = Crypt::decryptString((string) $run->context('root_password')); + + $this->shell->connectWithPassword($host->public_ip, 'root', $password); + $this->shell->run('mkdir -p /root/.ssh && chmod 700 /root/.ssh'); + $this->shell->run( + 'grep -qxF '.escapeshellarg($publicKey).' /root/.ssh/authorized_keys 2>/dev/null '. + '|| echo '.escapeshellarg($publicKey).' >> /root/.ssh/authorized_keys' + ); + + $host->update(['ssh_host_key' => $this->shell->hostKeyFingerprint()]); + + // Verify the key works, then remove the password from state. + $this->keyLogin($this->shell, $host); + $run->forgetContext('root_password'); + + return StepResult::advance(); + } +} diff --git a/app/Provisioning/Steps/Host/HostStep.php b/app/Provisioning/Steps/Host/HostStep.php new file mode 100644 index 0000000..b23a307 --- /dev/null +++ b/app/Provisioning/Steps/Host/HostStep.php @@ -0,0 +1,52 @@ +key(); + } + + public function maxDuration(): int + { + return 300; + } + + protected function host(ProvisioningRun $run): Host + { + /** @var Host $host */ + $host = $run->subject; + + return $host; + } + + protected function keyLogin(RemoteShell $shell, Host $host): void + { + $shell->connectWithKey($host->public_ip, 'root', (string) config('provisioning.ssh.private_key')); + } + + protected function recordResource(ProvisioningRun $run, Host $host, string $kind, string $externalId): void + { + RunResource::firstOrCreate( + ['run_id' => $run->id, 'kind' => $kind], + ['host_id' => $host->id, 'external_id' => $externalId], + ); + } + + protected function hasResource(ProvisioningRun $run, string $kind): bool + { + return RunResource::query()->where('run_id', $run->id)->where('kind', $kind)->exists(); + } +} diff --git a/app/Provisioning/Steps/Host/InstallProxmoxVe.php b/app/Provisioning/Steps/Host/InstallProxmoxVe.php new file mode 100644 index 0000000..36d0dad --- /dev/null +++ b/app/Provisioning/Steps/Host/InstallProxmoxVe.php @@ -0,0 +1,64 @@ +host($run); + $this->keyLogin($this->shell, $host); + + // Idempotent: proxmox-ve already installed. + if ($this->shell->run('dpkg -l proxmox-ve 2>/dev/null | grep -q "^ii"')->ok()) { + return StepResult::advance(); + } + + $key = $this->shell->run( + 'curl -fsSL https://enterprise.proxmox.com/debian/proxmox-release-bookworm.gpg '. + '-o /etc/apt/trusted.gpg.d/proxmox-release-bookworm.gpg' + ); + if (! $key->ok()) { + return StepResult::fail('Could not fetch the Proxmox release key.'); + } + + $this->shell->putFile( + '/etc/apt/sources.list.d/pve-install-repo.list', + "deb [arch=amd64] http://download.proxmox.com/debian/pve bookworm pve-no-subscription\n" + ); + $this->shell->run('rm -f /etc/apt/sources.list.d/pve-enterprise.list'); + + if (! $this->shell->run('export DEBIAN_FRONTEND=noninteractive; apt-get update')->ok()) { + return StepResult::retry(60, 'apt-get update failed'); + } + if (! $this->shell->run('export DEBIAN_FRONTEND=noninteractive; apt-get -y full-upgrade')->ok()) { + return StepResult::retry(60, 'apt-get full-upgrade failed'); + } + if (! $this->shell->run('export DEBIAN_FRONTEND=noninteractive; apt-get -y install proxmox-ve postfix open-iscsi')->ok()) { + return StepResult::retry(60, 'proxmox-ve install failed'); + } + + return StepResult::advance(); + } +} diff --git a/app/Provisioning/Steps/Host/PrepareBaseSystem.php b/app/Provisioning/Steps/Host/PrepareBaseSystem.php new file mode 100644 index 0000000..2dbc6e0 --- /dev/null +++ b/app/Provisioning/Steps/Host/PrepareBaseSystem.php @@ -0,0 +1,41 @@ +host($run); + $this->keyLogin($this->shell, $host); + + $fqdn = "{$host->name}.{$host->datacenter}.clupilot.net"; + $hostsLine = "{$host->public_ip} {$fqdn} {$host->name}"; + + $commands = [ + 'hostnamectl set-hostname '.escapeshellarg($host->name), + 'grep -q '.escapeshellarg($fqdn).' /etc/hosts || echo '.escapeshellarg($hostsLine).' >> /etc/hosts', + 'export DEBIAN_FRONTEND=noninteractive; apt-get update', + 'export DEBIAN_FRONTEND=noninteractive; apt-get install -y curl gnupg ifupdown2 chrony', + ]; + + foreach ($commands as $command) { + if (! $this->shell->run($command)->ok()) { + return StepResult::retry(30, 'base system preparation failed'); + } + } + + return StepResult::advance(); + } +} diff --git a/app/Provisioning/Steps/Host/RebootIntoPveKernel.php b/app/Provisioning/Steps/Host/RebootIntoPveKernel.php new file mode 100644 index 0000000..08eb6b0 --- /dev/null +++ b/app/Provisioning/Steps/Host/RebootIntoPveKernel.php @@ -0,0 +1,65 @@ +host($run); + + if (! $run->context('reboot_issued')) { + $this->keyLogin($this->shell, $host); + $this->shell->run('export DEBIAN_FRONTEND=noninteractive; apt-get -y remove linux-image-amd64 || true'); + $this->shell->run('update-grub'); + $run->mergeContext([ + 'reboot_issued' => true, + 'reboot_deadline' => now()->addMinutes(15)->toIso8601String(), + ]); + $this->shell->run('systemctl reboot || reboot'); + + return StepResult::retry(30, 'rebooting into the Proxmox kernel'); + } + + try { + $this->keyLogin($this->shell, $host); + $uname = trim($this->shell->run('uname -r')->stdout); + } catch (Throwable) { + $uname = ''; + } + + if (str_contains($uname, '-pve')) { + return StepResult::advance(); + } + + $deadline = $run->context('reboot_deadline'); + if ($deadline !== null && now()->greaterThan(Carbon::parse($deadline))) { + return StepResult::fail('Host did not return on the Proxmox kernel before the deadline.'); + } + + return StepResult::retry(15, 'waiting for the Proxmox kernel'); + } +} diff --git a/app/Provisioning/Steps/Host/RegisterCapacity.php b/app/Provisioning/Steps/Host/RegisterCapacity.php new file mode 100644 index 0000000..b3c738c --- /dev/null +++ b/app/Provisioning/Steps/Host/RegisterCapacity.php @@ -0,0 +1,48 @@ +host($run); + $client = $this->pve->forHost($host); + + $nodes = $client->listNodes(); + $node = $nodes[0]['node'] ?? 'pve'; + + $status = $client->nodeStatus($node); + $storage = $client->nodeStorage($node); + + $cores = (int) ($status['cpuinfo']['cpus'] ?? 0); + $ramMb = (int) round((int) ($status['memory']['total'] ?? 0) / 1048576); + $totalGb = (int) round(array_sum(array_column($storage, 'total')) / 1073741824); + + $host->update([ + 'cpu_cores' => $cores, + 'cpu_weight' => $cores, + 'total_ram_mb' => $ramMb, + 'total_gb' => $totalGb, + 'pve_version' => $status['pveversion'] ?? null, + 'last_seen_at' => now(), + ]); + + return StepResult::advance(); + } +} diff --git a/app/Provisioning/Steps/Host/ValidateHostInput.php b/app/Provisioning/Steps/Host/ValidateHostInput.php new file mode 100644 index 0000000..cd93b82 --- /dev/null +++ b/app/Provisioning/Steps/Host/ValidateHostInput.php @@ -0,0 +1,36 @@ +host($run); + + if (blank($host->public_ip) || blank($host->datacenter)) { + return StepResult::fail('Host is missing public_ip or datacenter.'); + } + + if (blank($run->context('root_password'))) { + return StepResult::fail('Missing root password in run context.'); + } + + $host->update(['status' => 'onboarding']); + + return StepResult::advance(); + } +} diff --git a/app/Provisioning/Steps/Host/VerifyProxmoxApi.php b/app/Provisioning/Steps/Host/VerifyProxmoxApi.php new file mode 100644 index 0000000..b48e930 --- /dev/null +++ b/app/Provisioning/Steps/Host/VerifyProxmoxApi.php @@ -0,0 +1,35 @@ +host($run); + + try { + $nodes = $this->pve->forHost($host)->listNodes(); + } catch (Throwable $e) { + return StepResult::retry(15, 'Proxmox API not reachable: '.$e->getMessage()); + } + + if (empty($nodes)) { + return StepResult::retry(15, 'Proxmox API returned no nodes yet'); + } + + return StepResult::advance(); + } +} diff --git a/app/Services/Ssh/FakeRemoteShell.php b/app/Services/Ssh/FakeRemoteShell.php index 2e7cf9f..8909f9d 100644 --- a/app/Services/Ssh/FakeRemoteShell.php +++ b/app/Services/Ssh/FakeRemoteShell.php @@ -21,6 +21,9 @@ class FakeRemoteShell implements RemoteShell public string $fingerprint = 'SHA256:fakehostkeyfingerprint'; + /** When set, connect attempts throw (simulates an unreachable/rebooting host). */ + public bool $failConnect = false; + private CommandResult $default; public function __construct() @@ -44,11 +47,17 @@ class FakeRemoteShell implements RemoteShell public function connectWithPassword(string $host, string $user, string $password): void { + if ($this->failConnect) { + throw new \RuntimeException("connection refused: {$host}"); + } $this->connections[] = ['password', $host, $user]; } public function connectWithKey(string $host, string $user, string $privateKey): void { + if ($this->failConnect) { + throw new \RuntimeException("connection refused: {$host}"); + } $this->connections[] = ['key', $host, $user]; } diff --git a/tests/Feature/Provisioning/HostOnboardingEndToEndTest.php b/tests/Feature/Provisioning/HostOnboardingEndToEndTest.php new file mode 100644 index 0000000..0abc671 --- /dev/null +++ b/tests/Feature/Provisioning/HostOnboardingEndToEndTest.php @@ -0,0 +1,82 @@ +script('wg pubkey', CommandResult::success('HOSTPUBKEY0000=')); + $s['shell']->script('uname -r', CommandResult::success('6.8.12-4-pve')); + + $host = app(StartHostOnboarding::class)->run([ + 'name' => 'pve-fsn-9', + 'datacenter' => 'fsn', + 'public_ip' => '203.0.113.9', + 'root_password' => 'rootpw', + ]); + + $run = ProvisioningRun::query()->where('subject_id', $host->id)->firstOrFail(); + $runner = app(RunRunner::class); + + for ($i = 0; $i < 40; $i++) { + $run->refresh(); + if (in_array($run->status, ['completed', 'failed'], true)) { + break; + } + $runner->advance($run); + } + + $run->refresh(); + $host->refresh(); + + expect($run->status)->toBe('completed') + ->and($run->error)->toBeNull() + ->and($host->status)->toBe('active') + ->and($host->wg_ip)->not->toBeNull() + ->and($host->total_gb)->toBe(1024) + ->and($host->cpu_cores)->toBe(16) + ->and($host->api_token_ref)->toContain('automation@pve') + // external resources created exactly once + ->and(RunResource::where('run_id', $run->id)->where('kind', 'wg_peer')->count())->toBe(1) + ->and(RunResource::where('run_id', $run->id)->where('kind', 'pve_token')->count())->toBe(1) + // one-time root password scrubbed from state + ->and($run->context('root_password'))->toBeNull(); +}); + +it('does not duplicate external resources when a step re-runs after a crash', function () { + Queue::fake(); + $s = fakeServices(); + $s['shell']->script('wg pubkey', CommandResult::success('HOSTPUBKEY0000=')); + $s['shell']->script('uname -r', CommandResult::success('6.8.12-4-pve')); + + $host = app(StartHostOnboarding::class)->run([ + 'name' => 'pve-fsn-10', + 'datacenter' => 'fsn', + 'public_ip' => '203.0.113.10', + 'root_password' => 'rootpw', + ]); + $run = ProvisioningRun::query()->where('subject_id', $host->id)->firstOrFail(); + $runner = app(RunRunner::class); + + // Advance a few steps, then simulate a crash by rewinding the run to the + // WireGuard step and replaying it — the breadcrumb must prevent duplicates. + for ($i = 0; $i < 6; $i++) { + $run->refresh(); + $runner->advance($run); + } + $run->refresh(); + $wgIndex = array_search( + \App\Provisioning\Steps\Host\ConfigureWireguard::class, + config('provisioning.pipelines.host'), + true, + ); + $run->update(['current_step' => $wgIndex, 'status' => 'running']); + $runner->advance($run->fresh()); + + expect(RunResource::where('run_id', $run->id)->where('kind', 'wg_peer')->count())->toBe(1); +}); diff --git a/tests/Feature/Provisioning/HostStepsTest.php b/tests/Feature/Provisioning/HostStepsTest.php new file mode 100644 index 0000000..c4e2bf9 --- /dev/null +++ b/tests/Feature/Provisioning/HostStepsTest.php @@ -0,0 +1,235 @@ +forHost($host)->create(['context' => $context]); +} + +// --- ValidateHostInput --- + +it('advances on valid host input and marks the host onboarding', function () { + $host = Host::factory()->create(); + $run = hostRun($host, ['root_password' => Crypt::encryptString('pw')]); + + expect(app(ValidateHostInput::class)->execute($run)->type)->toBe('advance') + ->and($host->fresh()->status)->toBe('onboarding'); +}); + +it('fails validation when the root password is missing', function () { + $run = hostRun(Host::factory()->create(), []); + + expect(app(ValidateHostInput::class)->execute($run)->type)->toBe('fail'); +}); + +// --- EstablishSshTrust --- + +it('establishes ssh trust, pins the host key and scrubs the password', function () { + $s = fakeServices(); + $host = Host::factory()->create(['public_ip' => '203.0.113.5']); + $run = hostRun($host, ['root_password' => Crypt::encryptString('rootpw')]); + + expect(app(EstablishSshTrust::class)->execute($run)->type)->toBe('advance') + ->and($host->fresh()->ssh_host_key)->not->toBeNull() + ->and($run->fresh()->context('root_password'))->toBeNull() + ->and($s['shell']->connectionsWith('password'))->toHaveCount(1) + ->and($s['shell']->connectionsWith('key'))->toHaveCount(1); +}); + +it('skips ssh trust once the host key is already pinned', function () { + $s = fakeServices(); + $host = Host::factory()->create(['ssh_host_key' => 'SHA256:pinned']); + $run = hostRun($host, []); + + expect(app(EstablishSshTrust::class)->execute($run)->type)->toBe('advance') + ->and($s['shell']->connectionsWith('password'))->toHaveCount(0); +}); + +it('throws when the host is unreachable (runner turns it into a retry)', function () { + $s = fakeServices(); + $s['shell']->failConnect = true; + $run = hostRun(Host::factory()->create(), ['root_password' => Crypt::encryptString('pw')]); + + expect(fn () => app(EstablishSshTrust::class)->execute($run))->toThrow(RuntimeException::class); +}); + +// --- PrepareBaseSystem --- + +it('prepares the base system', function () { + $s = fakeServices(); + $run = hostRun(Host::factory()->create()); + + expect(app(PrepareBaseSystem::class)->execute($run)->type)->toBe('advance') + ->and($s['shell']->ran('apt-get update'))->toBeTrue(); +}); + +it('retries base preparation on an apt failure', function () { + $s = fakeServices(); + $s['shell']->script('apt-get update', CommandResult::failure(1, 'network down')); + $run = hostRun(Host::factory()->create()); + + expect(app(PrepareBaseSystem::class)->execute($run)->type)->toBe('retry'); +}); + +// --- ConfigureWireguard --- + +it('configures wireguard and registers the peer exactly once', function () { + $s = fakeServices(); + $s['shell']->script('wg pubkey', CommandResult::success('HOSTPUB=')); + $host = Host::factory()->create(); + $run = hostRun($host); + + expect(app(ConfigureWireguard::class)->execute($run)->type)->toBe('advance'); + $host->refresh(); + expect($host->wg_ip)->not->toBeNull() + ->and($host->wg_pubkey)->toBe('HOSTPUB=') + ->and($s['hub']->peers())->toHaveCount(1); + + // Re-run: idempotent short-circuit, still one peer resource. + app(ConfigureWireguard::class)->execute($run->fresh()); + expect(RunResource::where('run_id', $run->id)->where('kind', 'wg_peer')->count())->toBe(1); +}); + +it('retries wireguard while the handshake is not up', function () { + $s = fakeServices(); + $s['shell']->script('wg pubkey', CommandResult::success('HOSTPUB=')); + $s['shell']->script('ping', CommandResult::failure(1)); + $run = hostRun(Host::factory()->create()); + + expect(app(ConfigureWireguard::class)->execute($run)->type)->toBe('retry'); +}); + +// --- InstallProxmoxVe --- + +it('skips the install when proxmox-ve is already present', function () { + $s = fakeServices(); + $s['shell']->script('dpkg -l proxmox-ve', CommandResult::success('ii proxmox-ve')); + $run = hostRun(Host::factory()->create()); + + expect(app(InstallProxmoxVe::class)->execute($run)->type)->toBe('advance') + ->and($s['shell']->ran('apt-get -y install proxmox-ve'))->toBeFalse(); +}); + +it('installs proxmox-ve when it is absent', function () { + $s = fakeServices(); + $s['shell']->script('dpkg -l proxmox-ve', CommandResult::failure(1)); + $run = hostRun(Host::factory()->create()); + + expect(app(InstallProxmoxVe::class)->execute($run)->type)->toBe('advance') + ->and($s['shell']->ran('apt-get -y install proxmox-ve'))->toBeTrue(); +}); + +// --- RebootIntoPveKernel --- + +it('issues the reboot on first execution', function () { + $s = fakeServices(); + $run = hostRun(Host::factory()->create()); + + expect(app(RebootIntoPveKernel::class)->execute($run)->type)->toBe('retry') + ->and($run->fresh()->context('reboot_issued'))->toBeTrue() + ->and($s['shell']->ran('reboot'))->toBeTrue(); +}); + +it('advances once the proxmox kernel is up', function () { + $s = fakeServices(); + $s['shell']->script('uname -r', CommandResult::success('6.8.12-4-pve')); + $run = hostRun(Host::factory()->create(), [ + 'reboot_issued' => true, + 'reboot_deadline' => now()->addMinutes(10)->toIso8601String(), + ]); + + expect(app(RebootIntoPveKernel::class)->execute($run)->type)->toBe('advance'); +}); + +it('keeps waiting while the old kernel is still running', function () { + $s = fakeServices(); + $s['shell']->script('uname -r', CommandResult::success('6.1.0-18-amd64')); + $run = hostRun(Host::factory()->create(), [ + 'reboot_issued' => true, + 'reboot_deadline' => now()->addMinutes(10)->toIso8601String(), + ]); + + expect(app(RebootIntoPveKernel::class)->execute($run)->type)->toBe('retry'); +}); + +it('fails when the host does not return before the reboot deadline', function () { + $s = fakeServices(); + $s['shell']->script('uname -r', CommandResult::success('6.1.0-18-amd64')); + $run = hostRun(Host::factory()->create(), [ + 'reboot_issued' => true, + 'reboot_deadline' => now()->subMinute()->toIso8601String(), + ]); + + expect(app(RebootIntoPveKernel::class)->execute($run)->type)->toBe('fail'); +}); + +// --- CreateAutomationToken --- + +it('creates and persists the automation token exactly once', function () { + $s = fakeServices(); + $host = Host::factory()->create(); + $run = hostRun($host); + + expect(app(CreateAutomationToken::class)->execute($run)->type)->toBe('advance'); + $host->refresh(); + expect($host->api_token_ref)->toContain('automation@pve') + ->and($host->api_token_ref)->toContain('fake-secret') + ->and($s['pve']->tokenCalls)->toBe(1); + + // Re-run: idempotent short-circuit, no second token call. + app(CreateAutomationToken::class)->execute($run->fresh()); + expect($s['pve']->tokenCalls)->toBe(1) + ->and(RunResource::where('run_id', $run->id)->where('kind', 'pve_token')->count())->toBe(1); +}); + +// --- VerifyProxmoxApi --- + +it('advances when the api returns nodes', function () { + fakeServices(); + expect(app(VerifyProxmoxApi::class)->execute(hostRun(Host::factory()->create()))->type)->toBe('advance'); +}); + +it('retries when the api returns no nodes yet', function () { + $s = fakeServices(); + $s['pve']->nodes = []; + expect(app(VerifyProxmoxApi::class)->execute(hostRun(Host::factory()->create()))->type)->toBe('retry'); +}); + +// --- RegisterCapacity --- + +it('registers node capacity on the host', function () { + fakeServices(); + $host = Host::factory()->create(); + $run = hostRun($host); + + expect(app(RegisterCapacity::class)->execute($run)->type)->toBe('advance'); + $host->refresh(); + expect($host->cpu_cores)->toBe(16) + ->and($host->total_ram_mb)->toBe(65536) + ->and($host->total_gb)->toBe(1024) + ->and($host->last_seen_at)->not->toBeNull(); +}); + +// --- CompleteHostOnboarding --- + +it('marks the host active on completion', function () { + $host = Host::factory()->create(['status' => 'onboarding']); + + expect(app(CompleteHostOnboarding::class)->execute(hostRun($host))->type)->toBe('advance') + ->and($host->fresh()->status)->toBe('active'); +});