CluPilotCloud/tests/Feature/Provisioning/HostOnboardingEndToEndTest.php

<?php

use App\Actions\StartHostOnboarding;
use App\Models\ProvisioningRun;
use App\Models\RunResource;
use App\Provisioning\RunRunner;
use App\Provisioning\Steps\Host\ConfigureWireguard;
use App\Services\Ssh\CommandResult;
use Illuminate\Support\Facades\Queue;

/*
| Both tests below drive the state machine against a fake whose default result is
| success, so what they prove is SEQUENCING — that the steps run in order, that
| the run reaches `completed`, and that external resources are created exactly
| once across a crash. They deliberately do not stand in for the per-step
| assertions in HostStepsTest: no command's content is checked here, and a step
| that stopped doing its work while still advancing would pass both.
*/

beforeEach(fn () => config()->set('admin_access.secrets_key', 'base64:'.base64_encode(random_bytes(32))));

it('drives a fresh host all the way to active (mocked)', function () {
    Queue::fake(); // drive the runner manually, one step per advance
    $s = fakeServices();
    $s['shell']->script('wg pubkey', CommandResult::success('HOSTPUBKEY0000='));
    $s['shell']->script('uname -r', CommandResult::success('6.8.12-4-pve'));
    $s['shell']->script('pveum user token add', CommandResult::success(
        json_encode(['full-tokenid' => 'automation@pve!clupilot', 'value' => 'tok-secret-123'])
    ));
    // What a healthy host answers to the checks the pipeline now actually makes:
    // a Debian release Proxmox publishes a suite for, a /boot it can reboot into,
    // a datacenter firewall that reads back as enabled, and a golden template to
    // clone from. None of these is scenery — every one of them is a step that
    // fails loudly rather than advancing on the fake's default success.
    $s['shell']->script('/etc/os-release', CommandResult::success("debian|trixie|Debian GNU/Linux 13 (trixie)\n"));
    $s['shell']->script('ls -1 /boot/vmlinuz-*-pve', CommandResult::success("/boot/vmlinuz-6.8.12-4-pve\n"));
    $s['shell']->script('ls -1 /boot/initrd.img-*-pve', CommandResult::success("/boot/initrd.img-6.8.12-4-pve\n"));
    $s['shell']->script('pvesh get /cluster/firewall/options', CommandResult::success('{"enable":1}'));
    $s['pve']->clonedVmids[] = 9000; // the template the seeded catalogue sells

    $host = app(StartHostOnboarding::class)->run([
        'name' => 'pve-fsn-9',
        'datacenter' => 'fsn',
        'public_ip' => '203.0.113.9',
        'root_password' => 'rootpw',
    ]);

    $run = ProvisioningRun::query()->where('subject_id', $host->id)->firstOrFail();
    $runner = app(RunRunner::class);

    for ($i = 0; $i < 40; $i++) {
        $run->refresh();
        if (in_array($run->status, ['completed', 'failed'], true)) {
            break;
        }
        if ($run->next_attempt_at?->isFuture()) {
            $run->update(['next_attempt_at' => now()]); // simulate the scheduler waking a due run
        }
        $runner->advance($run);
    }

    $run->refresh();
    $host->refresh();

    expect($run->status)->toBe('completed')
        ->and($run->error)->toBeNull()
        ->and($host->status)->toBe('active')
        ->and($host->wg_ip)->not->toBeNull()
        ->and($host->total_gb)->toBe(1024)
        ->and($host->cpu_cores)->toBe(16)
        ->and($host->api_token_ref)->toContain('automation@pve')
        // external resources created exactly once
        ->and(RunResource::where('run_id', $run->id)->where('kind', 'wg_peer')->count())->toBe(1)
        ->and(RunResource::where('run_id', $run->id)->where('kind', 'pve_token')->count())->toBe(1)
        ->and(RunResource::where('run_id', $run->id)->where('kind', 'host_firewall')->count())->toBe(1)
        // one-time root password scrubbed from state
        ->and($run->context('root_password'))->toBeNull();

    // The firewall step ran last, over the tunnel, and left its rules + the
    // manual emergency-release script on the host.
    expect($s['shell']->files())->toHaveKey('/etc/nftables.conf')
        ->and($s['shell']->files())->toHaveKey('/usr/local/sbin/clupilot-emergency-open-firewall.sh');
});

it('does not duplicate external resources when a step re-runs after a crash', function () {
    Queue::fake();
    $s = fakeServices();
    $s['shell']->script('wg pubkey', CommandResult::success('HOSTPUBKEY0000='));
    $s['shell']->script('uname -r', CommandResult::success('6.8.12-4-pve'));
    $s['shell']->script('pveum user token add', CommandResult::success(
        json_encode(['full-tokenid' => 'automation@pve!clupilot', 'value' => 'tok-secret-123'])
    ));
    // What a healthy host answers to the checks the pipeline now actually makes:
    // a Debian release Proxmox publishes a suite for, a /boot it can reboot into,
    // a datacenter firewall that reads back as enabled, and a golden template to
    // clone from. None of these is scenery — every one of them is a step that
    // fails loudly rather than advancing on the fake's default success.
    $s['shell']->script('/etc/os-release', CommandResult::success("debian|trixie|Debian GNU/Linux 13 (trixie)\n"));
    $s['shell']->script('ls -1 /boot/vmlinuz-*-pve', CommandResult::success("/boot/vmlinuz-6.8.12-4-pve\n"));
    $s['shell']->script('ls -1 /boot/initrd.img-*-pve', CommandResult::success("/boot/initrd.img-6.8.12-4-pve\n"));
    $s['shell']->script('pvesh get /cluster/firewall/options', CommandResult::success('{"enable":1}'));
    $s['pve']->clonedVmids[] = 9000; // the template the seeded catalogue sells

    $host = app(StartHostOnboarding::class)->run([
        'name' => 'pve-fsn-10',
        'datacenter' => 'fsn',
        'public_ip' => '203.0.113.10',
        'root_password' => 'rootpw',
    ]);
    $run = ProvisioningRun::query()->where('subject_id', $host->id)->firstOrFail();
    $runner = app(RunRunner::class);

    // Advance a few steps, then simulate a crash by rewinding the run to the
    // WireGuard step and replaying it — the breadcrumb must prevent duplicates.
    for ($i = 0; $i < 6; $i++) {
        $run->refresh();
        if ($run->next_attempt_at?->isFuture()) {
            $run->update(['next_attempt_at' => now()]);
        }
        $runner->advance($run);
    }
    $run->refresh();
    $wgIndex = array_search(
        ConfigureWireguard::class,
        config('provisioning.pipelines.host'),
        true,
    );
    $run->update(['current_step' => $wgIndex, 'status' => 'running']);
    $runner->advance($run->fresh());

    expect(RunResource::where('run_id', $run->id)->where('kind', 'wg_peer')->count())->toBe(1);
});