CluPilotCloud/tests/Feature/Provisioning/HostStepsTest.php

<?php

use App\Models\Host;
use App\Models\ProvisioningRun;
use App\Models\RunResource;
use App\Provisioning\Steps\Host\CompleteHostOnboarding;
use App\Provisioning\Steps\Host\ConfigureWireguard;
use App\Provisioning\Steps\Host\CreateAutomationToken;
use App\Provisioning\Steps\Host\EstablishSshTrust;
use App\Provisioning\Steps\Host\InstallProxmoxVe;
use App\Provisioning\Steps\Host\PrepareBaseSystem;
use App\Provisioning\Steps\Host\RebootIntoPveKernel;
use App\Provisioning\Steps\Host\RegisterCapacity;
use App\Provisioning\Steps\Host\ValidateHostInput;
use App\Provisioning\Steps\Host\VerifyProxmoxApi;
use App\Services\Ssh\CommandResult;
use Illuminate\Support\Facades\Crypt;

function hostRun(Host $host, array $context = []): ProvisioningRun
{
    return ProvisioningRun::factory()->forHost($host)->create(['context' => $context]);
}

// --- ValidateHostInput ---

it('advances on valid host input and marks the host onboarding', function () {
    $host = Host::factory()->create();
    $run = hostRun($host, ['root_password' => Crypt::encryptString('pw')]);

    expect(app(ValidateHostInput::class)->execute($run)->type)->toBe('advance')
        ->and($host->fresh()->status)->toBe('onboarding');
});

it('fails validation when the root password is missing', function () {
    $run = hostRun(Host::factory()->create(), []);

    expect(app(ValidateHostInput::class)->execute($run)->type)->toBe('fail');
});

// --- EstablishSshTrust ---

it('establishes ssh trust, pins the host key and scrubs the password', function () {
    $s = fakeServices();
    $host = Host::factory()->create(['public_ip' => '203.0.113.5']);
    $run = hostRun($host, ['root_password' => Crypt::encryptString('rootpw')]);

    expect(app(EstablishSshTrust::class)->execute($run)->type)->toBe('advance')
        ->and($host->fresh()->ssh_host_key)->not->toBeNull()
        ->and($run->fresh()->context('root_password'))->toBeNull()
        ->and($s['shell']->connectionsWith('password'))->toHaveCount(1)
        ->and($s['shell']->connectionsWith('key'))->toHaveCount(1);
});

it('skips ssh trust once the host key is already pinned', function () {
    $s = fakeServices();
    $host = Host::factory()->create(['ssh_host_key' => 'SHA256:pinned']);
    $run = hostRun($host, []);

    expect(app(EstablishSshTrust::class)->execute($run)->type)->toBe('advance')
        ->and($s['shell']->connectionsWith('password'))->toHaveCount(0);
});

it('throws when the host is unreachable (runner turns it into a retry)', function () {
    $s = fakeServices();
    $s['shell']->failConnect = true;
    $run = hostRun(Host::factory()->create(), ['root_password' => Crypt::encryptString('pw')]);

    expect(fn () => app(EstablishSshTrust::class)->execute($run))->toThrow(RuntimeException::class);
});

// --- PrepareBaseSystem ---

it('prepares the base system', function () {
    $s = fakeServices();
    $run = hostRun(Host::factory()->create());

    expect(app(PrepareBaseSystem::class)->execute($run)->type)->toBe('advance')
        ->and($s['shell']->ran('apt-get update'))->toBeTrue();
});

it('retries base preparation on an apt failure', function () {
    $s = fakeServices();
    $s['shell']->script('apt-get update', CommandResult::failure(1, 'network down'));
    $run = hostRun(Host::factory()->create());

    expect(app(PrepareBaseSystem::class)->execute($run)->type)->toBe('retry');
});

// --- ConfigureWireguard ---

it('configures wireguard and registers the peer exactly once', function () {
    $s = fakeServices();
    $s['shell']->script('wg pubkey', CommandResult::success('HOSTPUB='));
    $host = Host::factory()->create();
    $run = hostRun($host);

    expect(app(ConfigureWireguard::class)->execute($run)->type)->toBe('advance');
    $host->refresh();
    expect($host->wg_ip)->not->toBeNull()
        ->and($host->wg_pubkey)->toBe('HOSTPUB=')
        ->and($s['hub']->peerIps())->toHaveCount(1);

    // Re-run: idempotent short-circuit, still one peer resource.
    app(ConfigureWireguard::class)->execute($run->fresh());
    expect(RunResource::where('run_id', $run->id)->where('kind', 'wg_peer')->count())->toBe(1);
});

it('retries wireguard while the handshake is not up', function () {
    $s = fakeServices();
    $s['shell']->script('wg pubkey', CommandResult::success('HOSTPUB='));
    $s['shell']->script('ping', CommandResult::failure(1));
    $run = hostRun(Host::factory()->create());

    expect(app(ConfigureWireguard::class)->execute($run)->type)->toBe('retry');
});

// --- InstallProxmoxVe ---

it('skips the install when proxmox-ve is already present', function () {
    $s = fakeServices();
    $s['shell']->script('dpkg -l proxmox-ve', CommandResult::success('ii  proxmox-ve'));
    $run = hostRun(Host::factory()->create());

    expect(app(InstallProxmoxVe::class)->execute($run)->type)->toBe('advance')
        ->and($s['shell']->ran('apt-get -y install proxmox-ve'))->toBeFalse();
});

it('installs proxmox-ve when it is absent', function () {
    $s = fakeServices();
    $s['shell']->script('dpkg -l proxmox-ve', CommandResult::failure(1));
    $run = hostRun(Host::factory()->create());

    expect(app(InstallProxmoxVe::class)->execute($run)->type)->toBe('advance')
        ->and($s['shell']->ran('apt-get -y install proxmox-ve'))->toBeTrue();
});

// --- RebootIntoPveKernel ---

it('issues the reboot on first execution', function () {
    $s = fakeServices();
    $run = hostRun(Host::factory()->create());

    expect(app(RebootIntoPveKernel::class)->execute($run)->type)->toBe('poll')
        ->and($run->fresh()->context('reboot_issued'))->toBeTrue()
        ->and($s['shell']->ran('reboot'))->toBeTrue();
});

it('advances once the proxmox kernel is up', function () {
    $s = fakeServices();
    $s['shell']->script('uname -r', CommandResult::success('6.8.12-4-pve'));
    $run = hostRun(Host::factory()->create(), [
        'reboot_issued' => true,
        'reboot_deadline' => now()->addMinutes(10)->toIso8601String(),
    ]);

    expect(app(RebootIntoPveKernel::class)->execute($run)->type)->toBe('advance');
});

it('keeps waiting while the old kernel is still running', function () {
    $s = fakeServices();
    $s['shell']->script('uname -r', CommandResult::success('6.1.0-18-amd64'));
    $run = hostRun(Host::factory()->create(), [
        'reboot_issued' => true,
        'reboot_deadline' => now()->addMinutes(10)->toIso8601String(),
    ]);

    expect(app(RebootIntoPveKernel::class)->execute($run)->type)->toBe('poll');
});

it('fails when the host does not return before the reboot deadline', function () {
    $s = fakeServices();
    $s['shell']->script('uname -r', CommandResult::success('6.1.0-18-amd64'));
    $run = hostRun(Host::factory()->create(), [
        'reboot_issued' => true,
        'reboot_deadline' => now()->subMinute()->toIso8601String(),
    ]);

    expect(app(RebootIntoPveKernel::class)->execute($run)->type)->toBe('fail')
        ->and($run->fresh()->context('reboot_issued'))->toBeNull(); // cleared so a retry re-issues
});

// --- CreateAutomationToken ---

it('creates and persists the automation token via pveum exactly once', function () {
    $s = fakeServices();
    $s['shell']->script('pveum user token add', CommandResult::success(
        json_encode(['full-tokenid' => 'automation@pve!clupilot', 'value' => 'tok-secret-123'])
    ));
    $host = Host::factory()->create();
    $run = hostRun($host);

    expect(app(CreateAutomationToken::class)->execute($run)->type)->toBe('advance');
    $host->refresh();
    expect($host->api_token_ref)->toContain('automation@pve!clupilot')
        ->and($host->api_token_ref)->toContain('tok-secret-123');

    $tokenCalls = fn () => count(array_filter($s['shell']->recorded(), fn ($c) => str_contains($c, 'pveum user token add')));
    expect($tokenCalls())->toBe(1);

    // Re-run: idempotent short-circuit, no second token creation.
    app(CreateAutomationToken::class)->execute($run->fresh());
    expect($tokenCalls())->toBe(1)
        ->and(RunResource::where('run_id', $run->id)->where('kind', 'pve_token')->count())->toBe(1);
});

// --- VerifyProxmoxApi ---

it('advances when the api returns nodes', function () {
    fakeServices();
    expect(app(VerifyProxmoxApi::class)->execute(hostRun(Host::factory()->create()))->type)->toBe('advance');
});

it('retries when the api returns no nodes yet', function () {
    $s = fakeServices();
    $s['pve']->nodes = [];
    expect(app(VerifyProxmoxApi::class)->execute(hostRun(Host::factory()->create()))->type)->toBe('retry');
});

// --- RegisterCapacity ---

it('registers node capacity on the host', function () {
    fakeServices();
    $host = Host::factory()->create();
    $run = hostRun($host);

    expect(app(RegisterCapacity::class)->execute($run)->type)->toBe('advance');
    $host->refresh();
    expect($host->cpu_cores)->toBe(16)
        ->and($host->total_ram_mb)->toBe(65536)
        ->and($host->total_gb)->toBe(1024)
        ->and($host->last_seen_at)->not->toBeNull();
});

// --- CompleteHostOnboarding ---

it('marks the host active on completion', function () {
    $host = Host::factory()->create(['status' => 'onboarding']);

    expect(app(CompleteHostOnboarding::class)->execute(hostRun($host))->type)->toBe('advance')
        ->and($host->fresh()->status)->toBe('active');
});

it('gives the host a name that points at the tunnel, not at its public address', function () {
    $s = fakeServices();
    $host = App\Models\Host::factory()->active()->create([
        'datacenter' => 'fsn',
        'wg_ip' => '10.66.0.11',
        'public_ip' => '203.0.113.11',
        'dns_name' => null,
    ]);
    $run = App\Models\ProvisioningRun::factory()->forHost($host)->create(["pipeline" => "host"]);

    $result = (new App\Provisioning\Steps\Host\RegisterHostDns($s['dns']))->execute($run);

    expect($result->type)->toBe("advance")
        ->and($host->fresh()->dns_name)->toBe('fsn-01');

    // The management address: publishing a Proxmox host's public IP would hand
    // every scanner a target.
    expect($s['dns']->values['fsn-01.node.clupilot.com'])->toBe('10.66.0.11')
        ->and($s['dns']->values['fsn-01.node.clupilot.com'])->not->toBe('203.0.113.11');
});

it('numbers hosts per datacenter and never reuses a number', function () {
    $s = fakeServices();
    $names = [];

    foreach (['fsn', 'fsn', 'hel'] as $i => $dc) {
        $host = App\Models\Host::factory()->active()->create([
            'datacenter' => $dc, 'wg_ip' => '10.66.0.'.(20 + $i), 'dns_name' => null,
        ]);
        $run = App\Models\ProvisioningRun::factory()->forHost($host)->create(["pipeline" => "host"]);
        (new App\Provisioning\Steps\Host\RegisterHostDns($s['dns']))->execute($run);
        $names[] = $host->fresh()->dns_name;
    }

    expect($names)->toBe(['fsn-01', 'fsn-02', 'hel-01']);

    // Removing the HIGHEST one is the dangerous case: deriving the number from
    // live hosts would hand fsn-02 straight back out, and a cached name would
    // then point at a different machine.
    App\Models\Host::query()->where('dns_name', 'fsn-02')->delete();
    $host = App\Models\Host::factory()->active()->create(['datacenter' => 'fsn', 'wg_ip' => '10.66.0.30', 'dns_name' => null]);
    $run = App\Models\ProvisioningRun::factory()->forHost($host)->create(["pipeline" => "host"]);
    (new App\Provisioning\Steps\Host\RegisterHostDns($s['dns']))->execute($run);

    expect($host->fresh()->dns_name)->toBe('fsn-03');
});

it('does not strand an onboarding because DNS was unreachable', function () {
    $s = fakeServices();
    $s['dns']->failUpsert = true;
    $host = App\Models\Host::factory()->active()->create(['datacenter' => 'fsn', 'wg_ip' => '10.66.0.40', 'dns_name' => null]);
    $run = App\Models\ProvisioningRun::factory()->forHost($host)->create(["pipeline" => "host"]);

    // A name is convenience; the host works without it.
    expect((new App\Provisioning\Steps\Host\RegisterHostDns($s['dns']))->execute($run)->type)->toBe("advance");
    expect($run->events()->where('step', 'register_host_dns')->exists())->toBeTrue();
});

it('takes the host DNS record with it when the host is purged', function () {
    $s = fakeServices();
    app()->instance(App\Services\Dns\HetznerDnsClient::class, $s['dns']);

    $host = App\Models\Host::factory()->active()->create([
        'datacenter' => 'fsn', 'wg_ip' => '10.66.0.50', 'dns_name' => null,
    ]);
    $run = App\Models\ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);
    (new App\Provisioning\Steps\Host\RegisterHostDns($s['dns']))->execute($run);

    $fqdn = $host->fresh()->dns_name.'.node.clupilot.com';
    expect($s['dns']->records)->toHaveKey($fqdn);

    (new App\Provisioning\Jobs\PurgeHost($host->uuid))->handle();

    // Otherwise the machine's management address stays published for good, with
    // the identifier needed to remove it deleted along with the row.
    expect($s['dns']->records)->not->toHaveKey($fqdn);
});

it('keeps the host until its DNS record is really gone', function () {
    $s = fakeServices();
    app()->instance(App\Services\Dns\HetznerDnsClient::class, $s['dns']);

    $host = App\Models\Host::factory()->active()->create([
        'datacenter' => 'hel', 'wg_ip' => '10.66.0.60', 'dns_name' => null,
    ]);
    $run = App\Models\ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);
    (new App\Provisioning\Steps\Host\RegisterHostDns($s['dns']))->execute($run);

    $s['dns']->failDelete = true;

    // The row carries the only reference to that record — deleting it anyway
    // would publish the machine's management address for good.
    expect(fn () => (new App\Provisioning\Jobs\PurgeHost($host->uuid))->handle())
        ->toThrow(RuntimeException::class);

    expect(App\Models\Host::query()->whereKey($host->id)->exists())->toBeTrue();

    // Once DNS is back, the purge completes.
    $s['dns']->failDelete = false;
    (new App\Provisioning\Jobs\PurgeHost($host->uuid))->handle();

    expect(App\Models\Host::query()->whereKey($host->id)->exists())->toBeFalse()
        ->and($s['dns']->records)->toBe([]);
});

it('builds a valid DNS label even from an awkward datacenter code', function () {
    $s = fakeServices();
    // Inserted directly: the console now rejects this shape, but rows created
    // before the rule was tightened still exist.
    App\Models\Datacenter::factory()->create(['code' => 'eu_west', 'name' => 'EU West']);
    $host = App\Models\Host::factory()->active()->create([
        'datacenter' => 'eu_west', 'wg_ip' => '10.66.0.70', 'dns_name' => null,
    ]);
    $run = App\Models\ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);

    (new App\Provisioning\Steps\Host\RegisterHostDns($s['dns']))->execute($run);

    $name = $host->fresh()->dns_name;
    expect($name)->toBe('eu-west-01')
        ->and($name)->toMatch('/^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/');
});

it('does not hand the same name to two datacenter codes that look alike', function () {
    $s = fakeServices();
    App\Models\Datacenter::factory()->create(['code' => 'eu_west', 'name' => 'EU West (alt)']);
    App\Models\Datacenter::factory()->create(['code' => 'eu-west', 'name' => 'EU West']);

    $names = [];
    foreach (['eu_west', 'eu-west'] as $i => $dc) {
        $host = App\Models\Host::factory()->active()->create([
            'datacenter' => $dc, 'wg_ip' => '10.66.0.'.(80 + $i), 'dns_name' => null,
        ]);
        $run = App\Models\ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);
        (new App\Provisioning\Steps\Host\RegisterHostDns($s['dns']))->execute($run);
        $names[] = $host->fresh()->dns_name;
    }

    // Separate counters per raw code would have produced eu-west-01 twice and
    // failed the unique constraint mid-onboarding.
    expect($names)->toBe(['eu-west-01', 'eu-west-02']);
});