CluPilotCloud/tests/Feature/Provisioning/HostStepsTest.php

<?php

use App\Models\Datacenter;
use App\Models\Host;
use App\Models\ProvisioningRun;
use App\Models\RunResource;
use App\Provisioning\Jobs\PurgeHost;
use App\Provisioning\Steps\Host\CompleteHostOnboarding;
use App\Provisioning\Steps\Host\ConfigureProxmox;
use App\Provisioning\Steps\Host\ConfigureWireguard;
use App\Provisioning\Steps\Host\CreateAutomationToken;
use App\Provisioning\Steps\Host\EstablishSshTrust;
use App\Provisioning\Steps\Host\InstallProxmoxVe;
use App\Provisioning\Steps\Host\PrepareBaseSystem;
use App\Provisioning\Steps\Host\RebootIntoPveKernel;
use App\Provisioning\Steps\Host\RegisterCapacity;
use App\Provisioning\Steps\Host\RegisterHostDns;
use App\Provisioning\Steps\Host\SecureHostFirewall;
use App\Provisioning\Steps\Host\ValidateHostInput;
use App\Provisioning\Steps\Host\VerifyProxmoxApi;
use App\Services\Ssh\CommandResult;
use Illuminate\Support\Facades\Crypt;

function hostRun(Host $host, array $context = []): ProvisioningRun
{
    return ProvisioningRun::factory()->forHost($host)->create(['context' => $context]);
}

// --- ValidateHostInput ---

it('advances on valid host input and marks the host onboarding', function () {
    $host = Host::factory()->create();
    $run = hostRun($host, ['root_password' => Crypt::encryptString('pw')]);

    expect(app(ValidateHostInput::class)->execute($run)->type)->toBe('advance')
        ->and($host->fresh()->status)->toBe('onboarding');
});

it('fails validation when the root password is missing', function () {
    $run = hostRun(Host::factory()->create(), []);

    expect(app(ValidateHostInput::class)->execute($run)->type)->toBe('fail');
});

// --- EstablishSshTrust ---

it('establishes ssh trust, pins the host key and scrubs the password', function () {
    $s = fakeServices();
    $host = Host::factory()->create(['public_ip' => '203.0.113.5']);
    $run = hostRun($host, ['root_password' => Crypt::encryptString('rootpw')]);

    expect(app(EstablishSshTrust::class)->execute($run)->type)->toBe('advance')
        ->and($host->fresh()->ssh_host_key)->not->toBeNull()
        ->and($run->fresh()->context('root_password'))->toBeNull()
        ->and($s['shell']->connectionsWith('password'))->toHaveCount(1)
        ->and($s['shell']->connectionsWith('key'))->toHaveCount(1);
});

it('skips ssh trust once the host key is already pinned', function () {
    $s = fakeServices();
    $host = Host::factory()->create(['ssh_host_key' => 'SHA256:pinned']);
    $run = hostRun($host, []);

    expect(app(EstablishSshTrust::class)->execute($run)->type)->toBe('advance')
        ->and($s['shell']->connectionsWith('password'))->toHaveCount(0);
});

it('throws when the host is unreachable (runner turns it into a retry)', function () {
    $s = fakeServices();
    $s['shell']->failConnect = true;
    $run = hostRun(Host::factory()->create(), ['root_password' => Crypt::encryptString('pw')]);

    expect(fn () => app(EstablishSshTrust::class)->execute($run))->toThrow(RuntimeException::class);
});

// --- HostStep::keyLogin address preference ---
//
// Mutation boundary: every step after ConfigureWireguard (and every later
// maintenance run) MUST reach the host over the tunnel once wg_ip is set —
// closing SSH to the world without this would strand the host permanently.
// ConfigureProxmox is used as a plain probe: keyLogin then two commands, no
// other branching that could obscure which address was actually dialled.

it('uses the WireGuard address for ssh once the tunnel exists', function () {
    $s = fakeServices();
    $host = Host::factory()->create(['public_ip' => '203.0.113.20', 'wg_ip' => '10.66.0.20']);
    $run = hostRun($host);

    app(ConfigureProxmox::class)->execute($run);

    $connections = $s['shell']->connectionsWith('key');
    expect($connections)->not->toBeEmpty();
    foreach ($connections as $connection) {
        expect($connection[1])->toBe('10.66.0.20');
    }
});

it('falls back to the public IP before the tunnel exists', function () {
    $s = fakeServices();
    $host = Host::factory()->create(['public_ip' => '203.0.113.21', 'wg_ip' => null]);
    $run = hostRun($host);

    app(ConfigureProxmox::class)->execute($run);

    $connections = $s['shell']->connectionsWith('key');
    expect($connections)->not->toBeEmpty();
    foreach ($connections as $connection) {
        expect($connection[1])->toBe('203.0.113.21');
    }
});

// --- PrepareBaseSystem ---

it('prepares the base system', function () {
    $s = fakeServices();
    $run = hostRun(Host::factory()->create());

    expect(app(PrepareBaseSystem::class)->execute($run)->type)->toBe('advance')
        ->and($s['shell']->ran('apt-get update'))->toBeTrue();
});

it('retries base preparation on an apt failure', function () {
    $s = fakeServices();
    $s['shell']->script('apt-get update', CommandResult::failure(1, 'network down'));
    $run = hostRun(Host::factory()->create());

    expect(app(PrepareBaseSystem::class)->execute($run)->type)->toBe('retry');
});

// --- ConfigureWireguard ---

it('configures wireguard and registers the peer exactly once', function () {
    $s = fakeServices();
    $s['shell']->script('wg pubkey', CommandResult::success('HOSTPUB='));
    $host = Host::factory()->create();
    $run = hostRun($host);

    expect(app(ConfigureWireguard::class)->execute($run)->type)->toBe('advance');
    $host->refresh();
    expect($host->wg_ip)->not->toBeNull()
        ->and($host->wg_pubkey)->toBe('HOSTPUB=')
        ->and($s['hub']->peerIps())->toHaveCount(1);

    // Re-run: idempotent short-circuit, still one peer resource.
    app(ConfigureWireguard::class)->execute($run->fresh());
    expect(RunResource::where('run_id', $run->id)->where('kind', 'wg_peer')->count())->toBe(1);
});

it('retries wireguard while the handshake is not up', function () {
    $s = fakeServices();
    $s['shell']->script('wg pubkey', CommandResult::success('HOSTPUB='));
    $s['shell']->script('ping', CommandResult::failure(1));
    $run = hostRun(Host::factory()->create());

    expect(app(ConfigureWireguard::class)->execute($run)->type)->toBe('retry');
});

// --- InstallProxmoxVe ---

it('skips the install when proxmox-ve is already present', function () {
    $s = fakeServices();
    $s['shell']->script('dpkg -l proxmox-ve', CommandResult::success('ii  proxmox-ve'));
    $run = hostRun(Host::factory()->create());

    expect(app(InstallProxmoxVe::class)->execute($run)->type)->toBe('advance')
        ->and($s['shell']->ran('apt-get -y install proxmox-ve'))->toBeFalse();
});

it('installs proxmox-ve when it is absent', function () {
    $s = fakeServices();
    $s['shell']->script('dpkg -l proxmox-ve', CommandResult::failure(1));
    $run = hostRun(Host::factory()->create());

    expect(app(InstallProxmoxVe::class)->execute($run)->type)->toBe('advance')
        ->and($s['shell']->ran('apt-get -y install proxmox-ve'))->toBeTrue();
});

// --- RebootIntoPveKernel ---

it('issues the reboot on first execution', function () {
    $s = fakeServices();
    $run = hostRun(Host::factory()->create());

    expect(app(RebootIntoPveKernel::class)->execute($run)->type)->toBe('poll')
        ->and($run->fresh()->context('reboot_issued'))->toBeTrue()
        ->and($s['shell']->ran('reboot'))->toBeTrue();
});

it('advances once the proxmox kernel is up', function () {
    $s = fakeServices();
    $s['shell']->script('uname -r', CommandResult::success('6.8.12-4-pve'));
    $run = hostRun(Host::factory()->create(), [
        'reboot_issued' => true,
        'reboot_deadline' => now()->addMinutes(10)->toIso8601String(),
    ]);

    expect(app(RebootIntoPveKernel::class)->execute($run)->type)->toBe('advance');
});

it('keeps waiting while the old kernel is still running', function () {
    $s = fakeServices();
    $s['shell']->script('uname -r', CommandResult::success('6.1.0-18-amd64'));
    $run = hostRun(Host::factory()->create(), [
        'reboot_issued' => true,
        'reboot_deadline' => now()->addMinutes(10)->toIso8601String(),
    ]);

    expect(app(RebootIntoPveKernel::class)->execute($run)->type)->toBe('poll');
});

it('fails when the host does not return before the reboot deadline', function () {
    $s = fakeServices();
    $s['shell']->script('uname -r', CommandResult::success('6.1.0-18-amd64'));
    $run = hostRun(Host::factory()->create(), [
        'reboot_issued' => true,
        'reboot_deadline' => now()->subMinute()->toIso8601String(),
    ]);

    expect(app(RebootIntoPveKernel::class)->execute($run)->type)->toBe('fail')
        ->and($run->fresh()->context('reboot_issued'))->toBeNull(); // cleared so a retry re-issues
});

// --- CreateAutomationToken ---

it('creates and persists the automation token via pveum exactly once', function () {
    $s = fakeServices();
    $s['shell']->script('pveum user token add', CommandResult::success(
        json_encode(['full-tokenid' => 'automation@pve!clupilot', 'value' => 'tok-secret-123'])
    ));
    $host = Host::factory()->create();
    $run = hostRun($host);

    expect(app(CreateAutomationToken::class)->execute($run)->type)->toBe('advance');
    $host->refresh();
    expect($host->api_token_ref)->toContain('automation@pve!clupilot')
        ->and($host->api_token_ref)->toContain('tok-secret-123');

    $tokenCalls = fn () => count(array_filter($s['shell']->recorded(), fn ($c) => str_contains($c, 'pveum user token add')));
    expect($tokenCalls())->toBe(1);

    // Re-run: idempotent short-circuit, no second token creation.
    app(CreateAutomationToken::class)->execute($run->fresh());
    expect($tokenCalls())->toBe(1)
        ->and(RunResource::where('run_id', $run->id)->where('kind', 'pve_token')->count())->toBe(1);
});

// --- VerifyProxmoxApi ---

it('advances when the api returns nodes', function () {
    fakeServices();
    expect(app(VerifyProxmoxApi::class)->execute(hostRun(Host::factory()->create()))->type)->toBe('advance');
});

it('retries when the api returns no nodes yet', function () {
    $s = fakeServices();
    $s['pve']->nodes = [];
    expect(app(VerifyProxmoxApi::class)->execute(hostRun(Host::factory()->create()))->type)->toBe('retry');
});

// --- RegisterCapacity ---

it('registers node capacity on the host', function () {
    fakeServices();
    $host = Host::factory()->create();
    $run = hostRun($host);

    expect(app(RegisterCapacity::class)->execute($run)->type)->toBe('advance');
    $host->refresh();
    expect($host->cpu_cores)->toBe(16)
        ->and($host->total_ram_mb)->toBe(65536)
        ->and($host->total_gb)->toBe(1024)
        ->and($host->last_seen_at)->not->toBeNull();
});

// --- SecureHostFirewall ---

it('places SecureHostFirewall after every other host step and before completion', function () {
    $pipeline = config('provisioning.pipelines.host');

    $secureIndex = array_search(SecureHostFirewall::class, $pipeline, true);
    $capacityIndex = array_search(RegisterCapacity::class, $pipeline, true);
    $completeIndex = array_search(CompleteHostOnboarding::class, $pipeline, true);

    expect($secureIndex)->not->toBeFalse()
        ->and($secureIndex)->toBe($capacityIndex + 1)
        ->and($secureIndex)->toBe($completeIndex - 1);
});

it('refuses to apply the firewall when the tunnel is not confirmed up from the host', function () {
    // Mutation boundary: a host whose tunnel does not verify must come away
    // completely untouched — retrying is not enough on its own, nothing may
    // have been written either.
    $s = fakeServices();
    $s['shell']->script('ping', CommandResult::failure(1));
    $host = Host::factory()->active()->create();
    $run = hostRun($host);

    expect(app(SecureHostFirewall::class)->execute($run)->type)->toBe('retry');

    expect($s['shell']->files())->not->toHaveKey('/etc/nftables.conf')
        ->and($s['shell']->ran('apt-get install -y nftables'))->toBeFalse()
        ->and(RunResource::where('run_id', $run->id)->where('kind', 'host_firewall')->exists())->toBeFalse();
});

it('applies the firewall once the tunnel is confirmed up, over the tunnel itself', function () {
    $s = fakeServices();
    $host = Host::factory()->active()->create(['public_ip' => '203.0.113.30']);
    $run = hostRun($host);

    expect(app(SecureHostFirewall::class)->execute($run)->type)->toBe('advance');

    // Connected over the tunnel, not the public address.
    foreach ($s['shell']->connectionsWith('key') as $connection) {
        expect($connection[1])->toBe($host->wg_ip);
    }

    $wgSubnet = (string) config('provisioning.wireguard.subnet');
    $config = $s['shell']->files()['/etc/nftables.conf'];
    expect($config)->toContain('policy drop')
        ->and($config)->toContain('tcp dport { 80, 443 } accept')
        ->and($config)->toContain('ip saddr '.$wgSubnet.' tcp dport { 22, 8006 } accept')
        ->and($config)->toContain('ct state established,related accept');

    expect($s['shell']->files())->toHaveKey('/usr/local/sbin/clupilot-emergency-open-firewall.sh')
        ->and($s['shell']->ran('chmod 700'))->toBeTrue()
        ->and($s['shell']->ran('/usr/local/sbin/clupilot-emergency-open-firewall.sh'))->toBeTrue()
        ->and($s['shell']->ran('systemctl enable --now nftables'))->toBeTrue()
        ->and(RunResource::where('run_id', $run->id)->where('kind', 'host_firewall')->count())->toBe(1);
});

it("keeps the WireGuard interface's established connections working (never a bare drop-all)", function () {
    $s = fakeServices();
    $run = hostRun(Host::factory()->active()->create());

    app(SecureHostFirewall::class)->execute($run);

    $config = $s['shell']->files()['/etc/nftables.conf'];
    expect($config)->toContain('ct state established,related accept')
        ->and($config)->toContain('iif "lo" accept');
});

it('never reapplies the firewall on a re-run', function () {
    $s = fakeServices();
    $host = Host::factory()->active()->create();
    $run = hostRun($host);

    app(SecureHostFirewall::class)->execute($run);
    app(SecureHostFirewall::class)->execute($run->fresh());

    $installCalls = fn () => count(array_filter(
        $s['shell']->recorded(),
        fn ($c) => str_contains($c, 'apt-get install -y nftables'),
    ));

    expect($installCalls())->toBe(1)
        ->and(RunResource::where('run_id', $run->id)->where('kind', 'host_firewall')->count())->toBe(1);
});

// --- CompleteHostOnboarding ---

it('marks the host active on completion', function () {
    $host = Host::factory()->create(['status' => 'onboarding']);

    expect(app(CompleteHostOnboarding::class)->execute(hostRun($host))->type)->toBe('advance')
        ->and($host->fresh()->status)->toBe('active');
});

it('gives the host a name that points at the tunnel, not at its public address', function () {
    $s = fakeServices();
    $host = Host::factory()->active()->create([
        'datacenter' => 'fsn',
        'wg_ip' => '10.66.0.11',
        'public_ip' => '203.0.113.11',
        'dns_name' => null,
    ]);
    $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);

    $result = (new RegisterHostDns($s['hostDns']))->execute($run);

    expect($result->type)->toBe('advance')
        ->and($host->fresh()->dns_name)->toBe('fsn-01');

    // The management address: publishing a Proxmox host's public IP would hand
    // every scanner a target. Written to the internal hostsdir, not upserted
    // anywhere via the (public) Hetzner client — see the mutation test below.
    expect($s['hostDns']->fqdns['fsn-01'])->toBe('fsn-01.node.clupilot.com')
        ->and($s['hostDns']->ips['fsn-01'])->toBe('10.66.0.11')
        ->and($s['hostDns']->ips['fsn-01'])->not->toBe('203.0.113.11');
});

it('never reaches the public Hetzner DNS client for a host name', function () {
    // Mutation boundary: if RegisterHostDns is ever changed to also (or
    // instead) call the public client, this must fail. Verified by hand —
    // temporarily adding a `$this->publicDns->upsertRecord(...)` call to the
    // step and confirming this test goes red before reverting it.
    $s = fakeServices();
    $host = Host::factory()->active()->create([
        'datacenter' => 'fsn', 'wg_ip' => '10.66.0.12', 'dns_name' => null,
    ]);
    $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);

    (new RegisterHostDns($s['hostDns']))->execute($run);

    expect($s['dns']->records)->toBe([])
        ->and($s['dns']->values)->toBe([]);
});

it('numbers hosts per datacenter and never reuses a number', function () {
    $s = fakeServices();
    $names = [];

    foreach (['fsn', 'fsn', 'hel'] as $i => $dc) {
        $host = Host::factory()->active()->create([
            'datacenter' => $dc, 'wg_ip' => '10.66.0.'.(20 + $i), 'dns_name' => null,
        ]);
        $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);
        (new RegisterHostDns($s['hostDns']))->execute($run);
        $names[] = $host->fresh()->dns_name;
    }

    expect($names)->toBe(['fsn-01', 'fsn-02', 'hel-01']);

    // Removing the HIGHEST one is the dangerous case: deriving the number from
    // live hosts would hand fsn-02 straight back out, and a cached name would
    // then point at a different machine.
    Host::query()->where('dns_name', 'fsn-02')->delete();
    $host = Host::factory()->active()->create(['datacenter' => 'fsn', 'wg_ip' => '10.66.0.30', 'dns_name' => null]);
    $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);
    (new RegisterHostDns($s['hostDns']))->execute($run);

    expect($host->fresh()->dns_name)->toBe('fsn-03');
});

it('retries a failed DNS registration before giving up on the name', function () {
    $s = fakeServices();
    $s['hostDns']->failWrite = true;
    $host = Host::factory()->active()->create(['datacenter' => 'fsn', 'wg_ip' => '10.66.0.40', 'dns_name' => null]);
    $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host', 'attempt' => 0]);

    $step = new RegisterHostDns($s['hostDns']);

    // The failure may be a lost response to a request that did create the
    // record — advancing straight away would orphan it.
    expect($step->execute($run)->type)->toBe('retry');

    // Out of attempts, the host still finishes: a name is convenience.
    $run->update(['attempt' => 5]);
    expect($step->execute($run)->type)->toBe('advance')
        ->and($run->events()->where('step', 'register_host_dns')->exists())->toBeTrue();
});

it("takes the host's internal DNS entry with it when the host is purged", function () {
    $s = fakeServices();

    $host = Host::factory()->active()->create([
        'datacenter' => 'fsn', 'wg_ip' => '10.66.0.50', 'dns_name' => null,
    ]);
    $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);
    (new RegisterHostDns($s['hostDns']))->execute($run);

    $name = $host->fresh()->dns_name;
    expect($s['hostDns']->ips)->toHaveKey($name);

    (new PurgeHost($host->uuid))->handle();

    // Otherwise the machine's management address stays in the hostsdir for
    // good, with the name needed to remove it deleted along with the row.
    expect($s['hostDns']->ips)->not->toHaveKey($name);
});

it('keeps the host until its internal DNS entry is really gone', function () {
    $s = fakeServices();

    $host = Host::factory()->active()->create([
        'datacenter' => 'hel', 'wg_ip' => '10.66.0.60', 'dns_name' => null,
    ]);
    $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);
    (new RegisterHostDns($s['hostDns']))->execute($run);

    $s['hostDns']->failRemove = true;

    // The row carries the only name that maps to that entry — deleting it
    // anyway would leave the machine's management address in the hostsdir
    // for good.
    expect(fn () => (new PurgeHost($host->uuid))->handle())
        ->toThrow(RuntimeException::class);

    expect(Host::query()->whereKey($host->id)->exists())->toBeTrue();

    // Once the hostsdir write is back, the purge completes.
    $s['hostDns']->failRemove = false;
    (new PurgeHost($host->uuid))->handle();

    expect(Host::query()->whereKey($host->id)->exists())->toBeFalse()
        ->and($s['hostDns']->ips)->toBe([]);
});

it('still removes a legacy public Hetzner record when a pre-fix host is purged', function () {
    // Hosts onboarded before this change may still carry a public record id.
    // PurgeHost has always cleaned that up on removal — this keeps working
    // for those, independent of clupilot:prune-host-dns, which only handles
    // hosts that are NOT being deleted.
    $s = fakeServices();
    $recordId = $s['dns']->upsertRecord('fsn-legacy.node.clupilot.com', 'A', '10.66.0.55');
    $host = Host::factory()->active()->create([
        'datacenter' => 'fsn', 'wg_ip' => '10.66.0.55',
        'dns_name' => 'fsn-legacy', 'dns_record_id' => $recordId,
    ]);

    expect($s['dns']->records)->toHaveKey('fsn-legacy.node.clupilot.com');

    (new PurgeHost($host->uuid))->handle();

    expect($s['dns']->records)->not->toHaveKey('fsn-legacy.node.clupilot.com');
});

it('builds a valid DNS label even from an awkward datacenter code', function () {
    $s = fakeServices();
    // Inserted directly: the console now rejects this shape, but rows created
    // before the rule was tightened still exist.
    Datacenter::factory()->create(['code' => 'eu_west', 'name' => 'EU West']);
    $host = Host::factory()->active()->create([
        'datacenter' => 'eu_west', 'wg_ip' => '10.66.0.70', 'dns_name' => null,
    ]);
    $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);

    (new RegisterHostDns($s['hostDns']))->execute($run);

    $name = $host->fresh()->dns_name;
    expect($name)->toBe('eu-west-01')
        ->and($name)->toMatch('/^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/');
});

it('does not hand the same name to two datacenter codes that look alike', function () {
    $s = fakeServices();
    Datacenter::factory()->create(['code' => 'eu_west', 'name' => 'EU West (alt)']);
    Datacenter::factory()->create(['code' => 'eu-west', 'name' => 'EU West']);

    $names = [];
    foreach (['eu_west', 'eu-west'] as $i => $dc) {
        $host = Host::factory()->active()->create([
            'datacenter' => $dc, 'wg_ip' => '10.66.0.'.(80 + $i), 'dns_name' => null,
        ]);
        $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);
        (new RegisterHostDns($s['hostDns']))->execute($run);
        $names[] = $host->fresh()->dns_name;
    }

    // Separate counters per raw code would have produced eu-west-01 twice and
    // failed the unique constraint mid-onboarding.
    expect($names)->toBe(['eu-west-01', 'eu-west-02']);
});

it('sees an internal DNS entry registered while the purge was waiting', function () {
    $s = fakeServices();

    $host = Host::factory()->active()->create([
        'datacenter' => 'fsn', 'wg_ip' => '10.66.0.90', 'dns_name' => null,
    ]);
    $purge = new PurgeHost($host->uuid);

    // Registration finishes while the purge is blocked on the run lock: the
    // copy it loaded first still says there is no entry to remove.
    $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);
    (new RegisterHostDns($s['hostDns']))->execute($run);
    $name = $host->fresh()->dns_name;

    $purge->handle();

    expect($s['hostDns']->ips)->not->toHaveKey($name)
        ->and(Host::query()->whereKey($host->id)->exists())->toBeFalse();
});