CluPilotCloud/tests/Feature/Provisioning/HostStepsTest.php

597 lines
24 KiB
PHP

<?php
use App\Models\Datacenter;
use App\Models\Host;
use App\Models\Operator;
use App\Models\ProvisioningRun;
use App\Models\RunResource;
use App\Provisioning\Jobs\PurgeHost;
use App\Provisioning\Steps\Host\CompleteHostOnboarding;
use App\Provisioning\Steps\Host\ConfigureProxmox;
use App\Provisioning\Steps\Host\ConfigureWireguard;
use App\Provisioning\Steps\Host\CreateAutomationToken;
use App\Provisioning\Steps\Host\EstablishSshTrust;
use App\Provisioning\Steps\Host\InstallProxmoxVe;
use App\Provisioning\Steps\Host\PrepareBaseSystem;
use App\Provisioning\Steps\Host\RebootIntoPveKernel;
use App\Provisioning\Steps\Host\RegisterCapacity;
use App\Provisioning\Steps\Host\RegisterHostDns;
use App\Provisioning\Steps\Host\SecureHostFirewall;
use App\Provisioning\Steps\Host\ValidateHostInput;
use App\Provisioning\Steps\Host\VerifyProxmoxApi;
use App\Services\Secrets\SecretVault;
use App\Services\Ssh\CommandResult;
use Illuminate\Support\Facades\Crypt;
function hostRun(Host $host, array $context = []): ProvisioningRun
{
return ProvisioningRun::factory()->forHost($host)->create(['context' => $context]);
}
// --- ValidateHostInput ---
it('advances on valid host input and marks the host onboarding', function () {
$host = Host::factory()->create();
$run = hostRun($host, ['root_password' => Crypt::encryptString('pw')]);
expect(app(ValidateHostInput::class)->execute($run)->type)->toBe('advance')
->and($host->fresh()->status)->toBe('onboarding');
});
it('fails validation when the root password is missing', function () {
$run = hostRun(Host::factory()->create(), []);
expect(app(ValidateHostInput::class)->execute($run)->type)->toBe('fail');
});
// --- EstablishSshTrust ---
it('establishes ssh trust, pins the host key and scrubs the password', function () {
$s = fakeServices();
$host = Host::factory()->create(['public_ip' => '203.0.113.5']);
$run = hostRun($host, ['root_password' => Crypt::encryptString('rootpw')]);
expect(app(EstablishSshTrust::class)->execute($run)->type)->toBe('advance')
->and($host->fresh()->ssh_host_key)->not->toBeNull()
->and($run->fresh()->context('root_password'))->toBeNull()
->and($s['shell']->connectionsWith('password'))->toHaveCount(1)
->and($s['shell']->connectionsWith('key'))->toHaveCount(1);
});
it('skips ssh trust once the host key is already pinned', function () {
$s = fakeServices();
$host = Host::factory()->create(['ssh_host_key' => 'SHA256:pinned']);
$run = hostRun($host, []);
expect(app(EstablishSshTrust::class)->execute($run)->type)->toBe('advance')
->and($s['shell']->connectionsWith('password'))->toHaveCount(0);
});
it('throws when the host is unreachable (runner turns it into a retry)', function () {
$s = fakeServices();
$s['shell']->failConnect = true;
$run = hostRun(Host::factory()->create(), ['root_password' => Crypt::encryptString('pw')]);
expect(fn () => app(EstablishSshTrust::class)->execute($run))->toThrow(RuntimeException::class);
});
// --- HostStep::keyLogin address preference ---
//
// Mutation boundary: every step after ConfigureWireguard (and every later
// maintenance run) MUST reach the host over the tunnel once wg_ip is set —
// closing SSH to the world without this would strand the host permanently.
// ConfigureProxmox is used as a plain probe: keyLogin then two commands, no
// other branching that could obscure which address was actually dialled.
it('uses the WireGuard address for ssh once the tunnel exists', function () {
$s = fakeServices();
$host = Host::factory()->create(['public_ip' => '203.0.113.20', 'wg_ip' => '10.66.0.20']);
$run = hostRun($host);
app(ConfigureProxmox::class)->execute($run);
$connections = $s['shell']->connectionsWith('key');
expect($connections)->not->toBeEmpty();
foreach ($connections as $connection) {
expect($connection[1])->toBe('10.66.0.20');
}
});
it('falls back to the public IP before the tunnel exists', function () {
$s = fakeServices();
$host = Host::factory()->create(['public_ip' => '203.0.113.21', 'wg_ip' => null]);
$run = hostRun($host);
app(ConfigureProxmox::class)->execute($run);
$connections = $s['shell']->connectionsWith('key');
expect($connections)->not->toBeEmpty();
foreach ($connections as $connection) {
expect($connection[1])->toBe('203.0.113.21');
}
});
it('prefers the vault-stored SSH key over CLUPILOT_SSH_PRIVATE_KEY', function () {
// Mutation boundary (Part B): the SSH identity is the most sensitive
// value in the system — it opens every host CluPilot manages. Before
// this, HostStep::keyLogin() read config('provisioning.ssh.private_key')
// directly, so a key stored in the console was encrypted and kept, but
// never once reached an actual connection.
config()->set('admin_access.secrets_key', 'base64:'.base64_encode(random_bytes(32)));
config()->set('provisioning.ssh.private_key', 'env-key-should-not-be-used');
app(SecretVault::class)->put('ssh.private_key', 'vault-key-should-be-used', Operator::factory()->create());
$s = fakeServices();
$run = hostRun(Host::factory()->create());
app(ConfigureProxmox::class)->execute($run);
$connections = $s['shell']->connectionsWith('key');
expect($connections)->not->toBeEmpty();
foreach ($connections as $connection) {
expect($connection[3])->toBe('vault-key-should-be-used');
}
});
// --- PrepareBaseSystem ---
it('prepares the base system', function () {
$s = fakeServices();
$run = hostRun(Host::factory()->create());
expect(app(PrepareBaseSystem::class)->execute($run)->type)->toBe('advance')
->and($s['shell']->ran('apt-get update'))->toBeTrue();
});
it('retries base preparation on an apt failure', function () {
$s = fakeServices();
$s['shell']->script('apt-get update', CommandResult::failure(1, 'network down'));
$run = hostRun(Host::factory()->create());
expect(app(PrepareBaseSystem::class)->execute($run)->type)->toBe('retry');
});
// --- ConfigureWireguard ---
it('configures wireguard and registers the peer exactly once', function () {
$s = fakeServices();
$s['shell']->script('wg pubkey', CommandResult::success('HOSTPUB='));
$host = Host::factory()->create();
$run = hostRun($host);
expect(app(ConfigureWireguard::class)->execute($run)->type)->toBe('advance');
$host->refresh();
expect($host->wg_ip)->not->toBeNull()
->and($host->wg_pubkey)->toBe('HOSTPUB=')
->and($s['hub']->peerIps())->toHaveCount(1);
// Re-run: idempotent short-circuit, still one peer resource.
app(ConfigureWireguard::class)->execute($run->fresh());
expect(RunResource::where('run_id', $run->id)->where('kind', 'wg_peer')->count())->toBe(1);
});
it('retries wireguard while the handshake is not up', function () {
$s = fakeServices();
$s['shell']->script('wg pubkey', CommandResult::success('HOSTPUB='));
$s['shell']->script('ping', CommandResult::failure(1));
$run = hostRun(Host::factory()->create());
expect(app(ConfigureWireguard::class)->execute($run)->type)->toBe('retry');
});
// --- InstallProxmoxVe ---
it('skips the install when proxmox-ve is already present', function () {
$s = fakeServices();
$s['shell']->script('dpkg -l proxmox-ve', CommandResult::success('ii proxmox-ve'));
$run = hostRun(Host::factory()->create());
expect(app(InstallProxmoxVe::class)->execute($run)->type)->toBe('advance')
->and($s['shell']->ran('apt-get -y install proxmox-ve'))->toBeFalse();
});
it('installs proxmox-ve when it is absent', function () {
$s = fakeServices();
$s['shell']->script('dpkg -l proxmox-ve', CommandResult::failure(1));
$run = hostRun(Host::factory()->create());
expect(app(InstallProxmoxVe::class)->execute($run)->type)->toBe('advance')
->and($s['shell']->ran('apt-get -y install proxmox-ve'))->toBeTrue();
});
// --- RebootIntoPveKernel ---
it('issues the reboot on first execution', function () {
$s = fakeServices();
$run = hostRun(Host::factory()->create());
expect(app(RebootIntoPveKernel::class)->execute($run)->type)->toBe('poll')
->and($run->fresh()->context('reboot_issued'))->toBeTrue()
->and($s['shell']->ran('reboot'))->toBeTrue();
});
it('advances once the proxmox kernel is up', function () {
$s = fakeServices();
$s['shell']->script('uname -r', CommandResult::success('6.8.12-4-pve'));
$run = hostRun(Host::factory()->create(), [
'reboot_issued' => true,
'reboot_deadline' => now()->addMinutes(10)->toIso8601String(),
]);
expect(app(RebootIntoPveKernel::class)->execute($run)->type)->toBe('advance');
});
it('keeps waiting while the old kernel is still running', function () {
$s = fakeServices();
$s['shell']->script('uname -r', CommandResult::success('6.1.0-18-amd64'));
$run = hostRun(Host::factory()->create(), [
'reboot_issued' => true,
'reboot_deadline' => now()->addMinutes(10)->toIso8601String(),
]);
expect(app(RebootIntoPveKernel::class)->execute($run)->type)->toBe('poll');
});
it('fails when the host does not return before the reboot deadline', function () {
$s = fakeServices();
$s['shell']->script('uname -r', CommandResult::success('6.1.0-18-amd64'));
$run = hostRun(Host::factory()->create(), [
'reboot_issued' => true,
'reboot_deadline' => now()->subMinute()->toIso8601String(),
]);
expect(app(RebootIntoPveKernel::class)->execute($run)->type)->toBe('fail')
->and($run->fresh()->context('reboot_issued'))->toBeNull(); // cleared so a retry re-issues
});
// --- CreateAutomationToken ---
it('creates and persists the automation token via pveum exactly once', function () {
$s = fakeServices();
$s['shell']->script('pveum user token add', CommandResult::success(
json_encode(['full-tokenid' => 'automation@pve!clupilot', 'value' => 'tok-secret-123'])
));
$host = Host::factory()->create();
$run = hostRun($host);
expect(app(CreateAutomationToken::class)->execute($run)->type)->toBe('advance');
$host->refresh();
expect($host->api_token_ref)->toContain('automation@pve!clupilot')
->and($host->api_token_ref)->toContain('tok-secret-123');
$tokenCalls = fn () => count(array_filter($s['shell']->recorded(), fn ($c) => str_contains($c, 'pveum user token add')));
expect($tokenCalls())->toBe(1);
// Re-run: idempotent short-circuit, no second token creation.
app(CreateAutomationToken::class)->execute($run->fresh());
expect($tokenCalls())->toBe(1)
->and(RunResource::where('run_id', $run->id)->where('kind', 'pve_token')->count())->toBe(1);
});
// --- VerifyProxmoxApi ---
it('advances when the api returns nodes', function () {
fakeServices();
expect(app(VerifyProxmoxApi::class)->execute(hostRun(Host::factory()->create()))->type)->toBe('advance');
});
it('retries when the api returns no nodes yet', function () {
$s = fakeServices();
$s['pve']->nodes = [];
expect(app(VerifyProxmoxApi::class)->execute(hostRun(Host::factory()->create()))->type)->toBe('retry');
});
// --- RegisterCapacity ---
it('registers node capacity on the host', function () {
fakeServices();
$host = Host::factory()->create();
$run = hostRun($host);
expect(app(RegisterCapacity::class)->execute($run)->type)->toBe('advance');
$host->refresh();
expect($host->cpu_cores)->toBe(16)
->and($host->total_ram_mb)->toBe(65536)
->and($host->total_gb)->toBe(1024)
->and($host->last_seen_at)->not->toBeNull();
});
// --- SecureHostFirewall ---
it('places SecureHostFirewall after every other host step and before completion', function () {
$pipeline = config('provisioning.pipelines.host');
$secureIndex = array_search(SecureHostFirewall::class, $pipeline, true);
$capacityIndex = array_search(RegisterCapacity::class, $pipeline, true);
$completeIndex = array_search(CompleteHostOnboarding::class, $pipeline, true);
expect($secureIndex)->not->toBeFalse()
->and($secureIndex)->toBe($capacityIndex + 1)
->and($secureIndex)->toBe($completeIndex - 1);
});
it('refuses to apply the firewall when the tunnel is not confirmed up from the host', function () {
// Mutation boundary: a host whose tunnel does not verify must come away
// completely untouched — retrying is not enough on its own, nothing may
// have been written either.
$s = fakeServices();
$s['shell']->script('ping', CommandResult::failure(1));
$host = Host::factory()->active()->create();
$run = hostRun($host);
expect(app(SecureHostFirewall::class)->execute($run)->type)->toBe('retry');
expect($s['shell']->files())->not->toHaveKey('/etc/nftables.conf')
->and($s['shell']->ran('apt-get install -y nftables'))->toBeFalse()
->and(RunResource::where('run_id', $run->id)->where('kind', 'host_firewall')->exists())->toBeFalse();
});
it('applies the firewall once the tunnel is confirmed up, over the tunnel itself', function () {
$s = fakeServices();
$host = Host::factory()->active()->create(['public_ip' => '203.0.113.30']);
$run = hostRun($host);
expect(app(SecureHostFirewall::class)->execute($run)->type)->toBe('advance');
// Connected over the tunnel, not the public address.
foreach ($s['shell']->connectionsWith('key') as $connection) {
expect($connection[1])->toBe($host->wg_ip);
}
$wgSubnet = (string) config('provisioning.wireguard.subnet');
$config = $s['shell']->files()['/etc/nftables.conf'];
expect($config)->toContain('policy drop')
->and($config)->toContain('tcp dport { 80, 443 } accept')
->and($config)->toContain('ip saddr '.$wgSubnet.' tcp dport { 22, 8006 } accept')
->and($config)->toContain('ct state established,related accept');
expect($s['shell']->files())->toHaveKey('/usr/local/sbin/clupilot-emergency-open-firewall.sh')
->and($s['shell']->ran('chmod 700'))->toBeTrue()
->and($s['shell']->ran('/usr/local/sbin/clupilot-emergency-open-firewall.sh'))->toBeTrue()
->and($s['shell']->ran('systemctl enable --now nftables'))->toBeTrue()
->and(RunResource::where('run_id', $run->id)->where('kind', 'host_firewall')->count())->toBe(1);
});
it("keeps the WireGuard interface's established connections working (never a bare drop-all)", function () {
$s = fakeServices();
$run = hostRun(Host::factory()->active()->create());
app(SecureHostFirewall::class)->execute($run);
$config = $s['shell']->files()['/etc/nftables.conf'];
expect($config)->toContain('ct state established,related accept')
->and($config)->toContain('iif "lo" accept');
});
it('never reapplies the firewall on a re-run', function () {
$s = fakeServices();
$host = Host::factory()->active()->create();
$run = hostRun($host);
app(SecureHostFirewall::class)->execute($run);
app(SecureHostFirewall::class)->execute($run->fresh());
$installCalls = fn () => count(array_filter(
$s['shell']->recorded(),
fn ($c) => str_contains($c, 'apt-get install -y nftables'),
));
expect($installCalls())->toBe(1)
->and(RunResource::where('run_id', $run->id)->where('kind', 'host_firewall')->count())->toBe(1);
});
// --- CompleteHostOnboarding ---
it('marks the host active on completion', function () {
$host = Host::factory()->create(['status' => 'onboarding']);
expect(app(CompleteHostOnboarding::class)->execute(hostRun($host))->type)->toBe('advance')
->and($host->fresh()->status)->toBe('active');
});
it('gives the host a name that points at the tunnel, not at its public address', function () {
$s = fakeServices();
$host = Host::factory()->active()->create([
'datacenter' => 'fsn',
'wg_ip' => '10.66.0.11',
'public_ip' => '203.0.113.11',
'dns_name' => null,
]);
$run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);
$result = (new RegisterHostDns($s['hostDns']))->execute($run);
expect($result->type)->toBe('advance')
->and($host->fresh()->dns_name)->toBe('fsn-01');
// The management address: publishing a Proxmox host's public IP would hand
// every scanner a target. Written to the internal hostsdir, not upserted
// anywhere via the (public) Hetzner client — see the mutation test below.
expect($s['hostDns']->fqdns['fsn-01'])->toBe('fsn-01.node.clupilot.com')
->and($s['hostDns']->ips['fsn-01'])->toBe('10.66.0.11')
->and($s['hostDns']->ips['fsn-01'])->not->toBe('203.0.113.11');
});
it('never reaches the public Hetzner DNS client for a host name', function () {
// Mutation boundary: if RegisterHostDns is ever changed to also (or
// instead) call the public client, this must fail. Verified by hand —
// temporarily adding a `$this->publicDns->upsertRecord(...)` call to the
// step and confirming this test goes red before reverting it.
$s = fakeServices();
$host = Host::factory()->active()->create([
'datacenter' => 'fsn', 'wg_ip' => '10.66.0.12', 'dns_name' => null,
]);
$run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);
(new RegisterHostDns($s['hostDns']))->execute($run);
expect($s['dns']->records)->toBe([])
->and($s['dns']->values)->toBe([]);
});
it('numbers hosts per datacenter and never reuses a number', function () {
$s = fakeServices();
$names = [];
foreach (['fsn', 'fsn', 'hel'] as $i => $dc) {
$host = Host::factory()->active()->create([
'datacenter' => $dc, 'wg_ip' => '10.66.0.'.(20 + $i), 'dns_name' => null,
]);
$run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);
(new RegisterHostDns($s['hostDns']))->execute($run);
$names[] = $host->fresh()->dns_name;
}
expect($names)->toBe(['fsn-01', 'fsn-02', 'hel-01']);
// Removing the HIGHEST one is the dangerous case: deriving the number from
// live hosts would hand fsn-02 straight back out, and a cached name would
// then point at a different machine.
Host::query()->where('dns_name', 'fsn-02')->delete();
$host = Host::factory()->active()->create(['datacenter' => 'fsn', 'wg_ip' => '10.66.0.30', 'dns_name' => null]);
$run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);
(new RegisterHostDns($s['hostDns']))->execute($run);
expect($host->fresh()->dns_name)->toBe('fsn-03');
});
it('retries a failed DNS registration before giving up on the name', function () {
$s = fakeServices();
$s['hostDns']->failWrite = true;
$host = Host::factory()->active()->create(['datacenter' => 'fsn', 'wg_ip' => '10.66.0.40', 'dns_name' => null]);
$run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host', 'attempt' => 0]);
$step = new RegisterHostDns($s['hostDns']);
// The failure may be a lost response to a request that did create the
// record — advancing straight away would orphan it.
expect($step->execute($run)->type)->toBe('retry');
// Out of attempts, the host still finishes: a name is convenience.
$run->update(['attempt' => 5]);
expect($step->execute($run)->type)->toBe('advance')
->and($run->events()->where('step', 'register_host_dns')->exists())->toBeTrue();
});
it("takes the host's internal DNS entry with it when the host is purged", function () {
$s = fakeServices();
$host = Host::factory()->active()->create([
'datacenter' => 'fsn', 'wg_ip' => '10.66.0.50', 'dns_name' => null,
]);
$run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);
(new RegisterHostDns($s['hostDns']))->execute($run);
$name = $host->fresh()->dns_name;
expect($s['hostDns']->ips)->toHaveKey($name);
(new PurgeHost($host->uuid))->handle();
// Otherwise the machine's management address stays in the hostsdir for
// good, with the name needed to remove it deleted along with the row.
expect($s['hostDns']->ips)->not->toHaveKey($name);
});
it('keeps the host until its internal DNS entry is really gone', function () {
$s = fakeServices();
$host = Host::factory()->active()->create([
'datacenter' => 'hel', 'wg_ip' => '10.66.0.60', 'dns_name' => null,
]);
$run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);
(new RegisterHostDns($s['hostDns']))->execute($run);
$s['hostDns']->failRemove = true;
// The row carries the only name that maps to that entry — deleting it
// anyway would leave the machine's management address in the hostsdir
// for good.
expect(fn () => (new PurgeHost($host->uuid))->handle())
->toThrow(RuntimeException::class);
expect(Host::query()->whereKey($host->id)->exists())->toBeTrue();
// Once the hostsdir write is back, the purge completes.
$s['hostDns']->failRemove = false;
(new PurgeHost($host->uuid))->handle();
expect(Host::query()->whereKey($host->id)->exists())->toBeFalse()
->and($s['hostDns']->ips)->toBe([]);
});
it('still removes a legacy public Hetzner record when a pre-fix host is purged', function () {
// Hosts onboarded before this change may still carry a public record id.
// PurgeHost has always cleaned that up on removal — this keeps working
// for those, independent of clupilot:prune-host-dns, which only handles
// hosts that are NOT being deleted.
$s = fakeServices();
$recordId = $s['dns']->upsertRecord('fsn-legacy.node.clupilot.com', 'A', '10.66.0.55');
$host = Host::factory()->active()->create([
'datacenter' => 'fsn', 'wg_ip' => '10.66.0.55',
'dns_name' => 'fsn-legacy', 'dns_record_id' => $recordId,
]);
expect($s['dns']->records)->toHaveKey('fsn-legacy.node.clupilot.com');
(new PurgeHost($host->uuid))->handle();
expect($s['dns']->records)->not->toHaveKey('fsn-legacy.node.clupilot.com');
});
it('builds a valid DNS label even from an awkward datacenter code', function () {
$s = fakeServices();
// Inserted directly: the console now rejects this shape, but rows created
// before the rule was tightened still exist.
Datacenter::factory()->create(['code' => 'eu_west', 'name' => 'EU West']);
$host = Host::factory()->active()->create([
'datacenter' => 'eu_west', 'wg_ip' => '10.66.0.70', 'dns_name' => null,
]);
$run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);
(new RegisterHostDns($s['hostDns']))->execute($run);
$name = $host->fresh()->dns_name;
expect($name)->toBe('eu-west-01')
->and($name)->toMatch('/^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/');
});
it('does not hand the same name to two datacenter codes that look alike', function () {
$s = fakeServices();
Datacenter::factory()->create(['code' => 'eu_west', 'name' => 'EU West (alt)']);
Datacenter::factory()->create(['code' => 'eu-west', 'name' => 'EU West']);
$names = [];
foreach (['eu_west', 'eu-west'] as $i => $dc) {
$host = Host::factory()->active()->create([
'datacenter' => $dc, 'wg_ip' => '10.66.0.'.(80 + $i), 'dns_name' => null,
]);
$run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);
(new RegisterHostDns($s['hostDns']))->execute($run);
$names[] = $host->fresh()->dns_name;
}
// Separate counters per raw code would have produced eu-west-01 twice and
// failed the unique constraint mid-onboarding.
expect($names)->toBe(['eu-west-01', 'eu-west-02']);
});
it('sees an internal DNS entry registered while the purge was waiting', function () {
$s = fakeServices();
$host = Host::factory()->active()->create([
'datacenter' => 'fsn', 'wg_ip' => '10.66.0.90', 'dns_name' => null,
]);
$purge = new PurgeHost($host->uuid);
// Registration finishes while the purge is blocked on the run lock: the
// copy it loaded first still says there is no entry to remove.
$run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);
(new RegisterHostDns($s['hostDns']))->execute($run);
$name = $host->fresh()->dns_name;
$purge->handle();
expect($s['hostDns']->ips)->not->toHaveKey($name)
->and(Host::query()->whereKey($host->id)->exists())->toBeFalse();
});