forHost($host)->create(['context' => $context]); } // --- ValidateHostInput --- it('advances on valid host input and marks the host onboarding', function () { $host = Host::factory()->create(); $run = hostRun($host, ['root_password' => Crypt::encryptString('pw')]); expect(app(ValidateHostInput::class)->execute($run)->type)->toBe('advance') ->and($host->fresh()->status)->toBe('onboarding'); }); it('fails validation when the root password is missing', function () { $run = hostRun(Host::factory()->create(), []); expect(app(ValidateHostInput::class)->execute($run)->type)->toBe('fail'); }); // --- EstablishSshTrust --- it('establishes ssh trust, pins the host key and scrubs the password', function () { $s = fakeServices(); $host = Host::factory()->create(['public_ip' => '203.0.113.5']); $run = hostRun($host, ['root_password' => Crypt::encryptString('rootpw')]); expect(app(EstablishSshTrust::class)->execute($run)->type)->toBe('advance') ->and($host->fresh()->ssh_host_key)->not->toBeNull() ->and($run->fresh()->context('root_password'))->toBeNull() ->and($s['shell']->connectionsWith('password'))->toHaveCount(1) ->and($s['shell']->connectionsWith('key'))->toHaveCount(1); }); it('skips ssh trust once the host key is already pinned', function () { $s = fakeServices(); $host = Host::factory()->create(['ssh_host_key' => 'SHA256:pinned']); $run = hostRun($host, []); expect(app(EstablishSshTrust::class)->execute($run)->type)->toBe('advance') ->and($s['shell']->connectionsWith('password'))->toHaveCount(0); }); it('throws when the host is unreachable (runner turns it into a retry)', function () { $s = fakeServices(); $s['shell']->failConnect = true; $run = hostRun(Host::factory()->create(), ['root_password' => Crypt::encryptString('pw')]); expect(fn () => app(EstablishSshTrust::class)->execute($run))->toThrow(RuntimeException::class); }); // --- PrepareBaseSystem --- it('prepares the base system', function () { $s = fakeServices(); $run = hostRun(Host::factory()->create()); expect(app(PrepareBaseSystem::class)->execute($run)->type)->toBe('advance') ->and($s['shell']->ran('apt-get update'))->toBeTrue(); }); it('retries base preparation on an apt failure', function () { $s = fakeServices(); $s['shell']->script('apt-get update', CommandResult::failure(1, 'network down')); $run = hostRun(Host::factory()->create()); expect(app(PrepareBaseSystem::class)->execute($run)->type)->toBe('retry'); }); // --- ConfigureWireguard --- it('configures wireguard and registers the peer exactly once', function () { $s = fakeServices(); $s['shell']->script('wg pubkey', CommandResult::success('HOSTPUB=')); $host = Host::factory()->create(); $run = hostRun($host); expect(app(ConfigureWireguard::class)->execute($run)->type)->toBe('advance'); $host->refresh(); expect($host->wg_ip)->not->toBeNull() ->and($host->wg_pubkey)->toBe('HOSTPUB=') ->and($s['hub']->peerIps())->toHaveCount(1); // Re-run: idempotent short-circuit, still one peer resource. app(ConfigureWireguard::class)->execute($run->fresh()); expect(RunResource::where('run_id', $run->id)->where('kind', 'wg_peer')->count())->toBe(1); }); it('retries wireguard while the handshake is not up', function () { $s = fakeServices(); $s['shell']->script('wg pubkey', CommandResult::success('HOSTPUB=')); $s['shell']->script('ping', CommandResult::failure(1)); $run = hostRun(Host::factory()->create()); expect(app(ConfigureWireguard::class)->execute($run)->type)->toBe('retry'); }); // --- InstallProxmoxVe --- it('skips the install when proxmox-ve is already present', function () { $s = fakeServices(); $s['shell']->script('dpkg -l proxmox-ve', CommandResult::success('ii proxmox-ve')); $run = hostRun(Host::factory()->create()); expect(app(InstallProxmoxVe::class)->execute($run)->type)->toBe('advance') ->and($s['shell']->ran('apt-get -y install proxmox-ve'))->toBeFalse(); }); it('installs proxmox-ve when it is absent', function () { $s = fakeServices(); $s['shell']->script('dpkg -l proxmox-ve', CommandResult::failure(1)); $run = hostRun(Host::factory()->create()); expect(app(InstallProxmoxVe::class)->execute($run)->type)->toBe('advance') ->and($s['shell']->ran('apt-get -y install proxmox-ve'))->toBeTrue(); }); // --- RebootIntoPveKernel --- it('issues the reboot on first execution', function () { $s = fakeServices(); $run = hostRun(Host::factory()->create()); expect(app(RebootIntoPveKernel::class)->execute($run)->type)->toBe('poll') ->and($run->fresh()->context('reboot_issued'))->toBeTrue() ->and($s['shell']->ran('reboot'))->toBeTrue(); }); it('advances once the proxmox kernel is up', function () { $s = fakeServices(); $s['shell']->script('uname -r', CommandResult::success('6.8.12-4-pve')); $run = hostRun(Host::factory()->create(), [ 'reboot_issued' => true, 'reboot_deadline' => now()->addMinutes(10)->toIso8601String(), ]); expect(app(RebootIntoPveKernel::class)->execute($run)->type)->toBe('advance'); }); it('keeps waiting while the old kernel is still running', function () { $s = fakeServices(); $s['shell']->script('uname -r', CommandResult::success('6.1.0-18-amd64')); $run = hostRun(Host::factory()->create(), [ 'reboot_issued' => true, 'reboot_deadline' => now()->addMinutes(10)->toIso8601String(), ]); expect(app(RebootIntoPveKernel::class)->execute($run)->type)->toBe('poll'); }); it('fails when the host does not return before the reboot deadline', function () { $s = fakeServices(); $s['shell']->script('uname -r', CommandResult::success('6.1.0-18-amd64')); $run = hostRun(Host::factory()->create(), [ 'reboot_issued' => true, 'reboot_deadline' => now()->subMinute()->toIso8601String(), ]); expect(app(RebootIntoPveKernel::class)->execute($run)->type)->toBe('fail') ->and($run->fresh()->context('reboot_issued'))->toBeNull(); // cleared so a retry re-issues }); // --- CreateAutomationToken --- it('creates and persists the automation token via pveum exactly once', function () { $s = fakeServices(); $s['shell']->script('pveum user token add', CommandResult::success( json_encode(['full-tokenid' => 'automation@pve!clupilot', 'value' => 'tok-secret-123']) )); $host = Host::factory()->create(); $run = hostRun($host); expect(app(CreateAutomationToken::class)->execute($run)->type)->toBe('advance'); $host->refresh(); expect($host->api_token_ref)->toContain('automation@pve!clupilot') ->and($host->api_token_ref)->toContain('tok-secret-123'); $tokenCalls = fn () => count(array_filter($s['shell']->recorded(), fn ($c) => str_contains($c, 'pveum user token add'))); expect($tokenCalls())->toBe(1); // Re-run: idempotent short-circuit, no second token creation. app(CreateAutomationToken::class)->execute($run->fresh()); expect($tokenCalls())->toBe(1) ->and(RunResource::where('run_id', $run->id)->where('kind', 'pve_token')->count())->toBe(1); }); // --- VerifyProxmoxApi --- it('advances when the api returns nodes', function () { fakeServices(); expect(app(VerifyProxmoxApi::class)->execute(hostRun(Host::factory()->create()))->type)->toBe('advance'); }); it('retries when the api returns no nodes yet', function () { $s = fakeServices(); $s['pve']->nodes = []; expect(app(VerifyProxmoxApi::class)->execute(hostRun(Host::factory()->create()))->type)->toBe('retry'); }); // --- RegisterCapacity --- it('registers node capacity on the host', function () { fakeServices(); $host = Host::factory()->create(); $run = hostRun($host); expect(app(RegisterCapacity::class)->execute($run)->type)->toBe('advance'); $host->refresh(); expect($host->cpu_cores)->toBe(16) ->and($host->total_ram_mb)->toBe(65536) ->and($host->total_gb)->toBe(1024) ->and($host->last_seen_at)->not->toBeNull(); }); // --- CompleteHostOnboarding --- it('marks the host active on completion', function () { $host = Host::factory()->create(['status' => 'onboarding']); expect(app(CompleteHostOnboarding::class)->execute(hostRun($host))->type)->toBe('advance') ->and($host->fresh()->status)->toBe('active'); }); it('gives the host a name that points at the tunnel, not at its public address', function () { $s = fakeServices(); $host = Host::factory()->active()->create([ 'datacenter' => 'fsn', 'wg_ip' => '10.66.0.11', 'public_ip' => '203.0.113.11', 'dns_name' => null, ]); $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']); $result = (new RegisterHostDns($s['hostDns']))->execute($run); expect($result->type)->toBe('advance') ->and($host->fresh()->dns_name)->toBe('fsn-01'); // The management address: publishing a Proxmox host's public IP would hand // every scanner a target. Written to the internal hostsdir, not upserted // anywhere via the (public) Hetzner client — see the mutation test below. expect($s['hostDns']->fqdns['fsn-01'])->toBe('fsn-01.node.clupilot.com') ->and($s['hostDns']->ips['fsn-01'])->toBe('10.66.0.11') ->and($s['hostDns']->ips['fsn-01'])->not->toBe('203.0.113.11'); }); it('never reaches the public Hetzner DNS client for a host name', function () { // Mutation boundary: if RegisterHostDns is ever changed to also (or // instead) call the public client, this must fail. Verified by hand — // temporarily adding a `$this->publicDns->upsertRecord(...)` call to the // step and confirming this test goes red before reverting it. $s = fakeServices(); $host = Host::factory()->active()->create([ 'datacenter' => 'fsn', 'wg_ip' => '10.66.0.12', 'dns_name' => null, ]); $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']); (new RegisterHostDns($s['hostDns']))->execute($run); expect($s['dns']->records)->toBe([]) ->and($s['dns']->values)->toBe([]); }); it('numbers hosts per datacenter and never reuses a number', function () { $s = fakeServices(); $names = []; foreach (['fsn', 'fsn', 'hel'] as $i => $dc) { $host = Host::factory()->active()->create([ 'datacenter' => $dc, 'wg_ip' => '10.66.0.'.(20 + $i), 'dns_name' => null, ]); $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']); (new RegisterHostDns($s['hostDns']))->execute($run); $names[] = $host->fresh()->dns_name; } expect($names)->toBe(['fsn-01', 'fsn-02', 'hel-01']); // Removing the HIGHEST one is the dangerous case: deriving the number from // live hosts would hand fsn-02 straight back out, and a cached name would // then point at a different machine. Host::query()->where('dns_name', 'fsn-02')->delete(); $host = Host::factory()->active()->create(['datacenter' => 'fsn', 'wg_ip' => '10.66.0.30', 'dns_name' => null]); $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']); (new RegisterHostDns($s['hostDns']))->execute($run); expect($host->fresh()->dns_name)->toBe('fsn-03'); }); it('retries a failed DNS registration before giving up on the name', function () { $s = fakeServices(); $s['hostDns']->failWrite = true; $host = Host::factory()->active()->create(['datacenter' => 'fsn', 'wg_ip' => '10.66.0.40', 'dns_name' => null]); $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host', 'attempt' => 0]); $step = new RegisterHostDns($s['hostDns']); // The failure may be a lost response to a request that did create the // record — advancing straight away would orphan it. expect($step->execute($run)->type)->toBe('retry'); // Out of attempts, the host still finishes: a name is convenience. $run->update(['attempt' => 5]); expect($step->execute($run)->type)->toBe('advance') ->and($run->events()->where('step', 'register_host_dns')->exists())->toBeTrue(); }); it("takes the host's internal DNS entry with it when the host is purged", function () { $s = fakeServices(); $host = Host::factory()->active()->create([ 'datacenter' => 'fsn', 'wg_ip' => '10.66.0.50', 'dns_name' => null, ]); $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']); (new RegisterHostDns($s['hostDns']))->execute($run); $name = $host->fresh()->dns_name; expect($s['hostDns']->ips)->toHaveKey($name); (new PurgeHost($host->uuid))->handle(); // Otherwise the machine's management address stays in the hostsdir for // good, with the name needed to remove it deleted along with the row. expect($s['hostDns']->ips)->not->toHaveKey($name); }); it('keeps the host until its internal DNS entry is really gone', function () { $s = fakeServices(); $host = Host::factory()->active()->create([ 'datacenter' => 'hel', 'wg_ip' => '10.66.0.60', 'dns_name' => null, ]); $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']); (new RegisterHostDns($s['hostDns']))->execute($run); $s['hostDns']->failRemove = true; // The row carries the only name that maps to that entry — deleting it // anyway would leave the machine's management address in the hostsdir // for good. expect(fn () => (new PurgeHost($host->uuid))->handle()) ->toThrow(RuntimeException::class); expect(Host::query()->whereKey($host->id)->exists())->toBeTrue(); // Once the hostsdir write is back, the purge completes. $s['hostDns']->failRemove = false; (new PurgeHost($host->uuid))->handle(); expect(Host::query()->whereKey($host->id)->exists())->toBeFalse() ->and($s['hostDns']->ips)->toBe([]); }); it('still removes a legacy public Hetzner record when a pre-fix host is purged', function () { // Hosts onboarded before this change may still carry a public record id. // PurgeHost has always cleaned that up on removal — this keeps working // for those, independent of clupilot:prune-host-dns, which only handles // hosts that are NOT being deleted. $s = fakeServices(); $recordId = $s['dns']->upsertRecord('fsn-legacy.node.clupilot.com', 'A', '10.66.0.55'); $host = Host::factory()->active()->create([ 'datacenter' => 'fsn', 'wg_ip' => '10.66.0.55', 'dns_name' => 'fsn-legacy', 'dns_record_id' => $recordId, ]); expect($s['dns']->records)->toHaveKey('fsn-legacy.node.clupilot.com'); (new PurgeHost($host->uuid))->handle(); expect($s['dns']->records)->not->toHaveKey('fsn-legacy.node.clupilot.com'); }); it('builds a valid DNS label even from an awkward datacenter code', function () { $s = fakeServices(); // Inserted directly: the console now rejects this shape, but rows created // before the rule was tightened still exist. Datacenter::factory()->create(['code' => 'eu_west', 'name' => 'EU West']); $host = Host::factory()->active()->create([ 'datacenter' => 'eu_west', 'wg_ip' => '10.66.0.70', 'dns_name' => null, ]); $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']); (new RegisterHostDns($s['hostDns']))->execute($run); $name = $host->fresh()->dns_name; expect($name)->toBe('eu-west-01') ->and($name)->toMatch('/^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/'); }); it('does not hand the same name to two datacenter codes that look alike', function () { $s = fakeServices(); Datacenter::factory()->create(['code' => 'eu_west', 'name' => 'EU West (alt)']); Datacenter::factory()->create(['code' => 'eu-west', 'name' => 'EU West']); $names = []; foreach (['eu_west', 'eu-west'] as $i => $dc) { $host = Host::factory()->active()->create([ 'datacenter' => $dc, 'wg_ip' => '10.66.0.'.(80 + $i), 'dns_name' => null, ]); $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']); (new RegisterHostDns($s['hostDns']))->execute($run); $names[] = $host->fresh()->dns_name; } // Separate counters per raw code would have produced eu-west-01 twice and // failed the unique constraint mid-onboarding. expect($names)->toBe(['eu-west-01', 'eu-west-02']); }); it('sees an internal DNS entry registered while the purge was waiting', function () { $s = fakeServices(); $host = Host::factory()->active()->create([ 'datacenter' => 'fsn', 'wg_ip' => '10.66.0.90', 'dns_name' => null, ]); $purge = new PurgeHost($host->uuid); // Registration finishes while the purge is blocked on the run lock: the // copy it loaded first still says there is no entry to remove. $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']); (new RegisterHostDns($s['hostDns']))->execute($run); $name = $host->fresh()->dns_name; $purge->handle(); expect($s['hostDns']->ips)->not->toHaveKey($name) ->and(Host::query()->whereKey($host->id)->exists())->toBeFalse(); });