CluPilotCloud/tests/Feature/Provisioning/HostStepsTest.php

<?php

use App\Models\Datacenter;
use App\Models\Host;
use App\Models\Operator;
use App\Models\PlanVersion;
use App\Models\ProvisioningRun;
use App\Models\RunResource;
use App\Provisioning\Jobs\PurgeHost;
use App\Provisioning\Steps\Host\CompleteHostOnboarding;
use App\Provisioning\Steps\Host\ConfigureProxmox;
use App\Provisioning\Steps\Host\ConfigureWireguard;
use App\Provisioning\Steps\Host\CreateAutomationToken;
use App\Provisioning\Steps\Host\EstablishSshTrust;
use App\Provisioning\Steps\Host\InstallProxmoxVe;
use App\Provisioning\Steps\Host\PrepareBaseSystem;
use App\Provisioning\Steps\Host\RebootIntoPveKernel;
use App\Provisioning\Steps\Host\RegisterCapacity;
use App\Provisioning\Steps\Host\RegisterHostDns;
use App\Provisioning\Steps\Host\SecureHostFirewall;
use App\Provisioning\Steps\Host\ValidateHostInput;
use App\Provisioning\Steps\Host\VerifyProxmoxApi;
use App\Provisioning\Steps\Host\VerifyVmTemplate;
use App\Services\Secrets\SecretVault;
use App\Services\Ssh\CommandResult;
use App\Services\Ssh\FakeRemoteShell;
use Illuminate\Support\Facades\Crypt;

function hostRun(Host $host, array $context = []): ProvisioningRun
{
    return ProvisioningRun::factory()->forHost($host)->create(['context' => $context]);
}

/**
 * The `wg_peer` breadcrumb ConfigureWireguard records after a handshake actually
 * succeeds — the one fact that means "SSH may use the tunnel".
 *
 * Every step from ConfigureWireguard onwards runs with this present in real life,
 * so a test that exercises one of them without it is testing the pre-tunnel
 * fallback path by accident. See HostStep::keyLogin().
 */
function proveTunnel(ProvisioningRun $run, Host $host): void
{
    RunResource::create([
        'run_id' => $run->id,
        'host_id' => $host->id,
        'kind' => 'wg_peer',
        'external_id' => 'HOSTPUB=',
    ]);
}

/**
 * What ConfigureProxmox needs from a healthy host: a vmbr0 bridge, and a
 * datacenter firewall that reads back as enabled. The fake's default-success
 * would otherwise report "pvesh set worked" and then hand back an empty options
 * document, which the step correctly refuses to believe.
 */
function scriptHealthyProxmoxHost(FakeRemoteShell $shell): void
{
    $shell->script('pvesh get /cluster/firewall/options', CommandResult::success('{"enable":1,"policy_in":"ACCEPT"}'));
}

/** A /boot RebootIntoPveKernel is willing to reboot into. */
function scriptBootableProxmoxKernel(FakeRemoteShell $shell): void
{
    $shell->script('ls -1 /boot/vmlinuz-*-pve', CommandResult::success("/boot/vmlinuz-6.8.12-4-pve\n"));
    $shell->script('ls -1 /boot/initrd.img-*-pve', CommandResult::success("/boot/initrd.img-6.8.12-4-pve\n"));
}

/** A Debian release InstallProxmoxVe knows a Proxmox suite for. */
function scriptDebianRelease(FakeRemoteShell $shell, string $id, string $codename, string $prettyName): void
{
    $shell->script('/etc/os-release', CommandResult::success($id.'|'.$codename.'|'.$prettyName."\n"));
}

/**
 * Every ACCEPT rule in an nftables ruleset, one per entry, comments stripped.
 *
 * Reading the ruleset as a LIST rather than searching it for expected lines is
 * what lets a test catch a rule somebody ADDED — which is the mutation the old
 * firewall tests missed: they asserted the WireGuard-scoped accept was present
 * and said nothing about what else might be.
 *
 * @return array<int, string>
 */
function nftAcceptRules(string $config): array
{
    $rules = [];

    foreach (preg_split('/\R/', $config) ?: [] as $line) {
        $line = trim((string) preg_replace('/\s+/', ' ', $line));

        if ($line === '' || str_starts_with($line, '#') || ! str_contains($line, 'accept')) {
            continue;
        }

        $rules[] = $line;
    }

    return $rules;
}

/**
 * The destination ports an nftables ACCEPT rule opens — both spellings, a set
 * (`dport { 80, 443 }`) and a bare port (`dport 68`).
 *
 * @return array<int, int>
 */
function nftAcceptedPorts(string $rule): array
{
    if (! preg_match('/dport (?:\{([^}]*)\}|(\d+))/', $rule, $m)) {
        return [];
    }

    $list = trim($m[1] ?? '') !== '' ? $m[1] : ($m[2] ?? '');

    return array_map('intval', array_filter(array_map('trim', explode(',', $list)), fn ($p) => $p !== ''));
}

// --- ValidateHostInput ---

it('advances on valid host input and marks the host onboarding', function () {
    $host = Host::factory()->create();
    $run = hostRun($host, ['root_password' => Crypt::encryptString('pw')]);

    expect(app(ValidateHostInput::class)->execute($run)->type)->toBe('advance')
        ->and($host->fresh()->status)->toBe('onboarding');
});

it('fails validation when the root password is missing', function () {
    $run = hostRun(Host::factory()->create(), []);

    expect(app(ValidateHostInput::class)->execute($run)->type)->toBe('fail');
});

// --- EstablishSshTrust ---

it('establishes ssh trust, pins the host key and scrubs the password', function () {
    $s = fakeServices();
    $host = Host::factory()->create(['public_ip' => '203.0.113.5']);
    $run = hostRun($host, ['root_password' => Crypt::encryptString('rootpw')]);

    expect(app(EstablishSshTrust::class)->execute($run)->type)->toBe('advance')
        ->and($host->fresh()->ssh_host_key)->not->toBeNull()
        ->and($run->fresh()->context('root_password'))->toBeNull()
        ->and($s['shell']->connectionsWith('password'))->toHaveCount(1)
        ->and($s['shell']->connectionsWith('key'))->toHaveCount(1);
});

it('skips ssh trust once the host key is already pinned', function () {
    $s = fakeServices();
    $host = Host::factory()->create(['ssh_host_key' => 'SHA256:pinned']);
    $run = hostRun($host, []);

    expect(app(EstablishSshTrust::class)->execute($run)->type)->toBe('advance')
        ->and($s['shell']->connectionsWith('password'))->toHaveCount(0);
});

it('throws when the host is unreachable (runner turns it into a retry)', function () {
    $s = fakeServices();
    $s['shell']->failConnect = true;
    $run = hostRun(Host::factory()->create(), ['root_password' => Crypt::encryptString('pw')]);

    expect(fn () => app(EstablishSshTrust::class)->execute($run))->toThrow(RuntimeException::class);
});

// --- HostStep::keyLogin address preference ---
//
// Mutation boundary: every step after ConfigureWireguard (and every later
// maintenance run) MUST reach the host over the tunnel once the tunnel is
// PROVEN — closing SSH to the world without this would strand the host
// permanently. And it must NOT use the tunnel before then, which is the other
// half and the one that was broken. ConfigureProxmox is used as a plain probe:
// keyLogin then a handful of commands, no branching on the address.

it('uses the WireGuard address for ssh once the tunnel is proven', function () {
    $s = fakeServices();
    scriptHealthyProxmoxHost($s['shell']);
    $host = Host::factory()->create(['public_ip' => '203.0.113.20', 'wg_ip' => '10.66.0.20']);
    $run = hostRun($host);
    proveTunnel($run, $host);

    app(ConfigureProxmox::class)->execute($run);

    $connections = $s['shell']->connectionsWith('key');
    expect($connections)->not->toBeEmpty();
    foreach ($connections as $connection) {
        expect($connection[1])->toBe('10.66.0.20');
    }
});

it('falls back to the public IP before the tunnel exists', function () {
    $s = fakeServices();
    scriptHealthyProxmoxHost($s['shell']);
    $host = Host::factory()->create(['public_ip' => '203.0.113.21', 'wg_ip' => null]);
    $run = hostRun($host);

    app(ConfigureProxmox::class)->execute($run);

    $connections = $s['shell']->connectionsWith('key');
    expect($connections)->not->toBeEmpty();
    foreach ($connections as $connection) {
        expect($connection[1])->toBe('203.0.113.21');
    }
});

it('never dials the tunnel for ssh while it is only allocated, not proven', function () {
    // The dead end this closes: wg_ip is persisted inside the allocation lock
    // BEFORE the handshake is ever verified, so a run that failed at the
    // handshake left a host whose row said "10.66.0.x" and whose tunnel did not
    // work. Every later attempt — including an operator pressing Retry —
    // connected to that address and died before reaching the only code that
    // could have repaired wg0.conf. Recovery meant nulling hosts.wg_ip by hand.
    $s = fakeServices();
    scriptHealthyProxmoxHost($s['shell']);
    $host = Host::factory()->create(['public_ip' => '203.0.113.22', 'wg_ip' => '10.66.0.22']);
    $run = hostRun($host);

    // No wg_peer breadcrumb: an address was reserved, nothing handshaked.
    app(ConfigureProxmox::class)->execute($run);

    $connections = $s['shell']->connectionsWith('key');
    expect($connections)->not->toBeEmpty();
    foreach ($connections as $connection) {
        expect($connection[1])->toBe('203.0.113.22')
            ->and($connection[1])->not->toBe('10.66.0.22');
    }
});

it('keeps using the tunnel on a maintenance run started long after onboarding', function () {
    // tunnelProven() is HOST-scoped, not run-scoped, on purpose: a later run has
    // no wg_peer breadcrumb of its own, and reading only its own run would send
    // it to a public IP where SecureHostFirewall has since closed port 22.
    $s = fakeServices();
    scriptHealthyProxmoxHost($s['shell']);
    $host = Host::factory()->create(['public_ip' => '203.0.113.23', 'wg_ip' => '10.66.0.23']);
    $onboarding = hostRun($host);
    proveTunnel($onboarding, $host);

    $later = hostRun($host);
    app(ConfigureProxmox::class)->execute($later);

    foreach ($s['shell']->connectionsWith('key') as $connection) {
        expect($connection[1])->toBe('10.66.0.23');
    }
});

it('prefers the vault-stored SSH key over CLUPILOT_SSH_PRIVATE_KEY', function () {
    // Mutation boundary (Part B): the SSH identity is the most sensitive
    // value in the system — it opens every host CluPilot manages. Before
    // this, HostStep::keyLogin() read config('provisioning.ssh.private_key')
    // directly, so a key stored in the console was encrypted and kept, but
    // never once reached an actual connection.
    config()->set('admin_access.secrets_key', 'base64:'.base64_encode(random_bytes(32)));
    config()->set('provisioning.ssh.private_key', 'env-key-should-not-be-used');
    app(SecretVault::class)->put('ssh.private_key', 'vault-key-should-be-used', Operator::factory()->create());

    $s = fakeServices();
    scriptHealthyProxmoxHost($s['shell']);
    $run = hostRun(Host::factory()->create());

    app(ConfigureProxmox::class)->execute($run);

    $connections = $s['shell']->connectionsWith('key');
    expect($connections)->not->toBeEmpty();
    foreach ($connections as $connection) {
        expect($connection[3])->toBe('vault-key-should-be-used');
    }
});

// --- PrepareBaseSystem ---

it('prepares the base system', function () {
    $s = fakeServices();
    $run = hostRun(Host::factory()->create());

    expect(app(PrepareBaseSystem::class)->execute($run)->type)->toBe('advance')
        ->and($s['shell']->ran('apt-get update'))->toBeTrue();
});

it('retries base preparation on an apt failure', function () {
    $s = fakeServices();
    $s['shell']->script('apt-get update', CommandResult::failure(1, 'network down'));
    $run = hostRun(Host::factory()->create());

    expect(app(PrepareBaseSystem::class)->execute($run)->type)->toBe('retry');
});

// --- ConfigureWireguard ---

it('configures wireguard and registers the peer exactly once', function () {
    $s = fakeServices();
    $s['shell']->script('wg pubkey', CommandResult::success('HOSTPUB='));
    $host = Host::factory()->create();
    $run = hostRun($host);

    expect(app(ConfigureWireguard::class)->execute($run)->type)->toBe('advance');
    $host->refresh();
    expect($host->wg_ip)->not->toBeNull()
        ->and($host->wg_pubkey)->toBe('HOSTPUB=')
        ->and($s['hub']->peerIps())->toHaveCount(1);

    // Re-run: idempotent short-circuit, still one peer resource.
    app(ConfigureWireguard::class)->execute($run->fresh());
    expect(RunResource::where('run_id', $run->id)->where('kind', 'wg_peer')->count())->toBe(1);
});

it('retries wireguard while the handshake is not up', function () {
    $s = fakeServices();
    $s['shell']->script('wg pubkey', CommandResult::success('HOSTPUB='));
    $s['shell']->script('ping', CommandResult::failure(1));
    $run = hostRun(Host::factory()->create());

    expect(app(ConfigureWireguard::class)->execute($run)->type)->toBe('retry');
});

it('refuses to write wg0.conf without a hub public key, before touching anything', function () {
    // renderConfig() used to emit `PublicKey = ` quite happily, which is the
    // likeliest way to reach a tunnel that can never handshake in the first
    // place. Nothing may be installed, allocated or written on the way to
    // finding out — that is what makes it a clean report rather than another
    // half-configured host.
    $s = fakeServices();
    $s['hub']->publicKeyValue = '';
    $host = Host::factory()->create();
    $run = hostRun($host);

    $result = app(ConfigureWireguard::class)->execute($run);

    expect($result->type)->toBe('fail')
        ->and($result->reason)->toContain('CLUPILOT_WG_HUB_PUBKEY')
        ->and($s['shell']->ran('apt-get install -y wireguard'))->toBeFalse()
        ->and($s['shell']->files())->not->toHaveKey('/etc/wireguard/wg0.conf')
        ->and($host->fresh()->wg_ip)->toBeNull();
});

it('refuses to write wg0.conf without a hub endpoint', function () {
    $s = fakeServices();
    $s['hub']->endpointValue = '';
    $run = hostRun(Host::factory()->create());

    $result = app(ConfigureWireguard::class)->execute($run);

    expect($result->type)->toBe('fail')
        ->and($result->reason)->toContain('CLUPILOT_WG_ENDPOINT');
});

it('enables wg-quick@wg0 so the tunnel comes back after the reboot', function () {
    // The line this replaces was `systemctl enable --now wg-quick@wg0 ||
    // wg-quick up wg0`. Its fallback brought the interface up WITHOUT the
    // systemd enablement, so the tunnel did not return after the step-6 reboot —
    // and step 6 is the reboot this same pipeline performs two steps later.
    $s = fakeServices();
    $s['shell']->script('wg pubkey', CommandResult::success('HOSTPUB='));
    $s['shell']->script('ip link show wg0', CommandResult::failure(1));          // fresh host
    $s['shell']->script('systemctl is-enabled', CommandResult::failure(1));      // never enabled
    $run = hostRun(Host::factory()->create());

    expect(app(ConfigureWireguard::class)->execute($run)->type)->toBe('advance')
        ->and($s['shell']->ran('systemctl enable wg-quick@wg0'))->toBeTrue()
        ->and($s['shell']->ran('systemctl start wg-quick@wg0'))->toBeTrue()
        // The old fallback: an interface up outside systemd, lost on reboot.
        ->and($s['shell']->ran('wg-quick up wg0'))->toBeFalse();
});

it('does not fight an interface that is already up and correct', function () {
    // Both halves of the old line failed with "wg0 already exists" on the second
    // attempt, so the step retried forever against a tunnel that was working.
    $s = fakeServices();
    $s['shell']->script('wg pubkey', CommandResult::success('HOSTPUB='));
    $s['shell']->script('ip link show wg0', CommandResult::success('3: wg0: <POINTOPOINT,UP>'));
    $s['shell']->script('systemctl is-enabled', CommandResult::success('enabled'));
    $host = Host::factory()->create();
    $run = hostRun($host);

    // First pass writes the config, on a shell we then throw away.
    app(ConfigureWireguard::class)->execute($run);
    $written = $s['shell']->files()['/etc/wireguard/wg0.conf'];

    // Attempt 2 against exactly that file, on a fresh shell so `recorded` only
    // holds this attempt. The breadcrumb is rewound so the step really re-runs
    // its body instead of short-circuiting — the case that used to retry forever.
    RunResource::where('run_id', $run->id)->where('kind', 'wg_peer')->delete();
    $again = fakeServices();
    $again['shell']->script('wg pubkey', CommandResult::success('HOSTPUB='));
    $again['shell']->script('ip link show wg0', CommandResult::success('3: wg0: <POINTOPOINT,UP>'));
    $again['shell']->script('systemctl is-enabled', CommandResult::success('enabled'));
    $again['shell']->script("cat '/etc/wireguard/wg0.conf'", CommandResult::success($written));

    expect(app(ConfigureWireguard::class)->execute($run->fresh())->type)->toBe('advance')
        ->and($again['shell']->ran('systemctl restart wg-quick@wg0'))->toBeFalse()
        // Nor is the file rewritten for nothing.
        ->and($again['shell']->files())->not->toHaveKey('/etc/wireguard/wg0.conf');
});

it('actually applies a corrected hub key instead of only writing the file', function () {
    // An operator who fixes a wrong hub key and retries used to see the new file
    // on disk and no change in behaviour: nothing reloaded the interface, so the
    // running tunnel kept the old peer.
    $s = fakeServices();
    $s['shell']->script('wg pubkey', CommandResult::success('HOSTPUB='));
    $s['shell']->script('ip link show wg0', CommandResult::success('3: wg0: <POINTOPOINT,UP>'));
    $s['shell']->script('systemctl is-enabled', CommandResult::success('enabled'));
    $s['shell']->script('cat \'/etc/wireguard/wg0.conf\'', CommandResult::success(
        "[Interface]\nAddress = 10.66.0.9/24\n\n[Peer]\nPublicKey = THEWRONGKEY=\n"
    ));
    $run = hostRun(Host::factory()->create());

    expect(app(ConfigureWireguard::class)->execute($run)->type)->toBe('advance')
        ->and($s['shell']->files()['/etc/wireguard/wg0.conf'])->toContain($s['hub']->publicKeyValue)
        ->and($s['shell']->ran('systemctl restart wg-quick@wg0'))->toBeTrue();
});

// --- InstallProxmoxVe ---

it('skips the install when proxmox-ve is already present', function () {
    $s = fakeServices();
    $s['shell']->script('dpkg -l proxmox-ve', CommandResult::success('ii  proxmox-ve'));
    $run = hostRun(Host::factory()->create());

    expect(app(InstallProxmoxVe::class)->execute($run)->type)->toBe('advance')
        ->and($s['shell']->ran('apt-get -y install proxmox-ve'))->toBeFalse();
});

it('installs proxmox-ve against the suite the host actually runs', function () {
    // A server ordered today boots Debian 13 trixie. The step used to hard-code
    // bookworm, i.e. add the PVE 8 repo and keyring to a trixie base — a
    // hypervisor built against a different libc and kernel.
    $s = fakeServices();
    $s['shell']->script('dpkg -l proxmox-ve', CommandResult::failure(1));
    scriptDebianRelease($s['shell'], 'debian', 'trixie', 'Debian GNU/Linux 13 (trixie)');
    $run = hostRun(Host::factory()->create());

    expect(app(InstallProxmoxVe::class)->execute($run)->type)->toBe('advance')
        ->and($s['shell']->ran('apt-get -y install proxmox-ve'))->toBeTrue();

    $sources = $s['shell']->files()['/etc/apt/sources.list.d/pve-install-repo.sources'];
    expect($sources)->toContain('Suites: trixie')
        ->and($sources)->not->toContain('bookworm')
        // Scoped to this repository via Signed-By, not trusted for every repo on
        // the machine the way /etc/apt/trusted.gpg.d would be.
        ->and($sources)->toContain('Signed-By: /usr/share/keyrings/proxmox-archive-keyring.gpg')
        ->and($s['shell']->ran('proxmox-archive-keyring-trixie.gpg'))->toBeTrue()
        ->and($s['shell']->ran('/etc/apt/trusted.gpg.d/'))->toBeFalse();
});

it('installs the bookworm suite on a bookworm host', function () {
    $s = fakeServices();
    $s['shell']->script('dpkg -l proxmox-ve', CommandResult::failure(1));
    scriptDebianRelease($s['shell'], 'debian', 'bookworm', 'Debian GNU/Linux 12 (bookworm)');
    $run = hostRun(Host::factory()->create());

    expect(app(InstallProxmoxVe::class)->execute($run)->type)->toBe('advance');

    $sources = $s['shell']->files()['/etc/apt/sources.list.d/pve-install-repo.sources'];
    expect($sources)->toContain('Suites: bookworm')
        ->and($sources)->not->toContain('trixie')
        ->and($sources)->toContain('Signed-By: /usr/share/keyrings/proxmox-release-bookworm.gpg')
        ->and($s['shell']->ran('proxmox-release-bookworm.gpg'))->toBeTrue();
});

it('refuses an unknown release by name rather than guessing a suite', function () {
    $s = fakeServices();
    $s['shell']->script('dpkg -l proxmox-ve', CommandResult::failure(1));
    scriptDebianRelease($s['shell'], 'debian', 'forky', 'Debian GNU/Linux 14 (forky)');
    $run = hostRun(Host::factory()->create());

    $result = app(InstallProxmoxVe::class)->execute($run);

    expect($result->type)->toBe('fail')
        // Names what it found, so an operator does not have to go and look.
        ->and($result->reason)->toContain('forky')
        ->and($result->reason)->toContain('Debian GNU/Linux 14 (forky)')
        // And nothing was installed on the guess that newest-is-closest.
        ->and($s['shell']->ran('apt-get -y install proxmox-ve'))->toBeFalse()
        ->and($s['shell']->files())->not->toHaveKey('/etc/apt/sources.list.d/pve-install-repo.sources');
});

it('refuses a base system that is not Debian at all', function () {
    $s = fakeServices();
    $s['shell']->script('dpkg -l proxmox-ve', CommandResult::failure(1));
    scriptDebianRelease($s['shell'], 'ubuntu', 'noble', 'Ubuntu 24.04.1 LTS');
    $run = hostRun(Host::factory()->create());

    $result = app(InstallProxmoxVe::class)->execute($run);

    expect($result->type)->toBe('fail')
        ->and($result->reason)->toContain('ubuntu')
        ->and($s['shell']->ran('apt-get -y install proxmox-ve'))->toBeFalse();
});

it('refuses an image whose os-release names no codename', function () {
    // Debian unstable has no VERSION_CODENAME, so this is the branch that must
    // not read "" as "close enough to the newest one I know".
    $s = fakeServices();
    $s['shell']->script('dpkg -l proxmox-ve', CommandResult::failure(1));
    scriptDebianRelease($s['shell'], 'debian', '', 'Debian GNU/Linux trixie/sid');
    $run = hostRun(Host::factory()->create());

    $result = app(InstallProxmoxVe::class)->execute($run);

    expect($result->type)->toBe('fail')
        ->and($result->reason)->toContain('(empty)')
        ->and($result->reason)->toContain('trixie/sid');
});

// --- RebootIntoPveKernel ---

it('issues the reboot on first execution', function () {
    $s = fakeServices();
    scriptBootableProxmoxKernel($s['shell']);
    $run = hostRun(Host::factory()->create());

    expect(app(RebootIntoPveKernel::class)->execute($run)->type)->toBe('poll')
        ->and($run->fresh()->context('reboot_issued'))->toBeTrue()
        ->and($s['shell']->ran('reboot'))->toBeTrue();
});

it('refuses to reboot a host with no Proxmox kernel installed', function () {
    // Mutation boundary: the reboot is the one irreversible thing this pipeline
    // does. `apt-get remove linux-image-amd64 || true` followed by an unchecked
    // update-grub could leave a machine that only the provider's console can
    // reach — so nothing may be issued on a guess.
    $s = fakeServices();
    $s['shell']->script('ls -1 /boot/vmlinuz-*-pve', CommandResult::failure(2));
    $run = hostRun(Host::factory()->create());

    $result = app(RebootIntoPveKernel::class)->execute($run);

    expect($result->type)->toBe('fail')
        ->and($result->reason)->toContain('vmlinuz-*-pve')
        ->and($s['shell']->ran('reboot'))->toBeFalse()
        ->and($run->fresh()->context('reboot_issued'))->toBeNull();
});

it('refuses to reboot when the Proxmox kernel has no initramfs', function () {
    // What a /boot that filled up during the full-upgrade actually leaves behind.
    $s = fakeServices();
    $s['shell']->script('ls -1 /boot/vmlinuz-*-pve', CommandResult::success("/boot/vmlinuz-6.8.12-4-pve\n"));
    $s['shell']->script('ls -1 /boot/initrd.img-*-pve', CommandResult::failure(2));
    $run = hostRun(Host::factory()->create());

    $result = app(RebootIntoPveKernel::class)->execute($run);

    expect($result->type)->toBe('fail')
        ->and($result->reason)->toContain('initrd.img-*-pve')
        ->and($s['shell']->ran('reboot'))->toBeFalse();
});

it('refuses to reboot when update-grub failed', function () {
    $s = fakeServices();
    scriptBootableProxmoxKernel($s['shell']);
    $s['shell']->script('update-grub', CommandResult::failure(1, 'cannot write /boot/grub/grub.cfg: No space left'));
    $run = hostRun(Host::factory()->create());

    $result = app(RebootIntoPveKernel::class)->execute($run);

    expect($result->type)->toBe('fail')
        ->and($result->reason)->toContain('update-grub')
        ->and($result->reason)->toContain('No space left')
        ->and($s['shell']->ran('reboot'))->toBeFalse();
});

it('advances once the proxmox kernel is up', function () {
    $s = fakeServices();
    $s['shell']->script('uname -r', CommandResult::success('6.8.12-4-pve'));
    $run = hostRun(Host::factory()->create(), [
        'reboot_issued' => true,
        'reboot_deadline' => now()->addMinutes(10)->toIso8601String(),
    ]);

    expect(app(RebootIntoPveKernel::class)->execute($run)->type)->toBe('advance');
});

it('keeps waiting while the old kernel is still running', function () {
    $s = fakeServices();
    $s['shell']->script('uname -r', CommandResult::success('6.1.0-18-amd64'));
    $run = hostRun(Host::factory()->create(), [
        'reboot_issued' => true,
        'reboot_deadline' => now()->addMinutes(10)->toIso8601String(),
    ]);

    expect(app(RebootIntoPveKernel::class)->execute($run)->type)->toBe('poll');
});

it('fails when the host does not return before the reboot deadline', function () {
    $s = fakeServices();
    $s['shell']->script('uname -r', CommandResult::success('6.1.0-18-amd64'));
    $run = hostRun(Host::factory()->create(), [
        'reboot_issued' => true,
        'reboot_deadline' => now()->subMinute()->toIso8601String(),
    ]);

    expect(app(RebootIntoPveKernel::class)->execute($run)->type)->toBe('fail')
        ->and($run->fresh()->context('reboot_issued'))->toBeNull(); // cleared so a retry re-issues
});

// --- CreateAutomationToken ---

it('creates and persists the automation token via pveum exactly once', function () {
    config()->set('admin_access.secrets_key', 'base64:'.base64_encode(random_bytes(32)));
    $s = fakeServices();
    $s['shell']->script('pveum user token add', CommandResult::success(
        json_encode(['full-tokenid' => 'automation@pve!clupilot', 'value' => 'tok-secret-123'])
    ));
    $host = Host::factory()->create();
    $run = hostRun($host);

    expect(app(CreateAutomationToken::class)->execute($run)->type)->toBe('advance');
    $host->refresh();
    expect($host->api_token_ref)->toContain('automation@pve!clupilot')
        ->and($host->api_token_ref)->toContain('tok-secret-123');

    $tokenCalls = fn () => count(array_filter($s['shell']->recorded(), fn ($c) => str_contains($c, 'pveum user token add')));
    expect($tokenCalls())->toBe(1);

    // Re-run: idempotent short-circuit, no second token creation.
    app(CreateAutomationToken::class)->execute($run->fresh());
    expect($tokenCalls())->toBe(1)
        ->and(RunResource::where('run_id', $run->id)->where('kind', 'pve_token')->count())->toBe(1);
});

it('grants Sys.Modify, which every customer backup job needs', function () {
    // POST /cluster/backup checks ['perm', '/', ['Sys.Modify']]. Without it the
    // call comes back 403, RegisterBackup rethrows anything whose body does not
    // contain "already", and EVERY customer provisioning run failed at
    // register_backup — long after this host looked fine.
    expect(config('provisioning.proxmox.role_privs'))->toContain('Sys.Modify');
});

it('converges the role privileges on a host that was onboarded before they changed', function () {
    // `pveum role add … || true` only ever applied the privilege list on the run
    // that first created the role, so a privilege added to the config reached new
    // hosts and no existing one. The convergence must also sit BEFORE the token
    // short-circuit, or a replay never reaches it.
    config()->set('admin_access.secrets_key', 'base64:'.base64_encode(random_bytes(32)));
    $s = fakeServices();
    $s['shell']->script('pveum role add', CommandResult::failure(1, 'role CluPilotAutomation already exists'));
    $host = Host::factory()->create(['api_token_ref' => 'automation@pve!clupilot=already-there']);
    $run = hostRun($host);
    RunResource::create([
        'run_id' => $run->id, 'host_id' => $host->id, 'kind' => 'pve_token',
        'external_id' => 'automation@pve!clupilot',
    ]);

    expect(app(CreateAutomationToken::class)->execute($run)->type)->toBe('advance');

    $modify = collect($s['shell']->recorded())->first(fn ($c) => str_contains($c, 'pveum role modify'));
    expect($modify)->not->toBeNull()
        ->and($modify)->toContain('Sys.Modify')
        // Replaces rather than appends: the host matches the config, not the
        // accumulation of everything any past version of it once granted.
        ->and($modify)->not->toContain('-append')
        // And the short-circuit still holds for the token itself.
        ->and($s['shell']->ran('pveum user token add'))->toBeFalse();
});

it('retries rather than minting a token the role privileges did not converge for', function () {
    $s = fakeServices();
    $s['shell']->script('pveum role', CommandResult::failure(1, 'permission denied'));
    $run = hostRun(Host::factory()->create());

    expect(app(CreateAutomationToken::class)->execute($run)->type)->toBe('retry')
        ->and($s['shell']->ran('pveum user token add'))->toBeFalse();
});

// --- ConfigureProxmox ---

it('refuses to finish a host that has no vmbr0 bridge', function () {
    // Proxmox installed on top of Debian does not create vmbr0 — only the ISO
    // installer does. The step used to run `ip link show vmbr0`, throw the result
    // away, and advance, so onboarding reported `active` on a host with no bridge
    // to attach a customer VM to.
    $s = fakeServices();
    $s['shell']->script('ip link show vmbr0', CommandResult::failure(1));
    $s['shell']->script('ip -4 route show default', CommandResult::success('default via 203.0.113.1 dev enp0s31f6'));
    $run = hostRun(Host::factory()->create());

    $result = app(ConfigureProxmox::class)->execute($run);

    expect($result->type)->toBe('fail')
        ->and($result->reason)->toContain('vmbr0')
        // Tells the operator what to do, including which NIC carries the route.
        ->and($result->reason)->toContain('/etc/network/interfaces')
        ->and($result->reason)->toContain('enp0s31f6')
        // And never tries to build the bridge itself: getting that wrong over the
        // very link the shell runs on takes the machine off the network for good.
        ->and($s['shell']->ran('ifreload'))->toBeFalse()
        ->and($s['shell']->ran('brctl'))->toBeFalse();
});

it('enables the datacenter firewall without locking the host out', function () {
    // Proxmox applies a guest's firewall rules only while the firewall is enabled
    // at datacenter level, so applyFirewall()'s "80/443 only" rules were inert on
    // every host. Enabling it blindly is the other trap: the datacenter input
    // policy defaults to DROP, the node firewall defaults to ON, and the only
    // sources it admits to 22/8006 come from an ipset seeded with the PUBLIC
    // subnet — not the tunnel every later step uses.
    $s = fakeServices();
    scriptHealthyProxmoxHost($s['shell']);
    $host = Host::factory()->create(['public_ip' => '203.0.113.40', 'wg_ip' => '10.66.0.40']);
    $run = hostRun($host);
    proveTunnel($run, $host);

    expect(app(ConfigureProxmox::class)->execute($run)->type)->toBe('advance');

    $commands = collect($s['shell']->recorded());
    $policy = $commands->search(fn ($c) => str_contains($c, '--policy_in ACCEPT'));
    $nodeOff = $commands->search(fn ($c) => str_contains($c, '/firewall/options --enable 0'));
    $enable = $commands->search(fn ($c) => str_contains($c, '/cluster/firewall/options --enable 1'));

    expect($policy)->not->toBeFalse()
        ->and($nodeOff)->not->toBeFalse()
        ->and($enable)->not->toBeFalse()
        // Order is the safety property: both mitigations land before the master
        // switch, so there is no window in which pve-firewall compiles a DROP
        // policy that cannot see the tunnel.
        ->and($policy)->toBeLessThan($enable)
        ->and($nodeOff)->toBeLessThan($enable);
});

it('does not believe a datacenter firewall write it cannot read back', function () {
    // /etc/pve is a replicated filesystem: `pvesh set` exiting 0 is not the same
    // as the setting being there, and a lost write means every customer VM's
    // rules go on being inert while onboarding reports success.
    $s = fakeServices();
    $s['shell']->script('pvesh get /cluster/firewall/options', CommandResult::success('{"enable":0}'));
    $run = hostRun(Host::factory()->create());

    expect(app(ConfigureProxmox::class)->execute($run)->type)->toBe('retry');
});

it('drops the enterprise repo the proxmox-ve package brings back with it', function () {
    $s = fakeServices();
    scriptHealthyProxmoxHost($s['shell']);
    $run = hostRun(Host::factory()->create());

    app(ConfigureProxmox::class)->execute($run);

    expect($s['shell']->ran('pve-enterprise.list'))->toBeTrue()
        // PVE 9 ships the deb822 spelling instead, and one of the two is always
        // the file that is actually there.
        ->and($s['shell']->ran('pve-enterprise.sources'))->toBeTrue();
});

// --- VerifyVmTemplate ---
//
// The seeded catalogue every installation starts with points all four plans at
// template_vmid 9000, which is exactly the situation this step exists for.

it('refuses to finish a host that has no template to clone from', function () {
    // A host used to go `active` while it could not serve a single customer:
    // every plan version points at a template_vmid, nothing in the pipeline
    // creates or verifies it, and there is no console command for it either — so
    // the first PAID order died in clone_vm, after the money had changed hands.
    $s = fakeServices();
    $run = hostRun(Host::factory()->active()->create());

    $result = app(VerifyVmTemplate::class)->execute($run);

    expect(PlanVersion::query()->whereNotNull('published_at')->where('template_vmid', 9000)->exists())->toBeTrue()
        ->and($result->type)->toBe('fail')
        ->and($result->reason)->toContain('9000')
        ->and($result->reason)->toContain('clone_vm')
        // It reports; it does not invent a golden image nobody chose.
        ->and($s['pve']->clonedVmids)->toBe([]);
});

it('advances once the template a published plan points at is on the node', function () {
    $s = fakeServices();
    $s['pve']->clonedVmids[] = 9000; // the fake's stand-in for "this VMID exists"
    $run = hostRun(Host::factory()->active()->create());

    expect(app(VerifyVmTemplate::class)->execute($run)->type)->toBe('advance');
});

it('still checks a version that is published but not on sale until next week', function () {
    // PlanVersion::available() would exclude it, and its template gap would then
    // surface at its first order rather than here — the whole failure this step
    // removes. A CLOSED window is different: nobody can buy it any more.
    $s = fakeServices();
    PlanVersion::query()->update(['available_from' => now()->addWeek()]);
    $run = hostRun(Host::factory()->active()->create());

    expect(app(VerifyVmTemplate::class)->execute($run)->type)->toBe('fail');

    PlanVersion::query()->update(['available_until' => now()->subDay()]);
    expect(app(VerifyVmTemplate::class)->execute($run)->type)->toBe('advance')
        ->and($s['pve']->clonedVmids)->toBe([]);
});

it('requires no template on an installation that has not published a plan yet', function () {
    fakeServices();
    PlanVersion::query()->update(['published_at' => null]);
    $run = hostRun(Host::factory()->active()->create());

    expect(app(VerifyVmTemplate::class)->execute($run)->type)->toBe('advance');
});

// --- VerifyProxmoxApi ---

it('advances when the api returns nodes', function () {
    fakeServices();
    expect(app(VerifyProxmoxApi::class)->execute(hostRun(Host::factory()->create()))->type)->toBe('advance');
});

it('retries when the api returns no nodes yet', function () {
    $s = fakeServices();
    $s['pve']->nodes = [];
    expect(app(VerifyProxmoxApi::class)->execute(hostRun(Host::factory()->create()))->type)->toBe('retry');
});

// --- RegisterCapacity ---

it('registers node capacity on the host', function () {
    fakeServices();
    $host = Host::factory()->create();
    $run = hostRun($host);

    expect(app(RegisterCapacity::class)->execute($run)->type)->toBe('advance');
    $host->refresh();
    expect($host->cpu_cores)->toBe(16)
        ->and($host->total_ram_mb)->toBe(65536)
        ->and($host->total_gb)->toBe(1024)
        ->and($host->last_seen_at)->not->toBeNull();
});

// --- SecureHostFirewall ---

it('places SecureHostFirewall after every other host step and before completion', function () {
    $pipeline = config('provisioning.pipelines.host');

    $secureIndex = array_search(SecureHostFirewall::class, $pipeline, true);
    $capacityIndex = array_search(RegisterCapacity::class, $pipeline, true);
    $completeIndex = array_search(CompleteHostOnboarding::class, $pipeline, true);

    expect($secureIndex)->not->toBeFalse()
        ->and($secureIndex)->toBe($capacityIndex + 1)
        ->and($secureIndex)->toBe($completeIndex - 1);
});

it('refuses to apply the firewall when the tunnel is not confirmed up from the host', function () {
    // Mutation boundary: a host whose tunnel does not verify must come away
    // completely untouched — retrying is not enough on its own, nothing may
    // have been written either.
    $s = fakeServices();
    $s['shell']->script('ping', CommandResult::failure(1));
    $host = Host::factory()->active()->create();
    $run = hostRun($host);

    expect(app(SecureHostFirewall::class)->execute($run)->type)->toBe('retry');

    expect($s['shell']->files())->not->toHaveKey('/etc/nftables.conf')
        ->and($s['shell']->ran('apt-get install -y nftables'))->toBeFalse()
        ->and(RunResource::where('run_id', $run->id)->where('kind', 'host_firewall')->exists())->toBeFalse();
});

it('applies the firewall once the tunnel is confirmed up, over the tunnel itself', function () {
    $s = fakeServices();
    $host = Host::factory()->active()->create(['public_ip' => '203.0.113.30']);
    $run = hostRun($host);
    proveTunnel($run, $host);

    expect(app(SecureHostFirewall::class)->execute($run)->type)->toBe('advance');

    // Connected over the tunnel, not the public address.
    foreach ($s['shell']->connectionsWith('key') as $connection) {
        expect($connection[1])->toBe($host->wg_ip);
    }

    $wgSubnet = (string) config('provisioning.wireguard.subnet');
    $config = $s['shell']->files()['/etc/nftables.conf'];
    expect($config)->toContain('policy drop')
        ->and($config)->toContain('tcp dport { 80, 443 } accept')
        ->and($config)->toContain('ip saddr '.$wgSubnet.' tcp dport { 22, 8006 } accept')
        ->and($config)->toContain('ct state established,related accept');

    expect($s['shell']->files())->toHaveKey('/usr/local/sbin/clupilot-emergency-open-firewall.sh')
        ->and($s['shell']->ran('chmod 700 \'/usr/local/sbin/clupilot-emergency-open-firewall.sh\''))->toBeTrue()
        ->and($s['shell']->ran('systemctl enable --now nftables'))->toBeTrue()
        ->and(RunResource::where('run_id', $run->id)->where('kind', 'host_firewall')->count())->toBe(1);
});

it('closes 8006 and 22 to everything that is not the management subnet', function () {
    // The owner's stated requirement, and the half nothing asserted: the old
    // tests only checked that the WireGuard-scoped ACCEPT was PRESENT, so a
    // mutation adding a bare `tcp dport { 8006 } accept` passed the whole suite
    // while handing every scanner on the internet a Proxmox login form.
    //
    // Read as "which destination ports does this ruleset accept from a source it
    // has not restricted to the tunnel", rather than as a search for one
    // expected line — that is what makes an ADDED rule fail it.
    $s = fakeServices();
    $host = Host::factory()->active()->create();
    $run = hostRun($host);
    proveTunnel($run, $host);

    app(SecureHostFirewall::class)->execute($run);

    $wgSubnet = (string) config('provisioning.wireguard.subnet');
    $config = $s['shell']->files()['/etc/nftables.conf'];

    $worldOpen = [];
    $tunnelOnly = [];
    foreach (nftAcceptRules($config) as $rule) {
        // ICMP and the loopback/conntrack rules carry no destination port.
        foreach (nftAcceptedPorts($rule) as $port) {
            if (str_contains($rule, 'ip saddr '.$wgSubnet)) {
                $tunnelOnly[] = $port;
            } else {
                $worldOpen[] = $port;
            }
        }
    }

    sort($worldOpen);
    sort($tunnelOnly);

    expect($tunnelOnly)->toBe([22, 8006])
        // Exactly these and nothing else: public web, and the DHCP client reply
        // below. 22 and 8006 must NOT appear here.
        ->and($worldOpen)->toBe([68, 80, 443])
        ->and($worldOpen)->not->toContain(8006)
        ->and($worldOpen)->not->toContain(22)
        // A DHCP reply, not "anything to port 68".
        ->and($config)->toContain('udp sport 67 udp dport 68 accept')
        // And the chain still ends in drop, so an unmatched packet is refused
        // rather than accepted by omission.
        ->and($config)->toContain('policy drop')
        ->and($config)->not->toContain('policy accept');
});

it('accepts the ICMP a working host cannot do without', function () {
    // policy drop with no ICMP accept at all — grepped, zero mentions. On a real
    // host that breaks IPv6 outright as soon as the neighbour cache expires (NDP
    // 135/136 and router advertisements arrive in the INPUT chain) and
    // black-holes large transfers, because path-MTU discovery runs on
    // frag-needed / packet-too-big. Debian's own example ruleset accepts both.
    $s = fakeServices();
    $host = Host::factory()->active()->create();
    $run = hostRun($host);
    proveTunnel($run, $host);

    app(SecureHostFirewall::class)->execute($run);

    $config = $s['shell']->files()['/etc/nftables.conf'];
    $icmp = implode("\n", array_filter(
        nftAcceptRules($config),
        fn (string $rule) => str_contains($rule, 'icmp'),
    ));

    expect($icmp)->toContain('nd-neighbor-solicit')
        ->and($icmp)->toContain('nd-neighbor-advert')
        ->and($icmp)->toContain('nd-router-advert')
        // Path MTU discovery, both families.
        ->and($icmp)->toContain('packet-too-big')
        ->and($icmp)->toContain('destination-unreachable')
        // Reachability, including the handshake check this very step performs.
        ->and($icmp)->toContain('echo-request')
        // Still a policy-drop ruleset, not "ICMP made it open".
        ->and($config)->toContain('policy drop');
});

it("keeps the WireGuard interface's established connections working (never a bare drop-all)", function () {
    $s = fakeServices();
    $host = Host::factory()->active()->create();
    $run = hostRun($host);
    proveTunnel($run, $host);

    app(SecureHostFirewall::class)->execute($run);

    $config = $s['shell']->files()['/etc/nftables.conf'];
    expect($config)->toContain('ct state established,related accept')
        ->and($config)->toContain('iif "lo" accept');
});

it('deploys an emergency script that never reopens the firewall by itself', function () {
    // The owner's explicit requirement, and what nothing tested: the old
    // assertion was `ran('/usr/local/sbin/clupilot-emergency-open-firewall.sh')`,
    // which putFile() does not record — the only thing that substring matched was
    // the `chmod 700` on the line above, so the same fact was asserted twice and
    // the script's CONTENTS were never looked at.
    $s = fakeServices();
    $host = Host::factory()->active()->create();
    $run = hostRun($host);
    proveTunnel($run, $host);

    app(SecureHostFirewall::class)->execute($run);

    $script = $s['shell']->files()['/usr/local/sbin/clupilot-emergency-open-firewall.sh'];

    // It does the one thing it exists for.
    expect($script)->toStartWith('#!/bin/sh')
        ->and($script)->toContain('systemctl disable --now nftables')
        ->and($script)->toContain('nft flush ruleset');

    // And nothing in it can fire without a human at the out-of-band console. A
    // firewall that reopens itself under failure is not a firewall.
    foreach (['sleep', 'systemd-run', 'OnCalendar', 'OnBootSec', '.timer', 'crontab', 'cron.d', 'at now'] as $selfTrigger) {
        expect($script)->not->toContain($selfTrigger);
    }

    // Nothing detached into the background either, which is the other way a
    // script leaves something running after the operator has walked away.
    foreach (preg_split('/\R/', $script) ?: [] as $line) {
        $line = trim($line);
        if ($line === '' || str_starts_with($line, '#')) {
            continue;
        }
        expect($line)->not->toEndWith('&');
    }

    // Nor does the step arrange one on its behalf.
    expect($s['shell']->ran('systemd-run'))->toBeFalse()
        ->and($s['shell']->ran('crontab'))->toBeFalse()
        ->and($s['shell']->ran('.timer'))->toBeFalse();
});

it('never reapplies the firewall on a re-run', function () {
    $s = fakeServices();
    $host = Host::factory()->active()->create();
    $run = hostRun($host);
    proveTunnel($run, $host);

    app(SecureHostFirewall::class)->execute($run);
    app(SecureHostFirewall::class)->execute($run->fresh());

    $installCalls = fn () => count(array_filter(
        $s['shell']->recorded(),
        fn ($c) => str_contains($c, 'apt-get install -y nftables'),
    ));

    expect($installCalls())->toBe(1)
        ->and(RunResource::where('run_id', $run->id)->where('kind', 'host_firewall')->count())->toBe(1);
});

// --- CompleteHostOnboarding ---

it('marks the host active on completion', function () {
    $host = Host::factory()->create(['status' => 'onboarding']);

    expect(app(CompleteHostOnboarding::class)->execute(hostRun($host))->type)->toBe('advance')
        ->and($host->fresh()->status)->toBe('active');
});

it('gives the host a name that points at the tunnel, not at its public address', function () {
    $s = fakeServices();
    $host = Host::factory()->active()->create([
        'datacenter' => 'fsn',
        'wg_ip' => '10.66.0.11',
        'public_ip' => '203.0.113.11',
        'dns_name' => null,
    ]);
    $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);

    $result = (new RegisterHostDns($s['hostDns']))->execute($run);

    expect($result->type)->toBe('advance')
        ->and($host->fresh()->dns_name)->toBe('fsn-01');

    // The management address: publishing a Proxmox host's public IP would hand
    // every scanner a target. Written to the internal hostsdir, not upserted
    // anywhere via the (public) Hetzner client — see the mutation test below.
    expect($s['hostDns']->fqdns['fsn-01'])->toBe('fsn-01.node.clupilot.cloud')
        ->and($s['hostDns']->ips['fsn-01'])->toBe('10.66.0.11')
        ->and($s['hostDns']->ips['fsn-01'])->not->toBe('203.0.113.11');
});

it('never reaches the public Hetzner DNS client for a host name', function () {
    // Mutation boundary: if RegisterHostDns is ever changed to also (or
    // instead) call the public client, this must fail. Verified by hand —
    // temporarily adding a `$this->publicDns->upsertRecord(...)` call to the
    // step and confirming this test goes red before reverting it.
    $s = fakeServices();
    $host = Host::factory()->active()->create([
        'datacenter' => 'fsn', 'wg_ip' => '10.66.0.12', 'dns_name' => null,
    ]);
    $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);

    (new RegisterHostDns($s['hostDns']))->execute($run);

    expect($s['dns']->records)->toBe([])
        ->and($s['dns']->values)->toBe([]);
});

it('numbers hosts per datacenter and never reuses a number', function () {
    $s = fakeServices();
    $names = [];

    foreach (['fsn', 'fsn', 'hel'] as $i => $dc) {
        $host = Host::factory()->active()->create([
            'datacenter' => $dc, 'wg_ip' => '10.66.0.'.(20 + $i), 'dns_name' => null,
        ]);
        $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);
        (new RegisterHostDns($s['hostDns']))->execute($run);
        $names[] = $host->fresh()->dns_name;
    }

    expect($names)->toBe(['fsn-01', 'fsn-02', 'hel-01']);

    // Removing the HIGHEST one is the dangerous case: deriving the number from
    // live hosts would hand fsn-02 straight back out, and a cached name would
    // then point at a different machine.
    Host::query()->where('dns_name', 'fsn-02')->delete();
    $host = Host::factory()->active()->create(['datacenter' => 'fsn', 'wg_ip' => '10.66.0.30', 'dns_name' => null]);
    $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);
    (new RegisterHostDns($s['hostDns']))->execute($run);

    expect($host->fresh()->dns_name)->toBe('fsn-03');
});

it('retries a failed DNS registration before giving up on the name', function () {
    $s = fakeServices();
    $s['hostDns']->failWrite = true;
    $host = Host::factory()->active()->create(['datacenter' => 'fsn', 'wg_ip' => '10.66.0.40', 'dns_name' => null]);
    $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host', 'attempt' => 0]);

    $step = new RegisterHostDns($s['hostDns']);

    // The failure may be a lost response to a request that did create the
    // record — advancing straight away would orphan it.
    expect($step->execute($run)->type)->toBe('retry');

    // Out of attempts, the host still finishes: a name is convenience.
    $run->update(['attempt' => 5]);
    expect($step->execute($run)->type)->toBe('advance')
        ->and($run->events()->where('step', 'register_host_dns')->exists())->toBeTrue();
});

it("takes the host's internal DNS entry with it when the host is purged", function () {
    $s = fakeServices();

    $host = Host::factory()->active()->create([
        'datacenter' => 'fsn', 'wg_ip' => '10.66.0.50', 'dns_name' => null,
    ]);
    $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);
    (new RegisterHostDns($s['hostDns']))->execute($run);

    $name = $host->fresh()->dns_name;
    expect($s['hostDns']->ips)->toHaveKey($name);

    (new PurgeHost($host->uuid))->handle();

    // Otherwise the machine's management address stays in the hostsdir for
    // good, with the name needed to remove it deleted along with the row.
    expect($s['hostDns']->ips)->not->toHaveKey($name);
});

it('keeps the host until its internal DNS entry is really gone', function () {
    $s = fakeServices();

    $host = Host::factory()->active()->create([
        'datacenter' => 'hel', 'wg_ip' => '10.66.0.60', 'dns_name' => null,
    ]);
    $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);
    (new RegisterHostDns($s['hostDns']))->execute($run);

    $s['hostDns']->failRemove = true;

    // The row carries the only name that maps to that entry — deleting it
    // anyway would leave the machine's management address in the hostsdir
    // for good.
    expect(fn () => (new PurgeHost($host->uuid))->handle())
        ->toThrow(RuntimeException::class);

    expect(Host::query()->whereKey($host->id)->exists())->toBeTrue();

    // Once the hostsdir write is back, the purge completes.
    $s['hostDns']->failRemove = false;
    (new PurgeHost($host->uuid))->handle();

    expect(Host::query()->whereKey($host->id)->exists())->toBeFalse()
        ->and($s['hostDns']->ips)->toBe([]);
});

it('still removes a legacy public Hetzner record when a pre-fix host is purged', function () {
    // Hosts onboarded before this change may still carry a public record id.
    // PurgeHost has always cleaned that up on removal — this keeps working
    // for those, independent of clupilot:prune-host-dns, which only handles
    // hosts that are NOT being deleted.
    $s = fakeServices();
    $recordId = $s['dns']->upsertRecord('fsn-legacy.node.clupilot.com', 'A', '10.66.0.55');
    $host = Host::factory()->active()->create([
        'datacenter' => 'fsn', 'wg_ip' => '10.66.0.55',
        'dns_name' => 'fsn-legacy', 'dns_record_id' => $recordId,
    ]);

    expect($s['dns']->records)->toHaveKey('fsn-legacy.node.clupilot.com');

    (new PurgeHost($host->uuid))->handle();

    expect($s['dns']->records)->not->toHaveKey('fsn-legacy.node.clupilot.com');
});

it('builds a valid DNS label even from an awkward datacenter code', function () {
    $s = fakeServices();
    // Inserted directly: the console now rejects this shape, but rows created
    // before the rule was tightened still exist.
    Datacenter::factory()->create(['code' => 'eu_west', 'name' => 'EU West']);
    $host = Host::factory()->active()->create([
        'datacenter' => 'eu_west', 'wg_ip' => '10.66.0.70', 'dns_name' => null,
    ]);
    $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);

    (new RegisterHostDns($s['hostDns']))->execute($run);

    $name = $host->fresh()->dns_name;
    expect($name)->toBe('eu-west-01')
        ->and($name)->toMatch('/^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/');
});

it('does not hand the same name to two datacenter codes that look alike', function () {
    $s = fakeServices();
    Datacenter::factory()->create(['code' => 'eu_west', 'name' => 'EU West (alt)']);
    Datacenter::factory()->create(['code' => 'eu-west', 'name' => 'EU West']);

    $names = [];
    foreach (['eu_west', 'eu-west'] as $i => $dc) {
        $host = Host::factory()->active()->create([
            'datacenter' => $dc, 'wg_ip' => '10.66.0.'.(80 + $i), 'dns_name' => null,
        ]);
        $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);
        (new RegisterHostDns($s['hostDns']))->execute($run);
        $names[] = $host->fresh()->dns_name;
    }

    // Separate counters per raw code would have produced eu-west-01 twice and
    // failed the unique constraint mid-onboarding.
    expect($names)->toBe(['eu-west-01', 'eu-west-02']);
});

it('sees an internal DNS entry registered while the purge was waiting', function () {
    $s = fakeServices();

    $host = Host::factory()->active()->create([
        'datacenter' => 'fsn', 'wg_ip' => '10.66.0.90', 'dns_name' => null,
    ]);
    $purge = new PurgeHost($host->uuid);

    // Registration finishes while the purge is blocked on the run lock: the
    // copy it loaded first still says there is no entry to remove.
    $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);
    (new RegisterHostDns($s['hostDns']))->execute($run);
    $name = $host->fresh()->dns_name;

    $purge->handle();

    expect($s['hostDns']->ips)->not->toHaveKey($name)
        ->and(Host::query()->whereKey($host->id)->exists())->toBeFalse();
});