<?php

use App\Models\Datacenter;
use App\Models\Host;
use App\Models\Operator;
use App\Models\PlanFamily;
use App\Models\PlanVersion;
use App\Models\ProvisioningRun;
use App\Models\RunResource;
use App\Provisioning\Jobs\PurgeHost;
use App\Provisioning\Steps\Host\BuildVmTemplate;
use App\Provisioning\Steps\Host\CompleteHostOnboarding;
use App\Provisioning\Steps\Host\ConfigureProxmox;
use App\Provisioning\Steps\Host\ConfigureWireguard;
use App\Provisioning\Steps\Host\CreateAutomationToken;
use App\Provisioning\Steps\Host\EnsureNetworkBridge;
use App\Provisioning\Steps\Host\EstablishSshTrust;
use App\Provisioning\Steps\Host\InstallProxmoxVe;
use App\Provisioning\Steps\Host\PrepareBaseSystem;
use App\Provisioning\Steps\Host\RebootIntoPveKernel;
use App\Provisioning\Steps\Host\RegisterCapacity;
use App\Provisioning\Steps\Host\RegisterHostDns;
use App\Provisioning\Steps\Host\SecureHostFirewall;
use App\Provisioning\Steps\Host\ValidateHostInput;
use App\Provisioning\Steps\Host\VerifyProxmoxApi;
use App\Provisioning\Steps\Host\VerifyVmTemplate;
use App\Services\Secrets\SecretVault;
use App\Services\Ssh\CommandResult;
use App\Services\Ssh\FakeRemoteShell;
use App\Services\Wireguard\WireguardHub;
use Illuminate\Support\Facades\Crypt;
use Illuminate\Support\Facades\Process;
use Illuminate\Support\Str;

function hostRun(Host $host, array $context = []): ProvisioningRun
{
    return ProvisioningRun::factory()->forHost($host)->create(['context' => $context]);
}

/**
 * The `wg_peer` breadcrumb ConfigureWireguard records after a handshake actually
 * succeeds — the one fact that means "SSH may use the tunnel".
 *
 * Every step from ConfigureWireguard onwards runs with this present in real life,
 * so a test that exercises one of them without it is testing the pre-tunnel
 * fallback path by accident. See HostStep::keyLogin().
 */
function proveTunnel(ProvisioningRun $run, Host $host): void
{
    RunResource::create([
        'run_id' => $run->id,
        'host_id' => $host->id,
        'kind' => 'wg_peer',
        'external_id' => 'HOSTPUB=',
    ]);
}

/**
 * What ConfigureProxmox needs from a healthy host: a vmbr0 bridge, and a
 * datacenter firewall that reads back as enabled. The fake's default-success
 * would otherwise report "pvesh set worked" and then hand back an empty options
 * document, which the step correctly refuses to believe.
 */
function scriptHealthyProxmoxHost(FakeRemoteShell $shell): void
{
    $shell->script('pvesh get /cluster/firewall/options', CommandResult::success('{"enable":1,"policy_in":"ACCEPT"}'));
}

/** A /boot RebootIntoPveKernel is willing to reboot into. */
function scriptBootableProxmoxKernel(FakeRemoteShell $shell): void
{
    $shell->script('ls -1 /boot/vmlinuz-*-pve', CommandResult::success("/boot/vmlinuz-6.8.12-4-pve\n"));
    $shell->script('ls -1 /boot/initrd.img-*-pve', CommandResult::success("/boot/initrd.img-6.8.12-4-pve\n"));
}

/** A Debian release InstallProxmoxVe knows a Proxmox suite for. */
function scriptDebianRelease(FakeRemoteShell $shell, string $id, string $codename, string $prettyName): void
{
    $shell->script('/etc/os-release', CommandResult::success($id.'|'.$codename.'|'.$prettyName."\n"));
}

/**
 * Every ACCEPT rule in an nftables ruleset, one per entry, comments stripped.
 *
 * Reading the ruleset as a LIST rather than searching it for expected lines is
 * what lets a test catch a rule somebody ADDED — which is the mutation the old
 * firewall tests missed: they asserted the WireGuard-scoped accept was present
 * and said nothing about what else might be.
 *
 * @return array<int, string>
 */
function nftAcceptRules(string $config): array
{
    $rules = [];

    foreach (preg_split('/\R/', $config) ?: [] as $line) {
        $line = trim((string) preg_replace('/\s+/', ' ', $line));

        if ($line === '' || str_starts_with($line, '#') || ! str_contains($line, 'accept')) {
            continue;
        }

        $rules[] = $line;
    }

    return $rules;
}

/**
 * The destination ports an nftables ACCEPT rule opens — both spellings, a set
 * (`dport { 80, 443 }`) and a bare port (`dport 68`).
 *
 * @return array<int, int>
 */
function nftAcceptedPorts(string $rule): array
{
    if (! preg_match('/dport (?:\{([^}]*)\}|(\d+))/', $rule, $m)) {
        return [];
    }

    $list = trim($m[1] ?? '') !== '' ? $m[1] : ($m[2] ?? '');

    return array_map('intval', array_filter(array_map('trim', explode(',', $list)), fn ($p) => $p !== ''));
}

// --- ValidateHostInput ---

it('advances on valid host input and marks the host onboarding', function () {
    $host = Host::factory()->create();
    $run = hostRun($host, ['root_password' => Crypt::encryptString('pw')]);

    expect(app(ValidateHostInput::class)->execute($run)->type)->toBe('advance')
        ->and($host->fresh()->status)->toBe('onboarding');
});

it('fails validation when the root password is missing', function () {
    $run = hostRun(Host::factory()->create(), []);

    expect(app(ValidateHostInput::class)->execute($run)->type)->toBe('fail');
});

// --- EstablishSshTrust ---

it('establishes ssh trust, pins the host key and scrubs the password', function () {
    $s = fakeServices();
    $host = Host::factory()->create(['public_ip' => '203.0.113.5']);
    $run = hostRun($host, ['root_password' => Crypt::encryptString('rootpw')]);

    expect(app(EstablishSshTrust::class)->execute($run)->type)->toBe('advance')
        ->and($host->fresh()->ssh_host_key)->not->toBeNull()
        ->and($run->fresh()->context('root_password'))->toBeNull()
        ->and($s['shell']->connectionsWith('password'))->toHaveCount(1)
        ->and($s['shell']->connectionsWith('key'))->toHaveCount(1);
});

it('macht die Anmeldung mit Passwort zu, sobald der Schluessel nachweislich traegt', function () {
    // Bis hierher blieb root-mit-Passwort erlaubt, und Port 22 steht waehrend
    // der ganzen Uebernahme offen im Internet — die Host-Firewall macht ihn erst
    // als vorletzter Schritt zu. Dazwischen lag ein Fenster von Stunden, in dem
    // jede Maschine der Welt Root-Passwoerter durchprobieren durfte.
    $s = fakeServices();
    $host = Host::factory()->create(['public_ip' => '203.0.113.5']);
    $run = hostRun($host, ['root_password' => Crypt::encryptString('rootpw')]);

    app(EstablishSshTrust::class)->execute($run);

    $conf = $s['shell']->files()['/etc/ssh/sshd_config.d/99-clupilot.conf'] ?? '';

    expect($conf)->toContain('PasswordAuthentication no')
        ->and($conf)->toContain('PermitRootLogin prohibit-password');

    // Geprueft, bevor sshd sie zu sehen bekommt — eine Konfiguration, die er
    // nicht laden kann, darf er nie geladen bekommen. Und `reload`, nicht
    // `restart`: bestehende Sitzungen bleiben, auch die, ueber die dieser
    // Schritt gerade selbst spricht.
    expect($s['shell']->ran('sshd -t'))->toBeTrue()
        ->and($s['shell']->ran('systemctl reload ssh'))->toBeTrue()
        ->and($s['shell']->ran('systemctl restart ssh'))->toBeFalse();
});

it('skips ssh trust once the host key is already pinned', function () {
    $s = fakeServices();
    $host = Host::factory()->create(['ssh_host_key' => 'SHA256:pinned']);
    $run = hostRun($host, []);

    expect(app(EstablishSshTrust::class)->execute($run)->type)->toBe('advance')
        ->and($s['shell']->connectionsWith('password'))->toHaveCount(0);
});

it('throws when the host is unreachable (runner turns it into a retry)', function () {
    $s = fakeServices();
    $s['shell']->failConnect = true;
    $run = hostRun(Host::factory()->create(), ['root_password' => Crypt::encryptString('pw')]);

    expect(fn () => app(EstablishSshTrust::class)->execute($run))->toThrow(RuntimeException::class);
});

// --- HostStep::keyLogin address preference ---
//
// Mutation boundary: every step after ConfigureWireguard (and every later
// maintenance run) MUST reach the host over the tunnel once the tunnel is
// PROVEN — closing SSH to the world without this would strand the host
// permanently. And it must NOT use the tunnel before then, which is the other
// half and the one that was broken. ConfigureProxmox is used as a plain probe:
// keyLogin then a handful of commands, no branching on the address.

it('uses the WireGuard address for ssh once the tunnel is proven', function () {
    $s = fakeServices();
    scriptHealthyProxmoxHost($s['shell']);
    $host = Host::factory()->create(['public_ip' => '203.0.113.20', 'wg_ip' => '10.66.0.20']);
    $run = hostRun($host);
    proveTunnel($run, $host);

    app(ConfigureProxmox::class)->execute($run);

    $connections = $s['shell']->connectionsWith('key');
    expect($connections)->not->toBeEmpty();
    foreach ($connections as $connection) {
        expect($connection[1])->toBe('10.66.0.20');
    }
});

it('falls back to the public IP before the tunnel exists', function () {
    $s = fakeServices();
    scriptHealthyProxmoxHost($s['shell']);
    $host = Host::factory()->create(['public_ip' => '203.0.113.21', 'wg_ip' => null]);
    $run = hostRun($host);

    app(ConfigureProxmox::class)->execute($run);

    $connections = $s['shell']->connectionsWith('key');
    expect($connections)->not->toBeEmpty();
    foreach ($connections as $connection) {
        expect($connection[1])->toBe('203.0.113.21');
    }
});

it('never dials the tunnel for ssh while it is only allocated, not proven', function () {
    // The dead end this closes: wg_ip is persisted inside the allocation lock
    // BEFORE the handshake is ever verified, so a run that failed at the
    // handshake left a host whose row said "10.66.0.x" and whose tunnel did not
    // work. Every later attempt — including an operator pressing Retry —
    // connected to that address and died before reaching the only code that
    // could have repaired wg0.conf. Recovery meant nulling hosts.wg_ip by hand.
    $s = fakeServices();
    scriptHealthyProxmoxHost($s['shell']);
    $host = Host::factory()->create(['public_ip' => '203.0.113.22', 'wg_ip' => '10.66.0.22']);
    $run = hostRun($host);

    // No wg_peer breadcrumb: an address was reserved, nothing handshaked.
    app(ConfigureProxmox::class)->execute($run);

    $connections = $s['shell']->connectionsWith('key');
    expect($connections)->not->toBeEmpty();
    foreach ($connections as $connection) {
        expect($connection[1])->toBe('203.0.113.22')
            ->and($connection[1])->not->toBe('10.66.0.22');
    }
});

it('keeps using the tunnel on a maintenance run started long after onboarding', function () {
    // tunnelProven() is HOST-scoped, not run-scoped, on purpose: a later run has
    // no wg_peer breadcrumb of its own, and reading only its own run would send
    // it to a public IP where SecureHostFirewall has since closed port 22.
    $s = fakeServices();
    scriptHealthyProxmoxHost($s['shell']);
    $host = Host::factory()->create(['public_ip' => '203.0.113.23', 'wg_ip' => '10.66.0.23']);
    $onboarding = hostRun($host);
    proveTunnel($onboarding, $host);

    $later = hostRun($host);
    app(ConfigureProxmox::class)->execute($later);

    foreach ($s['shell']->connectionsWith('key') as $connection) {
        expect($connection[1])->toBe('10.66.0.23');
    }
});

it('prefers the vault-stored SSH key over CLUPILOT_SSH_PRIVATE_KEY', function () {
    // Mutation boundary (Part B): the SSH identity is the most sensitive
    // value in the system — it opens every host CluPilot manages. Before
    // this, HostStep::keyLogin() read config('provisioning.ssh.private_key')
    // directly, so a key stored in the console was encrypted and kept, but
    // never once reached an actual connection.
    config()->set('admin_access.secrets_key', 'base64:'.base64_encode(random_bytes(32)));
    config()->set('provisioning.ssh.private_key', 'env-key-should-not-be-used');
    app(SecretVault::class)->put('ssh.private_key', 'vault-key-should-be-used', Operator::factory()->create());

    $s = fakeServices();
    scriptHealthyProxmoxHost($s['shell']);
    $run = hostRun(Host::factory()->create());

    app(ConfigureProxmox::class)->execute($run);

    $connections = $s['shell']->connectionsWith('key');
    expect($connections)->not->toBeEmpty();
    foreach ($connections as $connection) {
        expect($connection[3])->toBe('vault-key-should-be-used');
    }
});

// --- PrepareBaseSystem ---

it('prepares the base system', function () {
    $s = fakeServices();
    $run = hostRun(Host::factory()->create());

    expect(app(PrepareBaseSystem::class)->execute($run)->type)->toBe('advance')
        ->and($s['shell']->ran('apt-get update'))->toBeTrue();
});

it('retries base preparation on an apt failure', function () {
    $s = fakeServices();
    $s['shell']->script('apt-get update', CommandResult::failure(1, 'network down'));
    $run = hostRun(Host::factory()->create());

    expect(app(PrepareBaseSystem::class)->execute($run)->type)->toBe('retry');
});

// --- ConfigureWireguard ---

it('configures wireguard and registers the peer exactly once', function () {
    $s = fakeServices();
    $s['shell']->script('wg pubkey', CommandResult::success('HOSTPUB='));
    scriptLiveTunnel($s['shell']);
    $host = Host::factory()->create();
    $run = hostRun($host);

    expect(app(ConfigureWireguard::class)->execute($run)->type)->toBe('advance');
    $host->refresh();
    expect($host->wg_ip)->not->toBeNull()
        ->and($host->wg_pubkey)->toBe('HOSTPUB=')
        ->and($s['hub']->peerIps())->toHaveCount(1);

    // Re-run: idempotent short-circuit, still one peer resource.
    app(ConfigureWireguard::class)->execute($run->fresh());
    expect(RunResource::where('run_id', $run->id)->where('kind', 'wg_peer')->count())->toBe(1);
});

it('retries wireguard while the handshake is not up', function () {
    $s = fakeServices();
    $s['shell']->script('wg pubkey', CommandResult::success('HOSTPUB='));
    $s['shell']->script('ping', CommandResult::failure(1));
    $run = hostRun(Host::factory()->create());

    expect(app(ConfigureWireguard::class)->execute($run)->type)->toBe('retry');
});

it('refuses to write wg0.conf without a hub public key, before touching anything', function () {
    // renderConfig() used to emit `PublicKey = ` quite happily, which is the
    // likeliest way to reach a tunnel that can never handshake in the first
    // place. Nothing may be installed, allocated or written on the way to
    // finding out — that is what makes it a clean report rather than another
    // half-configured host.
    $s = fakeServices();
    $s['hub']->publicKeyValue = '';
    $host = Host::factory()->create();
    $run = hostRun($host);

    $result = app(ConfigureWireguard::class)->execute($run);

    expect($result->type)->toBe('fail')
        ->and($result->reason)->toContain('CLUPILOT_WG_HUB_PUBKEY')
        ->and($s['shell']->ran('apt-get install -y wireguard'))->toBeFalse()
        ->and($s['shell']->files())->not->toHaveKey('/etc/wireguard/wg0.conf')
        ->and($host->fresh()->wg_ip)->toBeNull();
});

it('refuses to write wg0.conf without a hub endpoint', function () {
    $s = fakeServices();
    $s['hub']->endpointValue = '';
    $run = hostRun(Host::factory()->create());

    $result = app(ConfigureWireguard::class)->execute($run);

    expect($result->type)->toBe('fail')
        ->and($result->reason)->toContain('CLUPILOT_WG_ENDPOINT');
});

it('enables wg-quick@wg0 so the tunnel comes back after the reboot', function () {
    // The line this replaces was `systemctl enable --now wg-quick@wg0 ||
    // wg-quick up wg0`. Its fallback brought the interface up WITHOUT the
    // systemd enablement, so the tunnel did not return after the step-6 reboot —
    // and step 6 is the reboot this same pipeline performs two steps later.
    $s = fakeServices();
    $s['shell']->script('wg pubkey', CommandResult::success('HOSTPUB='));
    scriptLiveTunnel($s['shell']);
    $s['shell']->script('ip link show wg0', CommandResult::failure(1));          // fresh host
    $s['shell']->script('systemctl is-enabled', CommandResult::failure(1));      // never enabled
    $run = hostRun(Host::factory()->create());

    expect(app(ConfigureWireguard::class)->execute($run)->type)->toBe('advance')
        ->and($s['shell']->ran('systemctl enable wg-quick@wg0'))->toBeTrue()
        ->and($s['shell']->ran('systemctl start wg-quick@wg0'))->toBeTrue()
        // The old fallback: an interface up outside systemd, lost on reboot.
        ->and($s['shell']->ran('wg-quick up wg0'))->toBeFalse();
});

it('does not fight an interface that is already up and correct', function () {
    // Both halves of the old line failed with "wg0 already exists" on the second
    // attempt, so the step retried forever against a tunnel that was working.
    $s = fakeServices();
    $s['shell']->script('wg pubkey', CommandResult::success('HOSTPUB='));
    scriptLiveTunnel($s['shell']);
    $s['shell']->script('ip link show wg0', CommandResult::success('3: wg0: <POINTOPOINT,UP>'));
    $s['shell']->script('systemctl is-enabled', CommandResult::success('enabled'));
    $host = Host::factory()->create();
    $run = hostRun($host);

    // First pass writes the config, on a shell we then throw away.
    app(ConfigureWireguard::class)->execute($run);
    $written = $s['shell']->files()['/etc/wireguard/wg0.conf'];

    // Attempt 2 against exactly that file, on a fresh shell so `recorded` only
    // holds this attempt. The breadcrumb is rewound so the step really re-runs
    // its body instead of short-circuiting — the case that used to retry forever.
    RunResource::where('run_id', $run->id)->where('kind', 'wg_peer')->delete();
    $again = fakeServices();
    $again['shell']->script('wg pubkey', CommandResult::success('HOSTPUB='));
    scriptLiveTunnel($again['shell']);
    $again['shell']->script('ip link show wg0', CommandResult::success('3: wg0: <POINTOPOINT,UP>'));
    $again['shell']->script('systemctl is-enabled', CommandResult::success('enabled'));
    $again['shell']->script("cat '/etc/wireguard/wg0.conf'", CommandResult::success($written));

    expect(app(ConfigureWireguard::class)->execute($run->fresh())->type)->toBe('advance')
        ->and($again['shell']->ran('systemctl restart wg-quick@wg0'))->toBeFalse()
        // Nor is the file rewritten for nothing.
        ->and($again['shell']->files())->not->toHaveKey('/etc/wireguard/wg0.conf');
});

it('actually applies a corrected hub key instead of only writing the file', function () {
    // An operator who fixes a wrong hub key and retries used to see the new file
    // on disk and no change in behaviour: nothing reloaded the interface, so the
    // running tunnel kept the old peer.
    $s = fakeServices();
    $s['shell']->script('wg pubkey', CommandResult::success('HOSTPUB='));
    scriptLiveTunnel($s['shell']);
    $s['shell']->script('ip link show wg0', CommandResult::success('3: wg0: <POINTOPOINT,UP>'));
    $s['shell']->script('systemctl is-enabled', CommandResult::success('enabled'));
    $s['shell']->script('cat \'/etc/wireguard/wg0.conf\'', CommandResult::success(
        "[Interface]\nAddress = 10.66.0.9/24\n\n[Peer]\nPublicKey = THEWRONGKEY=\n"
    ));
    $run = hostRun(Host::factory()->create());

    expect(app(ConfigureWireguard::class)->execute($run)->type)->toBe('advance')
        ->and($s['shell']->files()['/etc/wireguard/wg0.conf'])->toContain($s['hub']->publicKeyValue)
        ->and($s['shell']->ran('systemctl restart wg-quick@wg0'))->toBeTrue();
});

// --- InstallProxmoxVe ---

it('skips the install when proxmox-ve is already present', function () {
    $s = fakeServices();
    $s['shell']->script('dpkg -l proxmox-ve', CommandResult::success('ii  proxmox-ve'));
    $run = hostRun(Host::factory()->create());

    expect(app(InstallProxmoxVe::class)->execute($run)->type)->toBe('advance')
        ->and($s['shell']->ran('apt-get -y install proxmox-ve'))->toBeFalse();
});

it('installs proxmox-ve against the suite the host actually runs', function () {
    // A server ordered today boots Debian 13 trixie. The step used to hard-code
    // bookworm, i.e. add the PVE 8 repo and keyring to a trixie base — a
    // hypervisor built against a different libc and kernel.
    $s = fakeServices();
    $s['shell']->script('dpkg -l proxmox-ve', CommandResult::failure(1));
    scriptDebianRelease($s['shell'], 'debian', 'trixie', 'Debian GNU/Linux 13 (trixie)');
    $run = hostRun(Host::factory()->create());

    expect(app(InstallProxmoxVe::class)->execute($run)->type)->toBe('advance')
        ->and($s['shell']->ran('apt-get -y install proxmox-ve'))->toBeTrue();

    $sources = $s['shell']->files()['/etc/apt/sources.list.d/pve-install-repo.sources'];
    expect($sources)->toContain('Suites: trixie')
        ->and($sources)->not->toContain('bookworm')
        // Scoped to this repository via Signed-By, not trusted for every repo on
        // the machine the way /etc/apt/trusted.gpg.d would be.
        ->and($sources)->toContain('Signed-By: /usr/share/keyrings/proxmox-archive-keyring.gpg')
        ->and($s['shell']->ran('proxmox-archive-keyring-trixie.gpg'))->toBeTrue()
        ->and($s['shell']->ran('/etc/apt/trusted.gpg.d/'))->toBeFalse();
});

it('installs the bookworm suite on a bookworm host', function () {
    $s = fakeServices();
    $s['shell']->script('dpkg -l proxmox-ve', CommandResult::failure(1));
    scriptDebianRelease($s['shell'], 'debian', 'bookworm', 'Debian GNU/Linux 12 (bookworm)');
    $run = hostRun(Host::factory()->create());

    expect(app(InstallProxmoxVe::class)->execute($run)->type)->toBe('advance');

    $sources = $s['shell']->files()['/etc/apt/sources.list.d/pve-install-repo.sources'];
    expect($sources)->toContain('Suites: bookworm')
        ->and($sources)->not->toContain('trixie')
        ->and($sources)->toContain('Signed-By: /usr/share/keyrings/proxmox-release-bookworm.gpg')
        ->and($s['shell']->ran('proxmox-release-bookworm.gpg'))->toBeTrue();
});

it('refuses an unknown release by name rather than guessing a suite', function () {
    $s = fakeServices();
    $s['shell']->script('dpkg -l proxmox-ve', CommandResult::failure(1));
    scriptDebianRelease($s['shell'], 'debian', 'forky', 'Debian GNU/Linux 14 (forky)');
    $run = hostRun(Host::factory()->create());

    $result = app(InstallProxmoxVe::class)->execute($run);

    expect($result->type)->toBe('fail')
        // Names what it found, so an operator does not have to go and look.
        ->and($result->reason)->toContain('forky')
        ->and($result->reason)->toContain('Debian GNU/Linux 14 (forky)')
        // And nothing was installed on the guess that newest-is-closest.
        ->and($s['shell']->ran('apt-get -y install proxmox-ve'))->toBeFalse()
        ->and($s['shell']->files())->not->toHaveKey('/etc/apt/sources.list.d/pve-install-repo.sources');
});

it('refuses a base system that is not Debian at all', function () {
    $s = fakeServices();
    $s['shell']->script('dpkg -l proxmox-ve', CommandResult::failure(1));
    scriptDebianRelease($s['shell'], 'ubuntu', 'noble', 'Ubuntu 24.04.1 LTS');
    $run = hostRun(Host::factory()->create());

    $result = app(InstallProxmoxVe::class)->execute($run);

    expect($result->type)->toBe('fail')
        ->and($result->reason)->toContain('ubuntu')
        ->and($s['shell']->ran('apt-get -y install proxmox-ve'))->toBeFalse();
});

it('refuses an image whose os-release names no codename', function () {
    // Debian unstable has no VERSION_CODENAME, so this is the branch that must
    // not read "" as "close enough to the newest one I know".
    $s = fakeServices();
    $s['shell']->script('dpkg -l proxmox-ve', CommandResult::failure(1));
    scriptDebianRelease($s['shell'], 'debian', '', 'Debian GNU/Linux trixie/sid');
    $run = hostRun(Host::factory()->create());

    $result = app(InstallProxmoxVe::class)->execute($run);

    expect($result->type)->toBe('fail')
        ->and($result->reason)->toContain('(empty)')
        ->and($result->reason)->toContain('trixie/sid');
});

// --- RebootIntoPveKernel ---

it('issues the reboot on first execution', function () {
    $s = fakeServices();
    scriptBootableProxmoxKernel($s['shell']);
    $run = hostRun(Host::factory()->create());

    expect(app(RebootIntoPveKernel::class)->execute($run)->type)->toBe('poll')
        ->and($run->fresh()->context('reboot_issued'))->toBeTrue()
        ->and($s['shell']->ran('reboot'))->toBeTrue();
});

it('refuses to reboot a host with no Proxmox kernel installed', function () {
    // Mutation boundary: the reboot is the one irreversible thing this pipeline
    // does. `apt-get remove linux-image-amd64 || true` followed by an unchecked
    // update-grub could leave a machine that only the provider's console can
    // reach — so nothing may be issued on a guess.
    $s = fakeServices();
    $s['shell']->script('ls -1 /boot/vmlinuz-*-pve', CommandResult::failure(2));
    $run = hostRun(Host::factory()->create());

    $result = app(RebootIntoPveKernel::class)->execute($run);

    expect($result->type)->toBe('fail')
        ->and($result->reason)->toContain('vmlinuz-*-pve')
        ->and($s['shell']->ran('reboot'))->toBeFalse()
        ->and($run->fresh()->context('reboot_issued'))->toBeNull();
});

it('refuses to reboot when the Proxmox kernel has no initramfs', function () {
    // What a /boot that filled up during the full-upgrade actually leaves behind.
    $s = fakeServices();
    $s['shell']->script('ls -1 /boot/vmlinuz-*-pve', CommandResult::success("/boot/vmlinuz-6.8.12-4-pve\n"));
    $s['shell']->script('ls -1 /boot/initrd.img-*-pve', CommandResult::failure(2));
    $run = hostRun(Host::factory()->create());

    $result = app(RebootIntoPveKernel::class)->execute($run);

    expect($result->type)->toBe('fail')
        ->and($result->reason)->toContain('initrd.img-*-pve')
        ->and($s['shell']->ran('reboot'))->toBeFalse();
});

it('refuses to reboot when update-grub failed', function () {
    $s = fakeServices();
    scriptBootableProxmoxKernel($s['shell']);
    $s['shell']->script('update-grub', CommandResult::failure(1, 'cannot write /boot/grub/grub.cfg: No space left'));
    $run = hostRun(Host::factory()->create());

    $result = app(RebootIntoPveKernel::class)->execute($run);

    expect($result->type)->toBe('fail')
        ->and($result->reason)->toContain('update-grub')
        ->and($result->reason)->toContain('No space left')
        ->and($s['shell']->ran('reboot'))->toBeFalse();
});

it('advances once the proxmox kernel is up', function () {
    $s = fakeServices();
    $s['shell']->script('uname -r', CommandResult::success('6.8.12-4-pve'));
    $run = hostRun(Host::factory()->create(), [
        'reboot_issued' => true,
        'reboot_deadline' => now()->addMinutes(10)->toIso8601String(),
    ]);

    expect(app(RebootIntoPveKernel::class)->execute($run)->type)->toBe('advance');
});

it('keeps waiting while the old kernel is still running', function () {
    $s = fakeServices();
    $s['shell']->script('uname -r', CommandResult::success('6.1.0-18-amd64'));
    $run = hostRun(Host::factory()->create(), [
        'reboot_issued' => true,
        'reboot_deadline' => now()->addMinutes(10)->toIso8601String(),
    ]);

    expect(app(RebootIntoPveKernel::class)->execute($run)->type)->toBe('poll');
});

it('fails when the host does not return before the reboot deadline', function () {
    $s = fakeServices();
    $s['shell']->script('uname -r', CommandResult::success('6.1.0-18-amd64'));
    $run = hostRun(Host::factory()->create(), [
        'reboot_issued' => true,
        'reboot_deadline' => now()->subMinute()->toIso8601String(),
    ]);

    expect(app(RebootIntoPveKernel::class)->execute($run)->type)->toBe('fail')
        ->and($run->fresh()->context('reboot_issued'))->toBeNull(); // cleared so a retry re-issues
});

// --- CreateAutomationToken ---

it('creates and persists the automation token via pveum exactly once', function () {
    config()->set('admin_access.secrets_key', 'base64:'.base64_encode(random_bytes(32)));
    $s = fakeServices();
    $s['shell']->script('pveum user token add', CommandResult::success(
        json_encode(['full-tokenid' => 'automation@pve!clupilot', 'value' => 'tok-secret-123'])
    ));
    $host = Host::factory()->create();
    $run = hostRun($host);

    expect(app(CreateAutomationToken::class)->execute($run)->type)->toBe('advance');
    $host->refresh();
    expect($host->api_token_ref)->toContain('automation@pve!clupilot')
        ->and($host->api_token_ref)->toContain('tok-secret-123');

    $tokenCalls = fn () => count(array_filter($s['shell']->recorded(), fn ($c) => str_contains($c, 'pveum user token add')));
    expect($tokenCalls())->toBe(1);

    // Re-run: idempotent short-circuit, no second token creation.
    app(CreateAutomationToken::class)->execute($run->fresh());
    expect($tokenCalls())->toBe(1)
        ->and(RunResource::where('run_id', $run->id)->where('kind', 'pve_token')->count())->toBe(1);
});

it('grants Sys.Modify, which every customer backup job needs', function () {
    // POST /cluster/backup checks ['perm', '/', ['Sys.Modify']]. Without it the
    // call comes back 403, RegisterBackup rethrows anything whose body does not
    // contain "already", and EVERY customer provisioning run failed at
    // register_backup — long after this host looked fine.
    expect(config('provisioning.proxmox.role_privs'))->toContain('Sys.Modify');
});

it('converges the role privileges on a host that was onboarded before they changed', function () {
    // `pveum role add … || true` only ever applied the privilege list on the run
    // that first created the role, so a privilege added to the config reached new
    // hosts and no existing one. The convergence must also sit BEFORE the token
    // short-circuit, or a replay never reaches it.
    config()->set('admin_access.secrets_key', 'base64:'.base64_encode(random_bytes(32)));
    $s = fakeServices();
    $s['shell']->script('pveum role add', CommandResult::failure(1, 'role CluPilotAutomation already exists'));
    $host = Host::factory()->create(['api_token_ref' => 'automation@pve!clupilot=already-there']);
    $run = hostRun($host);
    RunResource::create([
        'run_id' => $run->id, 'host_id' => $host->id, 'kind' => 'pve_token',
        'external_id' => 'automation@pve!clupilot',
    ]);

    expect(app(CreateAutomationToken::class)->execute($run)->type)->toBe('advance');

    $modify = collect($s['shell']->recorded())->first(fn ($c) => str_contains($c, 'pveum role modify'));
    expect($modify)->not->toBeNull()
        ->and($modify)->toContain('Sys.Modify')
        // Replaces rather than appends: the host matches the config, not the
        // accumulation of everything any past version of it once granted.
        ->and($modify)->not->toContain('-append')
        // And the short-circuit still holds for the token itself.
        ->and($s['shell']->ran('pveum user token add'))->toBeFalse();
});

it('retries rather than minting a token the role privileges did not converge for', function () {
    $s = fakeServices();
    $s['shell']->script('pveum role', CommandResult::failure(1, 'permission denied'));
    $run = hostRun(Host::factory()->create());

    expect(app(CreateAutomationToken::class)->execute($run)->type)->toBe('retry')
        ->and($s['shell']->ran('pveum user token add'))->toBeFalse();
});

// --- ConfigureProxmox ---

it('refuses to finish a host that has no vmbr0 bridge', function () {
    // Proxmox installed on top of Debian does not create vmbr0 — only the ISO
    // installer does. The step used to run `ip link show vmbr0`, throw the result
    // away, and advance, so onboarding reported `active` on a host with no bridge
    // to attach a customer VM to.
    $s = fakeServices();
    $s['shell']->script('ip link show vmbr0', CommandResult::failure(1));
    $s['shell']->script('ip -4 route show default', CommandResult::success('default via 203.0.113.1 dev enp0s31f6'));
    $run = hostRun(Host::factory()->create());

    $result = app(ConfigureProxmox::class)->execute($run);

    expect($result->type)->toBe('fail')
        ->and($result->reason)->toContain('vmbr0')
        // Tells the operator what to do, including which NIC carries the route.
        ->and($result->reason)->toContain('/etc/network/interfaces')
        ->and($result->reason)->toContain('enp0s31f6')
        // And never tries to build the bridge itself: getting that wrong over the
        // very link the shell runs on takes the machine off the network for good.
        ->and($s['shell']->ran('ifreload'))->toBeFalse()
        ->and($s['shell']->ran('brctl'))->toBeFalse();
});

it('enables the datacenter firewall without locking the host out', function () {
    // Proxmox applies a guest's firewall rules only while the firewall is enabled
    // at datacenter level, so applyFirewall()'s "80/443 only" rules were inert on
    // every host. Enabling it blindly is the other trap: the datacenter input
    // policy defaults to DROP, the node firewall defaults to ON, and the only
    // sources it admits to 22/8006 come from an ipset seeded with the PUBLIC
    // subnet — not the tunnel every later step uses.
    $s = fakeServices();
    scriptHealthyProxmoxHost($s['shell']);
    $host = Host::factory()->create(['public_ip' => '203.0.113.40', 'wg_ip' => '10.66.0.40']);
    $run = hostRun($host);
    proveTunnel($run, $host);

    expect(app(ConfigureProxmox::class)->execute($run)->type)->toBe('advance');

    $commands = collect($s['shell']->recorded());
    $policy = $commands->search(fn ($c) => str_contains($c, '--policy_in ACCEPT'));
    $nodeOff = $commands->search(fn ($c) => str_contains($c, '/firewall/options --enable 0'));
    $enable = $commands->search(fn ($c) => str_contains($c, '/cluster/firewall/options --enable 1'));

    expect($policy)->not->toBeFalse()
        ->and($nodeOff)->not->toBeFalse()
        ->and($enable)->not->toBeFalse()
        // Order is the safety property: both mitigations land before the master
        // switch, so there is no window in which pve-firewall compiles a DROP
        // policy that cannot see the tunnel.
        ->and($policy)->toBeLessThan($enable)
        ->and($nodeOff)->toBeLessThan($enable);
});

it('does not believe a datacenter firewall write it cannot read back', function () {
    // /etc/pve is a replicated filesystem: `pvesh set` exiting 0 is not the same
    // as the setting being there, and a lost write means every customer VM's
    // rules go on being inert while onboarding reports success.
    $s = fakeServices();
    $s['shell']->script('pvesh get /cluster/firewall/options', CommandResult::success('{"enable":0}'));
    $run = hostRun(Host::factory()->create());

    expect(app(ConfigureProxmox::class)->execute($run)->type)->toBe('retry');
});

it('drops the enterprise repo the proxmox-ve package brings back with it', function () {
    $s = fakeServices();
    scriptHealthyProxmoxHost($s['shell']);
    $run = hostRun(Host::factory()->create());

    app(ConfigureProxmox::class)->execute($run);

    expect($s['shell']->ran('pve-enterprise.list'))->toBeTrue()
        // PVE 9 ships the deb822 spelling instead, and one of the two is always
        // the file that is actually there.
        ->and($s['shell']->ran('pve-enterprise.sources'))->toBeTrue();
});

// --- VerifyVmTemplate ---
//
// The seeded catalogue every installation starts with points all four plans at
// template_vmid 9000, which is exactly the situation this step exists for.

it('refuses to finish a host that has no template to clone from', function () {
    // A host used to go `active` while it could not serve a single customer:
    // every plan version points at a template_vmid, nothing in the pipeline
    // creates or verifies it, and there is no console command for it either — so
    // the first PAID order died in clone_vm, after the money had changed hands.
    $s = fakeServices();
    $run = hostRun(Host::factory()->active()->create());

    $result = app(VerifyVmTemplate::class)->execute($run);

    expect(PlanVersion::query()->whereNotNull('published_at')->where('template_vmid', 9000)->exists())->toBeTrue()
        ->and($result->type)->toBe('fail')
        ->and($result->reason)->toContain('9000')
        ->and($result->reason)->toContain('clone_vm')
        // It reports; it does not invent a golden image nobody chose.
        ->and($s['pve']->clonedVmids)->toBe([]);
});

it('advances once the template a published plan points at is on the node', function () {
    $s = fakeServices();
    $s['pve']->clonedVmids[] = 9000;   // the fake's stand-in for "this VMID exists"
    $s['pve']->templateVmids[] = 9000; // …and this one for "it reports template: 1"
    $run = hostRun(Host::factory()->active()->create());

    expect(app(VerifyVmTemplate::class)->execute($run)->type)->toBe('advance');
});

it('refuses a virtual machine that merely carries the template vmid', function () {
    // Existence was the wrong question. An abandoned build leaves a plain VM
    // numbered 9000 behind, and a check that asks `vmExists` calls that fine —
    // so the gap re-surfaces at the first PAID clone, which is the one failure
    // this step exists to move forward in time. `template: 1` is a property
    // only a real template has.
    $s = fakeServices();
    $s['pve']->clonedVmids[] = 9000; // exists…
    // …but templateVmids stays empty: it is not a template.
    $run = hostRun(Host::factory()->active()->create());

    expect(app(VerifyVmTemplate::class)->execute($run)->type)->toBe('fail');
});

it('still checks a version that is published but not on sale until next week', function () {
    // PlanVersion::available() would exclude it, and its template gap would then
    // surface at its first order rather than here — the whole failure this step
    // removes. A CLOSED window is different: nobody can buy it any more.
    $s = fakeServices();
    PlanVersion::query()->update(['available_from' => now()->addWeek()]);
    $run = hostRun(Host::factory()->active()->create());

    expect(app(VerifyVmTemplate::class)->execute($run)->type)->toBe('fail');

    PlanVersion::query()->update(['available_until' => now()->subDay()]);
    expect(app(VerifyVmTemplate::class)->execute($run)->type)->toBe('advance')
        ->and($s['pve']->clonedVmids)->toBe([]);
});

it('requires no template on an installation that has not published a plan yet', function () {
    fakeServices();
    PlanVersion::query()->update(['published_at' => null]);
    $run = hostRun(Host::factory()->active()->create());

    expect(app(VerifyVmTemplate::class)->execute($run)->type)->toBe('advance');
});

// --- BuildVmTemplate ---
//
// The step that closes the last manual handle in the takeover: VerifyVmTemplate
// reports a missing template, this one builds it — by shipping the SAME
// deploy/bootstrap/lib/template.sh that the rescue path uses, never a second
// copy of its checks in PHP.

/** What the host answers when asked about a build in flight. */
function scriptBuildStatus(FakeRemoteShell $shell, string $state, bool $alive = false, string $phase = '', string $note = ''): void
{
    $shell->script('clupilot-template-status', CommandResult::success(
        "state={$state}\nalive=".($alive ? 'yes' : 'no')."\nphase={$phase}\nnote={$note}\n"
    ));
}

it('builds nothing on an installation that has not published a plan yet', function () {
    $s = fakeServices();
    PlanVersion::query()->update(['published_at' => null]);
    $run = hostRun(Host::factory()->active()->create());

    expect(app(BuildVmTemplate::class)->execute($run)->type)->toBe('advance')
        ->and($s['shell']->files())->toBe([])
        ->and($s['shell']->ran('clupilot-template-start'))->toBeFalse();
});

it('leaves a template that is already built alone', function () {
    $s = fakeServices();
    $s['pve']->templateVmids[] = 9000;
    $run = hostRun(Host::factory()->active()->create());

    // Twenty minutes and a `qm destroy --purge` is what NOT skipping costs, and
    // this step runs again on every retry and every later maintenance run.
    expect(app(BuildVmTemplate::class)->execute($run)->type)->toBe('advance')
        ->and($s['shell']->ran('clupilot-template-start'))->toBeFalse();
});

it('builds when the vmid exists but is not a template', function () {
    // The half-built leftover. Asking `vmExists` here would inherit exactly the
    // weakness VerifyVmTemplate just lost, one step earlier.
    $s = fakeServices();
    $s['pve']->clonedVmids[] = 9000;
    $run = hostRun(Host::factory()->active()->create());

    expect(app(BuildVmTemplate::class)->execute($run)->type)->toBe('poll')
        ->and($s['shell']->ran('clupilot-template-start'))->toBeTrue();
});

it('ships the template script verbatim and starts it detached', function () {
    $s = fakeServices();
    $run = hostRun(Host::factory()->active()->create());

    $result = app(BuildVmTemplate::class)->execute($run);
    $files = $s['shell']->files();

    expect($result->type)->toBe('poll')
        // Verbatim, byte for byte. The moment this is rendered from a PHP
        // template instead of read from the file, there are two fassungen of
        // the three traps again — the one thing the design forbids.
        ->and($files['/var/lib/clupilot/template-build/template.sh'] ?? null)
        ->toBe(file_get_contents(base_path('deploy/bootstrap/lib/template.sh')))
        ->and($files['/var/lib/clupilot/template-build/template-run.sh'] ?? null)
        ->toBe(file_get_contents(base_path('deploy/bootstrap/lib/template-run.sh')))
        ->and($files['/var/lib/clupilot/template-build/docker-compose.yml'] ?? null)
        ->toBe(file_get_contents(base_path('deploy/bootstrap/assets/docker-compose.yml')))
        // The VMIDs the catalogue actually asks for, not a number baked in here.
        ->and($files['/var/lib/clupilot/template-build/env'] ?? '')->toContain('9000')
        ->and($s['shell']->ran('nohup'))->toBeTrue();
});

it('keeps polling while the build process is alive', function () {
    $s = fakeServices();
    scriptBuildStatus($s['shell'], 'running', alive: true, phase: 'umbauen');
    $run = hostRun(Host::factory()->active()->create(), ['template_build_deadline' => now()->addMinutes(30)->toIso8601String()]);

    $result = app(BuildVmTemplate::class)->execute($run);

    // Starting a second one means two virt-customize on one file, and both of
    // them calling `qm create 9000`.
    expect($result->type)->toBe('poll')
        ->and($result->reason)->toContain('umbauen')
        ->and($s['shell']->ran('clupilot-template-start'))->toBeFalse();
});

it('fails at once when the state file says running but the process is gone', function () {
    // A reboot, an OOM kill, an operator's Ctrl-C. The file still says running
    // and always will — nobody is left to change it. Polling on the strength of
    // that word waits forty-five minutes against a corpse.
    $s = fakeServices();
    scriptBuildStatus($s['shell'], 'running', alive: false);
    $run = hostRun(Host::factory()->active()->create(), ['template_build_deadline' => now()->addMinutes(30)->toIso8601String()]);

    expect(app(BuildVmTemplate::class)->execute($run)->type)->toBe('fail');
});

it('fails when a build that was started left no trace behind', function () {
    // The start command writes `state` itself, in the same breath as launching,
    // so an empty status on a LATER visit is not "not started yet" — it is a
    // start that did not take (a full disk, a wiped work dir). Reading it as
    // "not started yet" re-uploads and re-launches every thirty seconds for an
    // hour, because poll() never spends the retry budget.
    $s = fakeServices();
    $run = hostRun(Host::factory()->active()->create(), ['template_build_deadline' => now()->addMinutes(30)->toIso8601String()]);

    expect(app(BuildVmTemplate::class)->execute($run)->type)->toBe('fail')
        ->and($s['shell']->ran('clupilot-template-start'))->toBeFalse();
});

it('lets an operator retry a failed build from scratch', function () {
    // giveUp() clears the deadline as well as the state — otherwise the check
    // above turns every Retry after a failure into an instant second failure,
    // without the host being touched at all.
    $s = fakeServices();
    scriptBuildStatus($s['shell'], 'failed', note: 'kein Platz');
    $run = hostRun(Host::factory()->active()->create(), ['template_build_deadline' => now()->addMinutes(30)->toIso8601String()]);

    app(BuildVmTemplate::class)->execute($run);

    expect($run->fresh()->context('template_build_deadline'))->toBeNull();
});

it('reports the reason the build itself wrote, and clears the state for a retry', function () {
    $s = fakeServices();
    scriptBuildStatus($s['shell'], 'failed', note: 'kein Proxmox-Speicher gefunden, der Platten aufnimmt');
    $run = hostRun(Host::factory()->active()->create(), ['template_build_deadline' => now()->addMinutes(30)->toIso8601String()]);

    $result = app(BuildVmTemplate::class)->execute($run);

    expect($result->type)->toBe('fail')
        ->and($result->reason)->toContain('Platten aufnimmt')
        // Otherwise the operator's Retry reads the old verdict and fails again
        // without the host being touched.
        ->and($s['shell']->ran('clupilot-template-reset'))->toBeTrue();
});

it('refuses to advance when the build says ok but no template is there', function () {
    // The script's own last check and this one are deliberately both present:
    // one asks the host, the other asks the API the customer pipeline will use.
    $s = fakeServices();
    scriptBuildStatus($s['shell'], 'ok');
    $run = hostRun(Host::factory()->active()->create(), ['template_build_deadline' => now()->addMinutes(30)->toIso8601String()]);

    expect(app(BuildVmTemplate::class)->execute($run)->type)->toBe('fail');
});

it('advances once the finished build is visible as a template through the api', function () {
    $s = fakeServices();
    $s['pve']->templateVmids[] = 9000;
    scriptBuildStatus($s['shell'], 'ok');
    $run = hostRun(Host::factory()->active()->create(), ['template_build_deadline' => now()->addMinutes(30)->toIso8601String()]);

    expect(app(BuildVmTemplate::class)->execute($run)->type)->toBe('advance');
});

it('gives up on a build that outlives its deadline', function () {
    $s = fakeServices();
    scriptBuildStatus($s['shell'], 'running', alive: true);
    $run = hostRun(Host::factory()->active()->create(), ['template_build_deadline' => now()->subMinute()->toIso8601String()]);

    expect(app(BuildVmTemplate::class)->execute($run)->type)->toBe('fail');
});

it('stops a build it is giving up on, instead of leaving it running', function () {
    // Giving up while the process is still alive and only deleting the status
    // files makes the run retryable while the old build carries on: two
    // virt-customize on one build.qcow2, and both of them calling
    // `qm destroy 9000 --purge` followed by `qm create 9000`.
    $s = fakeServices();
    scriptBuildStatus($s['shell'], 'running', alive: true);
    $run = hostRun(Host::factory()->active()->create(), ['template_build_deadline' => now()->subMinute()->toIso8601String()]);

    app(BuildVmTemplate::class)->execute($run);

    expect($s['shell']->ran('clupilot-template-reset'))->toBeTrue()
        // The whole process group, not just the driver: killing `sh` alone
        // leaves the virt-customize it is waiting on chewing on the image.
        ->and($s['shell']->ran('kill -TERM'))->toBeTrue();
});

it('ignores a finished status left behind by an earlier run', function () {
    // Status files outlive the run that wrote them. A later run — a new
    // template in the catalogue, a maintenance pass — finds `ok` on disk from
    // the build that produced the OTHER template. Read as this run's own
    // result, it says "the build I started finished without producing what I
    // asked for", and fails a host that only needed a build to be started.
    $s = fakeServices();
    scriptBuildStatus($s['shell'], 'ok');
    $run = hostRun(Host::factory()->active()->create()); // kein Termin: dieser Lauf hat nichts gestartet

    expect(app(BuildVmTemplate::class)->execute($run)->type)->toBe('poll')
        ->and($s['shell']->ran('clupilot-template-start'))->toBeTrue();
});

it('adopts a build another run already has in flight', function () {
    // The one status from an earlier run that must NOT be overwritten: a live
    // process building the very same thing. Starting beside it is the two-on-
    // one-image collision again, arrived at from the other side.
    $s = fakeServices();
    scriptBuildStatus($s['shell'], 'running', alive: true, phase: 'umbauen');
    $run = hostRun(Host::factory()->active()->create());

    expect(app(BuildVmTemplate::class)->execute($run)->type)->toBe('poll')
        ->and($s['shell']->ran('clupilot-template-start'))->toBeFalse()
        // Adopted with a deadline of its own, so it cannot be polled forever.
        ->and($run->fresh()->context('template_build_deadline'))->not->toBeNull();
});

it('sends only shell that a shell can actually parse', function () {
    // Every other test here reads these commands as strings — FakeRemoteShell
    // never runs one. So a stray quote in the start or reset command would
    // reach a real host as the first thing anybody executes, and the step it
    // breaks is the one that destroys and rebuilds VMs. `sh -n` parses without
    // executing, which is exactly the reassurance that is missing.
    $s = fakeServices();
    $host = Host::factory()->active()->create();

    app(BuildVmTemplate::class)->execute(hostRun($host));                 // status + start
    scriptBuildStatus($s['shell'], 'failed', note: 'nope');
    // Mit Termin, damit der Status als DIESES Laufs gelesen wird und der
    // Aufgeben-Zweig (Protokoll lesen, Prozessgruppe beenden) wirklich läuft.
    app(BuildVmTemplate::class)->execute(
        hostRun($host, ['template_build_deadline' => now()->addMinutes(30)->toIso8601String()])
    );                                                                    // status + log + reset

    $checked = 0;
    foreach ($s['shell']->recorded() as $command) {
        expect(Process::input($command)->run('sh -n')->successful())
            ->toBeTrue("not valid POSIX shell:\n".$command);
        $checked++;
    }

    expect($checked)->toBeGreaterThanOrEqual(4);
});

it('rebuilds only the template that is actually missing', function () {
    // Two published packages pointing at two different templates, one of them
    // already built. Handing both VMIDs to the script destroys the healthy one
    // on the way past — create_proxmox_template purges before it creates — and
    // if the rebuild then fails, a template that was fine is simply gone.
    $s = fakeServices();
    $family = PlanFamily::query()->create(['key' => 'zweite-vorlage', 'name' => 'Zweite Vorlage', 'tier' => 9]);
    $version = $family->versions()->create([
        'version' => 1, 'quota_gb' => 10, 'traffic_gb' => 100, 'seats' => 1, 'ram_mb' => 1024,
        'cores' => 1, 'disk_gb' => 20, 'performance' => 'standard', 'features' => [], 'available_from' => now(),
    ]);
    $version->update(['published_at' => now(), 'template_vmid' => 9001]);

    $s['pve']->templateVmids[] = 9000; // die gesunde, vorhandene Vorlage
    $run = hostRun(Host::factory()->active()->create());

    $result = app(BuildVmTemplate::class)->execute($run);
    $env = $s['shell']->files()['/var/lib/clupilot/template-build/env'] ?? '';

    expect($result->type)->toBe('poll')
        ->and($env)->toContain('9001')
        ->and($env)->not->toContain('9000');
});

// --- VerifyProxmoxApi ---

it('advances when the api returns nodes', function () {
    fakeServices();
    expect(app(VerifyProxmoxApi::class)->execute(hostRun(Host::factory()->create()))->type)->toBe('advance');
});

it('retries when the api returns no nodes yet', function () {
    $s = fakeServices();
    $s['pve']->nodes = [];
    expect(app(VerifyProxmoxApi::class)->execute(hostRun(Host::factory()->create()))->type)->toBe('retry');
});

// --- RegisterCapacity ---

it('registers node capacity on the host', function () {
    fakeServices();
    $host = Host::factory()->create();
    $run = hostRun($host);

    expect(app(RegisterCapacity::class)->execute($run)->type)->toBe('advance');
    $host->refresh();
    expect($host->cpu_cores)->toBe(16)
        ->and($host->total_ram_mb)->toBe(65536)
        ->and($host->total_gb)->toBe(1024)
        ->and($host->last_seen_at)->not->toBeNull();
});

// --- SecureHostFirewall ---

it('places SecureHostFirewall after every other host step and before completion', function () {
    $pipeline = config('provisioning.pipelines.host');

    $secureIndex = array_search(SecureHostFirewall::class, $pipeline, true);
    $capacityIndex = array_search(RegisterCapacity::class, $pipeline, true);
    $completeIndex = array_search(CompleteHostOnboarding::class, $pipeline, true);

    expect($secureIndex)->not->toBeFalse()
        ->and($secureIndex)->toBe($capacityIndex + 1)
        ->and($secureIndex)->toBe($completeIndex - 1);
});

it('refuses to apply the firewall when the tunnel is not confirmed up from the host', function () {
    // Mutation boundary: a host whose tunnel does not verify must come away
    // completely untouched — retrying is not enough on its own, nothing may
    // have been written either.
    $s = fakeServices();
    $s['shell']->script('ping', CommandResult::failure(1));
    $host = Host::factory()->active()->create();
    $run = hostRun($host);

    expect(app(SecureHostFirewall::class)->execute($run)->type)->toBe('retry');

    expect($s['shell']->files())->not->toHaveKey('/etc/nftables.conf')
        ->and($s['shell']->ran('apt-get install -y nftables'))->toBeFalse()
        ->and(RunResource::where('run_id', $run->id)->where('kind', 'host_firewall')->exists())->toBeFalse();
});

it('applies the firewall once the tunnel is confirmed up, over the tunnel itself', function () {
    $s = fakeServices();
    $host = Host::factory()->active()->create(['public_ip' => '203.0.113.30']);
    $run = hostRun($host);
    proveTunnel($run, $host);

    expect(app(SecureHostFirewall::class)->execute($run)->type)->toBe('advance');

    // Connected over the tunnel, not the public address.
    foreach ($s['shell']->connectionsWith('key') as $connection) {
        expect($connection[1])->toBe($host->wg_ip);
    }

    $wgSubnet = (string) config('provisioning.wireguard.subnet');
    $config = $s['shell']->files()['/etc/nftables.conf'];
    expect($config)->toContain('policy drop')
        ->and($config)->toContain('tcp dport { 80, 443 } accept')
        ->and($config)->toContain('ip saddr '.$wgSubnet.' tcp dport { 22, 8006 } accept')
        ->and($config)->toContain('ct state established,related accept');

    expect($s['shell']->files())->toHaveKey('/usr/local/sbin/clupilot-emergency-open-firewall.sh')
        ->and($s['shell']->ran('chmod 700 \'/usr/local/sbin/clupilot-emergency-open-firewall.sh\''))->toBeTrue()
        ->and($s['shell']->ran('systemctl enable --now nftables'))->toBeTrue()
        ->and(RunResource::where('run_id', $run->id)->where('kind', 'host_firewall')->count())->toBe(1);
});

it('closes 8006 and 22 to everything that is not the management subnet', function () {
    // The owner's stated requirement, and the half nothing asserted: the old
    // tests only checked that the WireGuard-scoped ACCEPT was PRESENT, so a
    // mutation adding a bare `tcp dport { 8006 } accept` passed the whole suite
    // while handing every scanner on the internet a Proxmox login form.
    //
    // Read as "which destination ports does this ruleset accept from a source it
    // has not restricted to the tunnel", rather than as a search for one
    // expected line — that is what makes an ADDED rule fail it.
    $s = fakeServices();
    $host = Host::factory()->active()->create();
    $run = hostRun($host);
    proveTunnel($run, $host);

    app(SecureHostFirewall::class)->execute($run);

    $wgSubnet = (string) config('provisioning.wireguard.subnet');
    $config = $s['shell']->files()['/etc/nftables.conf'];

    $worldOpen = [];
    $tunnelOnly = [];
    foreach (nftAcceptRules($config) as $rule) {
        // ICMP and the loopback/conntrack rules carry no destination port.
        foreach (nftAcceptedPorts($rule) as $port) {
            if (str_contains($rule, 'ip saddr '.$wgSubnet)) {
                $tunnelOnly[] = $port;
            } else {
                $worldOpen[] = $port;
            }
        }
    }

    sort($worldOpen);
    sort($tunnelOnly);

    expect($tunnelOnly)->toBe([22, 8006])
        // Exactly these and nothing else: public web, and the DHCP client reply
        // below. 22 and 8006 must NOT appear here.
        ->and($worldOpen)->toBe([68, 80, 443])
        ->and($worldOpen)->not->toContain(8006)
        ->and($worldOpen)->not->toContain(22)
        // A DHCP reply, not "anything to port 68".
        ->and($config)->toContain('udp sport 67 udp dport 68 accept')
        // And the chain still ends in drop, so an unmatched packet is refused
        // rather than accepted by omission.
        ->and($config)->toContain('policy drop')
        ->and($config)->not->toContain('policy accept');
});

it('accepts the ICMP a working host cannot do without', function () {
    // policy drop with no ICMP accept at all — grepped, zero mentions. On a real
    // host that breaks IPv6 outright as soon as the neighbour cache expires (NDP
    // 135/136 and router advertisements arrive in the INPUT chain) and
    // black-holes large transfers, because path-MTU discovery runs on
    // frag-needed / packet-too-big. Debian's own example ruleset accepts both.
    $s = fakeServices();
    $host = Host::factory()->active()->create();
    $run = hostRun($host);
    proveTunnel($run, $host);

    app(SecureHostFirewall::class)->execute($run);

    $config = $s['shell']->files()['/etc/nftables.conf'];
    $icmp = implode("\n", array_filter(
        nftAcceptRules($config),
        fn (string $rule) => str_contains($rule, 'icmp'),
    ));

    expect($icmp)->toContain('nd-neighbor-solicit')
        ->and($icmp)->toContain('nd-neighbor-advert')
        ->and($icmp)->toContain('nd-router-advert')
        // Path MTU discovery, both families.
        ->and($icmp)->toContain('packet-too-big')
        ->and($icmp)->toContain('destination-unreachable')
        // Reachability, including the handshake check this very step performs.
        ->and($icmp)->toContain('echo-request')
        // Still a policy-drop ruleset, not "ICMP made it open".
        ->and($config)->toContain('policy drop');
});

it("keeps the WireGuard interface's established connections working (never a bare drop-all)", function () {
    $s = fakeServices();
    $host = Host::factory()->active()->create();
    $run = hostRun($host);
    proveTunnel($run, $host);

    app(SecureHostFirewall::class)->execute($run);

    $config = $s['shell']->files()['/etc/nftables.conf'];
    expect($config)->toContain('ct state established,related accept')
        ->and($config)->toContain('iif "lo" accept');
});

it('deploys an emergency script that never reopens the firewall by itself', function () {
    // The owner's explicit requirement, and what nothing tested: the old
    // assertion was `ran('/usr/local/sbin/clupilot-emergency-open-firewall.sh')`,
    // which putFile() does not record — the only thing that substring matched was
    // the `chmod 700` on the line above, so the same fact was asserted twice and
    // the script's CONTENTS were never looked at.
    $s = fakeServices();
    $host = Host::factory()->active()->create();
    $run = hostRun($host);
    proveTunnel($run, $host);

    app(SecureHostFirewall::class)->execute($run);

    $script = $s['shell']->files()['/usr/local/sbin/clupilot-emergency-open-firewall.sh'];

    // It does the one thing it exists for.
    expect($script)->toStartWith('#!/bin/sh')
        ->and($script)->toContain('systemctl disable --now nftables')
        ->and($script)->toContain('nft flush ruleset');

    // And nothing in it can fire without a human at the out-of-band console. A
    // firewall that reopens itself under failure is not a firewall.
    foreach (['sleep', 'systemd-run', 'OnCalendar', 'OnBootSec', '.timer', 'crontab', 'cron.d', 'at now'] as $selfTrigger) {
        expect($script)->not->toContain($selfTrigger);
    }

    // Nothing detached into the background either, which is the other way a
    // script leaves something running after the operator has walked away.
    foreach (preg_split('/\R/', $script) ?: [] as $line) {
        $line = trim($line);
        if ($line === '' || str_starts_with($line, '#')) {
            continue;
        }
        expect($line)->not->toEndWith('&');
    }

    // Nor does the step arrange one on its behalf.
    expect($s['shell']->ran('systemd-run'))->toBeFalse()
        ->and($s['shell']->ran('crontab'))->toBeFalse()
        ->and($s['shell']->ran('.timer'))->toBeFalse();
});

it('never reapplies the firewall on a re-run', function () {
    $s = fakeServices();
    $host = Host::factory()->active()->create();
    $run = hostRun($host);
    proveTunnel($run, $host);

    app(SecureHostFirewall::class)->execute($run);
    app(SecureHostFirewall::class)->execute($run->fresh());

    $installCalls = fn () => count(array_filter(
        $s['shell']->recorded(),
        fn ($c) => str_contains($c, 'apt-get install -y nftables'),
    ));

    expect($installCalls())->toBe(1)
        ->and(RunResource::where('run_id', $run->id)->where('kind', 'host_firewall')->count())->toBe(1);
});

// --- CompleteHostOnboarding ---

it('marks the host active on completion', function () {
    $host = Host::factory()->create(['status' => 'onboarding']);

    expect(app(CompleteHostOnboarding::class)->execute(hostRun($host))->type)->toBe('advance')
        ->and($host->fresh()->status)->toBe('active');
});

it('publishes the name it was given, and points it at the tunnel, not at the public address', function () {
    // The zone is pinned here rather than read from the machine's own .env. It
    // used to assert the literal clupilot.com, which was true until somebody
    // set CLUPILOT_PLATFORM_ZONE to clupilot.cloud on their installation — and
    // then this test failed on a correct configuration. What it is about is
    // the SHAPE of the name and which address it carries, not which domain
    // this particular box is configured for.
    config()->set('provisioning.dns.platform_zone', 'clupilot.com');

    $s = fakeServices();
    $host = Host::factory()->active()->create([
        'datacenter' => 'fsn',
        'wg_ip' => '10.66.0.11',
        'public_ip' => '203.0.113.11',
        'name' => 'fsn-01',
    ]);
    $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);

    $result = (new RegisterHostDns($s['hostDns']))->execute($run);

    expect($result->type)->toBe('advance')
        // Kein Beweis für den Schritt selbst — der Name steht schon vor
        // dem Aufruf fest, also bliebe diese Zeile auch grün, wenn der
        // Schritt ihn einfach nur unangetastet ließe. Sie ist ein
        // Stolperdraht: bekäme RegisterHostDns wieder ein `$host->update([
        // 'name' => ...])` verpasst (wie früher `dns_name`), müsste ein
        // ANDERER Wert hier auffallen, sobald jemand den Namen tatsächlich
        // ändert.
        ->and($host->fresh()->name)->toBe('fsn-01');

    // The management address: publishing a Proxmox host's public IP would hand
    // every scanner a target. Written to the internal hostsdir, not upserted
    // anywhere via the (public) Hetzner client — see the mutation test below.
    expect($s['hostDns']->fqdns['fsn-01'])->toBe('fsn-01.node.clupilot.com')
        ->and($s['hostDns']->ips['fsn-01'])->toBe('10.66.0.11')
        ->and($s['hostDns']->ips['fsn-01'])->not->toBe('203.0.113.11');
});

it('never reaches the public Hetzner DNS client for a host name', function () {
    // Mutation boundary: if RegisterHostDns is ever changed to also (or
    // instead) call the public client, this must fail. Verified by hand —
    // temporarily adding a `$this->publicDns->upsertRecord(...)` call to the
    // step and confirming this test goes red before reverting it.
    $s = fakeServices();
    $host = Host::factory()->active()->create([
        'datacenter' => 'fsn', 'wg_ip' => '10.66.0.12', 'name' => 'fsn-02',
    ]);
    $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);

    (new RegisterHostDns($s['hostDns']))->execute($run);

    expect($s['dns']->records)->toBe([])
        ->and($s['dns']->values)->toBe([]);
});

it('retries a failed DNS registration before giving up on the name', function () {
    $s = fakeServices();
    $s['hostDns']->failWrite = true;
    $host = Host::factory()->active()->create(['datacenter' => 'fsn', 'wg_ip' => '10.66.0.40', 'name' => 'fsn-04']);
    $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host', 'attempt' => 0]);

    $step = new RegisterHostDns($s['hostDns']);

    // The failure may be a lost response to a request that did create the
    // record — advancing straight away would orphan it.
    expect($step->execute($run)->type)->toBe('retry');

    // Out of attempts, the host still finishes: a name is convenience.
    $run->update(['attempt' => 5]);
    expect($step->execute($run)->type)->toBe('advance')
        ->and($run->events()->where('step', 'register_host_dns')->exists())->toBeTrue();
});

it("takes the host's internal DNS entry with it when the host is purged", function () {
    $s = fakeServices();

    $host = Host::factory()->active()->create([
        'datacenter' => 'fsn', 'wg_ip' => '10.66.0.50', 'name' => 'fsn-05',
    ]);
    $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);
    (new RegisterHostDns($s['hostDns']))->execute($run);

    $name = $host->name;
    expect($s['hostDns']->ips)->toHaveKey($name);

    (new PurgeHost($host->uuid))->handle();

    // Otherwise the machine's management address stays in the hostsdir for
    // good, with the name needed to remove it deleted along with the row.
    expect($s['hostDns']->ips)->not->toHaveKey($name);
});

it('keeps the host until its internal DNS entry is really gone', function () {
    $s = fakeServices();

    $host = Host::factory()->active()->create([
        'datacenter' => 'hel', 'wg_ip' => '10.66.0.60', 'name' => 'hel-01',
    ]);
    $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);
    (new RegisterHostDns($s['hostDns']))->execute($run);

    $s['hostDns']->failRemove = true;

    // The row carries the only name that maps to that entry — deleting it
    // anyway would leave the machine's management address in the hostsdir
    // for good.
    expect(fn () => (new PurgeHost($host->uuid))->handle())
        ->toThrow(RuntimeException::class);

    expect(Host::query()->whereKey($host->id)->exists())->toBeTrue();

    // Once the hostsdir write is back, the purge completes.
    $s['hostDns']->failRemove = false;
    (new PurgeHost($host->uuid))->handle();

    expect(Host::query()->whereKey($host->id)->exists())->toBeFalse()
        ->and($s['hostDns']->ips)->toBe([]);
});

it('still removes a legacy public Hetzner record when a pre-fix host is purged', function () {
    // Hosts onboarded before this change may still carry a public record id.
    // PurgeHost has always cleaned that up on removal — this keeps working
    // for those, independent of clupilot:prune-host-dns, which only handles
    // hosts that are NOT being deleted.
    $s = fakeServices();
    $recordId = $s['dns']->upsertRecord('fsn-legacy.node.clupilot.cloud', 'A', '10.66.0.55');
    $host = Host::factory()->active()->create([
        'datacenter' => 'fsn', 'wg_ip' => '10.66.0.55',
        'name' => 'fsn-legacy', 'dns_record_id' => $recordId,
    ]);

    expect($s['dns']->records)->toHaveKey('fsn-legacy.node.clupilot.cloud');

    (new PurgeHost($host->uuid))->handle();

    expect($s['dns']->records)->not->toHaveKey('fsn-legacy.node.clupilot.cloud');
});

it('sees an internal DNS entry registered while the purge was waiting', function () {
    $s = fakeServices();

    $host = Host::factory()->active()->create([
        'datacenter' => 'fsn', 'wg_ip' => '10.66.0.90', 'name' => 'fsn-06',
    ]);
    $purge = new PurgeHost($host->uuid);

    // Registration finishes while the purge is blocked on the run lock: the
    // copy it loaded first still says there is no entry to remove.
    $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']);
    (new RegisterHostDns($s['hostDns']))->execute($run);
    $name = $host->name;

    $purge->handle();

    expect($s['hostDns']->ips)->not->toHaveKey($name)
        ->and(Host::query()->whereKey($host->id)->exists())->toBeFalse();
});

it('does not call a tunnel up that has never handshaken', function () {
    // `0` ist WireGuards Antwort für „noch nie". Die Prüfung davor war ein
    // `ping` auf die Hub-Adresse — die fragte drei Dinge auf einmal und nannte
    // nur eines: ob der Handshake steht, ob ICMP durchkommt, und ob es `ping`
    // überhaupt GIBT. Auf Hetzners `Debian-…-base` gibt es das nicht, und der
    // Schritt scheiterte fünfmal über einem Tunnel, der stehen konnte.
    $s = fakeServices();
    $s['shell']->script('wg pubkey', CommandResult::success('HOSTPUB='));
    $s['shell']->script('wg show wg0 latest-handshakes', CommandResult::success(time()."\n".trim(app(WireguardHub::class)->publicKey())."\t0"));
    $run = hostRun(Host::factory()->create());

    expect(app(ConfigureWireguard::class)->execute($run)->type)->toBe('retry');
});

it('gives a host an address from the host range, not the next free one', function () {
    // Fortlaufend vergeben landete der erste Host zwischen zwei Notebooks. Wer
    // eine Adresse in einem Protokoll liest, soll sehen, ob dahinter ein Mensch
    // oder eine Maschine steht.
    $s = fakeServices();
    $s['shell']->script('wg pubkey', CommandResult::success('HOSTPUB='));
    scriptLiveTunnel($s['shell']);
    $host = Host::factory()->create(['wg_ip' => null]);

    app(ConfigureWireguard::class)->execute(hostRun($host));

    expect((int) Str::afterLast($host->fresh()->wg_ip, '.'))
        ->toBeGreaterThanOrEqual(config('provisioning.wireguard.host_offset'));
});

it('does not accept a handshake that has gone stale', function () {
    // `> 0` allein hiesse „hat jemals" (Codex P1): WireGuard behält den
    // Zeitstempel unbegrenzt. Ein Wiederholungslauf über einem längst toten
    // Tunnel meldete damit „steht" — und die Schritte danach wählten die
    // Tunneladresse für SSH, über die niemand mehr hinkommt.
    $s = fakeServices();
    $s['shell']->script('wg pubkey', CommandResult::success('HOSTPUB='));
    $now = time();
    $s['shell']->script('wg show wg0 latest-handshakes', CommandResult::success(
        $now."\n".trim(app(WireguardHub::class)->publicKey())."\t".($now - 3600)
    ));
    $run = hostRun(Host::factory()->create());

    expect(app(ConfigureWireguard::class)->execute($run)->type)->toBe('retry');
});

it('ignores a stranger on wg0 and asks only about the configured hub', function () {
    // Ein fremder Peer auf der Schnittstelle ist kein Beweis dafür, dass WIR
    // erreichbar sind.
    $s = fakeServices();
    $s['shell']->script('wg pubkey', CommandResult::success('HOSTPUB='));
    $now = time();
    $s['shell']->script('wg show wg0 latest-handshakes', CommandResult::success(
        $now."\n".'FREMDERPEER0000000000000000000000000000000='."\t".$now
    ));
    $run = hostRun(Host::factory()->create());

    expect(app(ConfigureWireguard::class)->execute($run)->type)->toBe('retry');
});

it('names the reason pveum gave instead of only that it failed', function () {
    // Fünf Wiederholungen mit "could not converge the Proxmox automation role
    // privileges" sagten, WAS scheiterte, und verschwiegen das Einzige, was
    // weiterhilft: welches Privileg diese Proxmox-Fassung nicht kennt. Der
    // Betreiber musste sich auf den Host melden und den Befehl von Hand
    // nachstellen, um zu erfahren, was daneben schon gestanden hatte.
    $s = fakeServices();
    $s['shell']->script('pveum role add', CommandResult::failure(2, 'role CluPilotAutomation already exists'));
    $s['shell']->script('pveum role modify', CommandResult::failure(2, "unknown privilege 'VM.Snapshot.Rollback'\nUSAGE: pveum role modify …"));
    $run = hostRun(Host::factory()->create());

    $result = app(CreateAutomationToken::class)->execute($run);

    expect($result->type)->toBe('retry')
        ->and($result->reason)->toContain("unknown privilege 'VM.Snapshot.Rollback'")
        // Nur die erste Zeile: die Aufrufhilfe danach wäre ein Bildschirm Text
        // in einer Ereigniszeile.
        ->and($result->reason)->not->toContain('USAGE');
});

it('asks only for privileges Proxmox 9 still knows', function () {
    // `pveum role modify` lehnt die GANZE Liste ab, sobald ein Name darin
    // unbekannt ist. `VM.Monitor` gibt es in PVE 9 nicht mehr, und die
    // Übernahme des ersten PVE-9-Hosts blieb daran hängen — fünf
    // Wiederholungen mit einer Meldung, die den Namen nicht nannte.
    //
    // Die Liste ist die Verabredung zwischen CluPilot und jedem Host, den es je
    // übernimmt. Sie gegen die gültigen Namen zu halten, ist billiger als der
    // Abbruch auf einer Maschine, die schon halb eingerichtet ist.
    $gueltig = [
        'Datastore.Allocate', 'Datastore.AllocateSpace', 'Datastore.AllocateTemplate', 'Datastore.Audit',
        'Group.Allocate', 'Mapping.Audit', 'Mapping.Modify', 'Mapping.Use', 'Permissions.Modify',
        'Pool.Allocate', 'Pool.Audit', 'Realm.Allocate', 'Realm.AllocateUser',
        'SDN.Allocate', 'SDN.Audit', 'SDN.Use',
        'Sys.AccessNetwork', 'Sys.Audit', 'Sys.Console', 'Sys.Incoming', 'Sys.Modify',
        'Sys.PowerMgmt', 'Sys.Syslog', 'User.Modify',
        'VM.Allocate', 'VM.Audit', 'VM.Backup', 'VM.Clone',
        'VM.Config.CDROM', 'VM.Config.CPU', 'VM.Config.Cloudinit', 'VM.Config.Disk',
        'VM.Config.HWType', 'VM.Config.Memory', 'VM.Config.Network', 'VM.Config.Options',
        'VM.Console', 'VM.GuestAgent.Audit', 'VM.GuestAgent.FileRead',
        'VM.GuestAgent.FileSystemMgmt', 'VM.GuestAgent.FileWrite', 'VM.GuestAgent.Unrestricted',
        'VM.Migrate', 'VM.PowerMgmt', 'VM.Replicate', 'VM.Snapshot', 'VM.Snapshot.Rollback',
    ];

    $angefordert = explode(',', (string) config('provisioning.proxmox.role_privs'));

    expect(array_diff($angefordert, $gueltig))->toBe([]);
});

// --- EnsureNetworkBridge ---
//
// Der Schritt, der an dem Ast sägt, auf dem er sitzt: SSH läuft hier über den
// Tunnel, und WireGuard hängt an derselben Karte, die in die Brücke wandert.
// Deshalb: erst den Zeitgeber stellen, dann umstellen, dann neu verbinden,
// nachsehen, abbestellen.

it('fasst einen Host, der die Brücke schon hat, nicht an', function () {
    // pve-fns-1 hat sie von Hand. Ein Wiederanlauf, der sie umbaut, baut ein
    // funktionierendes Netz um — und riskiert dafür genau das, wofür es die
    // Rückfahrkarte gibt.
    $s = fakeServices();
    $host = Host::factory()->active()->create();
    $run = hostRun($host);
    proveTunnel($run, $host);
    scriptBridgeState($s['shell'], up: true);

    expect(app(EnsureNetworkBridge::class)->execute($run)->type)->toBe('advance')
        ->and($s['shell']->files())->toBe([])
        ->and($s['shell']->ran('clupilot-bridge-start'))->toBeFalse();
});

it('baut keine Brücke auf einem Bond oder einer bestehenden Brücke', function () {
    // bridge_ports darauf ist falsch: die Brücke nähme sich ihren eigenen
    // Unterbau als Port, und was dabei herauskommt, ist aus der Ferne nicht mehr
    // zu reparieren.
    $s = fakeServices();
    $host = Host::factory()->active()->create();
    $run = hostRun($host);
    proveTunnel($run, $host);
    scriptBridgeState($s['shell'], up: false, iface: 'bond0', physical: false);

    $result = app(EnsureNetworkBridge::class)->execute($run);

    expect($result->type)->toBe('fail')
        ->and($result->reason)->toContain('bond0')
        ->and($s['shell']->ran('clupilot-bridge-start'))->toBeFalse();
});

it('gibt auf, wenn gar keine Karte die Standardroute trägt', function () {
    $s = fakeServices();
    $host = Host::factory()->active()->create();
    $run = hostRun($host);
    proveTunnel($run, $host);
    scriptBridgeState($s['shell'], up: false, iface: '', physical: false);

    expect(app(EnsureNetworkBridge::class)->execute($run)->type)->toBe('fail')
        ->and($s['shell']->ran('clupilot-bridge-start'))->toBeFalse();
});

it('lädt bridge.sh und bridge-run.sh wortgleich hoch und startet abgekoppelt', function () {
    // Byte für Byte. In dem Moment, in dem diese Datei aus einer PHP-Vorlage
    // gerendert statt gelesen wird, gibt es wieder zwei Fassungen der
    // Anbieterformen — das eine, was der Entwurf verbietet.
    $s = fakeServices();
    $host = Host::factory()->active()->create();
    $run = hostRun($host);
    proveTunnel($run, $host);
    scriptBridgeState($s['shell'], up: false);

    $result = app(EnsureNetworkBridge::class)->execute($run);
    $files = $s['shell']->files();

    expect($result->type)->toBe('poll')
        ->and($files['/var/lib/clupilot/bridge/bridge.sh'] ?? null)
        ->toBe(file_get_contents(base_path('deploy/bootstrap/lib/bridge.sh')))
        ->and($files['/var/lib/clupilot/bridge/bridge-run.sh'] ?? null)
        ->toBe(file_get_contents(base_path('deploy/bootstrap/lib/bridge-run.sh')))
        // Der Hub-Schlüssel muss mit, sonst prüft der Treiber den Handshake
        // gegen irgendeinen Peer statt gegen den, über den CluPilot kommt.
        ->and($files['/var/lib/clupilot/bridge/env'] ?? '')->toContain('CLUPILOT_WG_HUB_PUBKEY=')
        ->and($files['/var/lib/clupilot/bridge/env'] ?? '')->toContain('CLUPILOT_ROLLBACK_MINUTES=5')
        ->and($s['shell']->ran('nohup'))->toBeTrue()
        ->and($s['shell']->ran('setsid'))->toBeTrue()
        ->and($run->fresh()->context('bridge_deadline'))->not->toBeNull();
});

it('pollt weiter, wenn der Host während der Umstellung nicht antwortet', function () {
    // DIE tragende Zeile. RunRunner verwandelt jede geworfene Ausnahme in ein
    // retry(), und retry verbraucht das Versuchskonto — ungefangen brennt der
    // Verbindungsabriss die fünf Versuche in wenigen Minuten durch und lässt den
    // Lauf scheitern, BEVOR der Host wieder da ist.
    $s = fakeServices();
    $host = Host::factory()->active()->create();
    $run = hostRun($host, ['bridge_deadline' => now()->addMinutes(10)->toIso8601String()]);
    proveTunnel($run, $host);
    $s['shell']->failConnect = true;

    $result = app(EnsureNetworkBridge::class)->execute($run);

    expect($result->type)->toBe('poll')
        ->and($result->reason)->toContain('Brücke');
});

it('lässt einen Verbindungsfehler VOR der Umstellung ein gewöhnlicher bleiben', function () {
    // Ohne Termin hat dieser Lauf nichts angefasst. Dann ist ein
    // Verbindungsfehler kein erwarteter Abriss, sondern ein Fehler — und darf
    // das Versuchskonto kosten, statt eine Viertelstunde lang gepollt zu werden.
    $s = fakeServices();
    $host = Host::factory()->active()->create();
    $run = hostRun($host);
    proveTunnel($run, $host);
    $s['shell']->failConnect = true;

    expect(app(EnsureNetworkBridge::class)->execute($run)->type)->toBe('retry');
});

it('scheitert nach der Frist, wenn der Host gar nicht mehr antwortet', function () {
    // Der Zeitgeber hat dann laengst zurückgespielt — seine Frist ist ein
    // Drittel dieser. Genau deshalb heißt Aufgeben hier „der Host ist wieder da,
    // nur ohne Brücke" und nicht „der Host ist weg".
    $s = fakeServices();
    $host = Host::factory()->active()->create();
    $run = hostRun($host, ['bridge_deadline' => now()->subMinute()->toIso8601String()]);
    proveTunnel($run, $host);
    $s['shell']->failConnect = true;

    $result = app(EnsureNetworkBridge::class)->execute($run);

    expect($result->type)->toBe('fail')
        ->and($result->reason)->toContain('rollback timer')
        ->and($run->fresh()->context('bridge_deadline'))->toBeNull();
});

it('startet keinen zweiten Treiber, solange der erste lebt', function () {
    // Zwei Treiber auf einer /etc/network/interfaces, jeder mit eigenem
    // Zeitgeber: der eine bestellt ab, was der andere gestellt hat.
    $s = fakeServices();
    $host = Host::factory()->active()->create();
    $run = hostRun($host, ['bridge_deadline' => now()->addMinutes(10)->toIso8601String()]);
    proveTunnel($run, $host);
    scriptBridgeState($s['shell'], up: false);
    scriptBridgeStatus($s['shell'], 'running', alive: true, phase: 'umstellen');

    $result = app(EnsureNetworkBridge::class)->execute($run);

    expect($result->type)->toBe('poll')
        ->and($result->reason)->toContain('umstellen')
        ->and($s['shell']->ran('clupilot-bridge-start'))->toBeFalse();
});

it('bestellt den Zeitgeber erst ab, nachdem die Brücke nachgeprüft ist', function () {
    // Reihenfolge, nicht Geschmack. Abbestellen heißt: die Rückfahrkarte
    // wegwerfen. Wer das vor der Nachprüfung tut, tut es auf Verdacht.
    //
    // Dass dieser Zweig überhaupt erreicht wird, IST der Beweis: SSH kam über
    // den Tunnel an, und vmbr0 trägt Standardroute und Adresse. Das ist
    // bridge_proven, von außen gefragt.
    $s = fakeServices();
    $host = Host::factory()->active()->create();
    $run = hostRun($host, ['bridge_deadline' => now()->addMinutes(10)->toIso8601String()]);
    proveTunnel($run, $host);
    scriptBridgeState($s['shell'], up: true);

    $result = app(EnsureNetworkBridge::class)->execute($run);

    $stateAt = $cancelAt = null;
    foreach ($s['shell']->recorded() as $i => $command) {
        if ($stateAt === null && str_contains($command, 'clupilot-bridge-state')) {
            $stateAt = $i;
        }
        if ($cancelAt === null && str_contains($command, 'clupilot-bridge-cancel')) {
            $cancelAt = $i;
        }
    }

    expect($result->type)->toBe('advance')
        ->and($stateAt)->not->toBeNull()
        ->and($cancelAt)->not->toBeNull()
        ->and($stateAt)->toBeLessThan($cancelAt)
        // Und der Termin ist weg, sonst liest ein späterer Besuch ihn als
        // „mitten in einer Umstellung".
        ->and($run->fresh()->context('bridge_deadline'))->toBeNull();
});

it('bestellt nichts ab, wenn die Brücke schon vor diesem Lauf da war', function () {
    // Ohne Termin hat dieser Lauf nie einen Zeitgeber gestellt. Ein `stop` und
    // ein `rm` auf fremde Unit-Dateien wären ein Eingriff ohne Anlass.
    $s = fakeServices();
    $host = Host::factory()->active()->create();
    $run = hostRun($host);
    proveTunnel($run, $host);
    scriptBridgeState($s['shell'], up: true);

    expect(app(EnsureNetworkBridge::class)->execute($run)->type)->toBe('advance')
        ->and($s['shell']->ran('clupilot-bridge-cancel'))->toBeFalse();
});

it('wartet auf die Rücknahme, statt einen halb umgestellten Host liegen zu lassen', function () {
    // Solange der Zeitgeber aussteht, steckt die Maschine mitten in einer
    // Umstellung. Ein fail() ließe sie dort liegen, und der Nächste, der
    // hinsieht, fände einen halben Host ohne Erklärung.
    $s = fakeServices();
    $host = Host::factory()->active()->create();
    $run = hostRun($host, ['bridge_deadline' => now()->addMinutes(10)->toIso8601String()]);
    proveTunnel($run, $host);
    scriptBridgeState($s['shell'], up: false);
    scriptBridgeStatus($s['shell'], 'failed', note: 'Tunnel steht nicht', rolledBack: false);

    $result = app(EnsureNetworkBridge::class)->execute($run);

    expect($result->type)->toBe('poll')
        // Der Termin bleibt stehen, sonst gilt der nächste Besuch als erster und
        // startet einen zweiten Treiber.
        ->and($run->fresh()->context('bridge_deadline'))->not->toBeNull();
});

it('scheitert mit dem Grund des Treibers, sobald die Rücknahme durch ist', function () {
    $s = fakeServices();
    $host = Host::factory()->active()->create();
    $run = hostRun($host, ['bridge_deadline' => now()->addMinutes(10)->toIso8601String()]);
    proveTunnel($run, $host);
    scriptBridgeState($s['shell'], up: false);
    scriptBridgeStatus($s['shell'], 'failed', note: 'Tunnel steht nicht', rolledBack: true);

    $result = app(EnsureNetworkBridge::class)->execute($run);

    expect($result->type)->toBe('fail')
        ->and($result->reason)->toContain('Tunnel steht nicht')
        ->and($run->fresh()->context('bridge_deadline'))->toBeNull()
        ->and($s['shell']->ran('clupilot-bridge-reset'))->toBeTrue();
});

it('bestellt den Zeitgeber beim Aufgeben NICHT ab', function () {
    // Er ist die Rückfahrkarte. Steht er noch, hat er seinen Grund — und wer ihn
    // beim Aufgeben abbestellt, lässt den Host genau in dem Zustand stehen, für
    // den er gestellt wurde.
    $s = fakeServices();
    $host = Host::factory()->active()->create();
    $run = hostRun($host, ['bridge_deadline' => now()->subMinute()->toIso8601String()]);
    proveTunnel($run, $host);
    scriptBridgeState($s['shell'], up: false);
    scriptBridgeStatus($s['shell'], 'running', alive: true, phase: 'nachsehen');

    expect(app(EnsureNetworkBridge::class)->execute($run)->type)->toBe('fail')
        ->and($s['shell']->ran('clupilot-bridge-cancel'))->toBeFalse();
});

it('versucht es höchstens zweimal je Lauf', function () {
    // Der zweite ist für den Fall, dass der Betreiber zwischen den Versuchen
    // etwas repariert hat. Danach die Handarbeit-Meldung — ein dritter Versuch
    // gegen dieselbe Maschine wäre nur dieselbe Viertelstunde noch einmal.
    $s = fakeServices();
    $host = Host::factory()->active()->create();
    $run = hostRun($host, ['bridge_attempts' => 2]);
    proveTunnel($run, $host);
    scriptBridgeState($s['shell'], up: false);

    $result = app(EnsureNetworkBridge::class)->execute($run);

    expect($result->type)->toBe('fail')
        ->and($result->reason)->toContain('by hand')
        ->and($s['shell']->ran('clupilot-bridge-start'))->toBeFalse();
});

it('erlaubt einem Betreiber, nach einem Fehlschlag von vorn anzufangen', function () {
    // giveUp() löscht den Termin — sonst macht die Erst-Besuch-Prüfung aus jedem
    // Retry nach einem Fehlschlag einen sofortigen zweiten Fehlschlag, ohne dass
    // der Host überhaupt angefasst wurde. Der ZÄHLER bleibt dagegen stehen,
    // sonst ist der Deckel von zwei Versuchen keiner.
    $s = fakeServices();
    $host = Host::factory()->active()->create();
    $run = hostRun($host, [
        'bridge_deadline' => now()->addMinutes(10)->toIso8601String(),
        'bridge_attempts' => 1,
    ]);
    proveTunnel($run, $host);
    scriptBridgeState($s['shell'], up: false);
    scriptBridgeStatus($s['shell'], 'failed', note: 'irgendwas', rolledBack: true);

    app(EnsureNetworkBridge::class)->execute($run);

    expect($run->fresh()->context('bridge_deadline'))->toBeNull()
        ->and($run->fresh()->context('bridge_attempts'))->toBe(1);
});

it('schickt nur Shell, die eine Shell auch parsen kann', function () {
    // FakeRemoteShell führt keinen dieser Befehle aus. Ein verrutschtes
    // Anführungszeichen käme sonst zuerst auf einem echten Host zur Ausführung —
    // und der Schritt, den es kaputtmacht, ist der, der das Netz umstellt.
    $s = fakeServices();
    $host = Host::factory()->active()->create();

    $run = hostRun($host);
    proveTunnel($run, $host);
    scriptBridgeState($s['shell'], up: false);
    app(EnsureNetworkBridge::class)->execute($run);                       // state + start

    scriptBridgeStatus($s['shell'], 'failed', note: 'nope', rolledBack: true);
    app(EnsureNetworkBridge::class)->execute(
        hostRun($host, ['bridge_deadline' => now()->addMinutes(10)->toIso8601String()])
    );                                                                    // state + status + log + reset

    scriptBridgeState($s['shell'], up: true);
    app(EnsureNetworkBridge::class)->execute(
        hostRun($host, ['bridge_deadline' => now()->addMinutes(10)->toIso8601String()])
    );                                                                    // state + cancel

    $checked = 0;
    foreach ($s['shell']->recorded() as $command) {
        expect(Process::input($command)->run('sh -n')->successful())
            ->toBeTrue("kein gültiges POSIX-sh:\n".$command);
        $checked++;
    }

    expect($checked)->toBeGreaterThanOrEqual(6);
});

it('baut die Brücke nach dem Neustart und vor ConfigureProxmox', function () {
    // Nach dem Neustart, weil dort ifupdown2 steht und der Tunnel gerade
    // bewiesen hat, dass er einen Neustart überlebt. Vor ConfigureProxmox, weil
    // dessen refuseWithoutBridge() die Nachprüfung ist — bauen UND prüfen,
    // dasselbe Paar wie BuildVmTemplate → VerifyVmTemplate.
    $pipeline = config('provisioning.pipelines.host');

    $bridge = array_search(EnsureNetworkBridge::class, $pipeline, true);
    $reboot = array_search(RebootIntoPveKernel::class, $pipeline, true);
    $proxmox = array_search(ConfigureProxmox::class, $pipeline, true);

    expect($bridge)->not->toBeFalse()
        ->and($reboot)->toBeLessThan($bridge)
        ->and($bridge)->toBeLessThan($proxmox);
});

it('lässt ConfigureProxmox die Nachprüfung bleiben', function () {
    // Der Schritt davor baut. Dieser prüft. Beides, nicht eines von beiden —
    // sonst meldet ein Bau, der still danebenging, trotzdem Erfolg.
    //
    // CommandResult::failure(int $exitCode = 1, string $stderr = '') — zwei
    // Parameter, nicht drei.
    $s = fakeServices();
    $host = Host::factory()->active()->create();
    $run = hostRun($host);
    proveTunnel($run, $host);
    $s['shell']->script('ip link show vmbr0', CommandResult::failure(1, 'Device does not exist'));

    expect(app(ConfigureProxmox::class)->execute($run)->type)->toBe('fail');
});

it('beschriftet den Schritt in beiden Sprachen', function () {
    // Die Konsole zeigt eine Zeile je Schritt. Eine ohne Beschriftung zeigt den
    // Schlüssel — der Betreiber säße beim Zusehen vor 'hosts.step.…'.
    foreach (['de', 'en'] as $locale) {
        expect(trans('hosts.step.ensure_network_bridge', [], $locale))
            ->not->toBe('hosts.step.ensure_network_bridge', "Beschriftung fehlt in {$locale}");
    }
});

it('legt jeden Pipeline-Schritt mit einer Beschriftung ab', function () {
    // Nicht nur den neuen. Wer künftig einen Schritt einhängt und die
    // Sprachdateien vergisst, fällt hier auf statt in der Konsole.
    foreach (config('provisioning.pipelines.host') as $class) {
        $key = app($class)->key();

        foreach (['de', 'en'] as $locale) {
            expect(trans("hosts.step.{$key}", [], $locale))
                ->not->toBe("hosts.step.{$key}", "Beschriftung fehlt: {$key} ({$locale})");
        }
    }
});
