CluPilotCloud/app/Provisioning/Steps/Host/RebootIntoPveKernel.php

<?php

namespace App\Provisioning\Steps\Host;

use App\Models\ProvisioningRun;
use App\Provisioning\StepResult;
use App\Services\Ssh\RemoteShell;
use Illuminate\Support\Carbon;
use Throwable;

/**
 * Removes the Debian kernel and reboots into the Proxmox kernel. The reboot is
 * modelled as a retry loop: issue once, then poll until `uname -r` reports a
 * -pve kernel. The state machine survives the reboot by design.
 *
 * The reboot is the one irreversible thing this pipeline does. Everything else
 * can be retried from CluPilot; a machine that does not come back needs the
 * provider's out-of-band console, and on a dedicated server that can mean hours.
 * So the boot path is PROVEN before the reboot is issued — never assumed from
 * "apt did not complain".
 */
class RebootIntoPveKernel extends HostStep
{
    public function __construct(private RemoteShell $shell) {}

    public function key(): string
    {
        return 'reboot_into_pve_kernel';
    }

    public function maxDuration(): int
    {
        // Above the 15-min reboot deadline below, so the step's own deadline
        // governs failure rather than the generic step timeout.
        return 1800;
    }

    public function execute(ProvisioningRun $run): StepResult
    {
        $host = $this->host($run);

        if (! $run->context('reboot_issued')) {
            $this->keyLogin($this->shell, $host);

            // Tolerated, deliberately: an image that never had the
            // linux-image-amd64 metapackage installed makes apt exit non-zero
            // with nothing wrong. What matters is not whether the Debian kernel
            // went away but whether a Proxmox one can boot, and that is checked
            // below on its own terms rather than inferred from this line.
            $this->shell->run('export DEBIAN_FRONTEND=noninteractive; apt-get -y remove linux-image-amd64 || true');

            if (($refusal = $this->refuseUnbootableReboot()) !== null) {
                return $refusal;
            }

            $run->mergeContext([
                'reboot_issued' => true,
                'reboot_deadline' => now()->addMinutes(15)->toIso8601String(),
            ]);
            $this->shell->run('systemctl reboot || reboot');

            return StepResult::poll(30, 'rebooting into the Proxmox kernel');
        }

        try {
            $this->keyLogin($this->shell, $host);
            $uname = trim($this->shell->run('uname -r')->stdout);
        } catch (Throwable) {
            $uname = '';
        }

        if (str_contains($uname, '-pve')) {
            return StepResult::advance();
        }

        $deadline = $run->context('reboot_deadline');
        if ($deadline !== null && now()->greaterThan(Carbon::parse($deadline))) {
            // Clear the reboot markers so a manual retry re-issues a fresh reboot
            // instead of immediately re-hitting the expired deadline.
            $run->forgetContext('reboot_issued');
            $run->forgetContext('reboot_deadline');

            return StepResult::fail('Host did not return on the Proxmox kernel before the deadline.');
        }

        return StepResult::poll(15, 'waiting for the Proxmox kernel');
    }

    /**
     * Everything that has to be true for this machine to come back, or a fail
     * naming what is not.
     *
     * The three checks are the three ways the old code could reboot into
     * nothing. It removed the Debian kernel with `|| true` and ignored
     * update-grub's exit status, so a /boot that had filled up — common enough
     * on a provider image with a small boot partition, and made likelier by the
     * full-upgrade the previous step runs — produced a truncated initramfs, a
     * half-written grub.cfg, and a reboot issued on the strength of neither.
     *
     * A fail rather than a retry: none of these three conditions fixes itself,
     * and an operator who is already watching the onboarding can clear /boot or
     * reinstall the kernel. Retrying would only spend the budget and then say
     * something vaguer.
     */
    private function refuseUnbootableReboot(): ?StepResult
    {
        // A kernel image is not enough on its own — grub needs the initramfs
        // that goes with it, and a truncated or absent initrd is exactly what a
        // full /boot leaves behind. Both are listed so the message can say which
        // half is missing.
        $kernels = trim($this->shell->run('ls -1 /boot/vmlinuz-*-pve 2>/dev/null')->stdout);
        $initrds = trim($this->shell->run('ls -1 /boot/initrd.img-*-pve 2>/dev/null')->stdout);

        if ($kernels === '') {
            return StepResult::fail(
                'Refusing to reboot: no Proxmox kernel image (/boot/vmlinuz-*-pve) is installed on this host, '.
                'so it would come back on the Debian kernel at best and not at all if that one was removed. '.
                'Check `apt-get -y install proxmox-ve` and the free space in /boot, then retry.'
            );
        }

        if ($initrds === '') {
            return StepResult::fail(
                'Refusing to reboot: a Proxmox kernel is installed ('.$this->firstLine($kernels).') but it has no '.
                'initramfs (/boot/initrd.img-*-pve), which is what a full /boot leaves behind. Free space in /boot, '.
                'run `update-initramfs -u -k all`, then retry.'
            );
        }

        // Checked for its EXIT STATUS, which the old code discarded. update-grub
        // failing is the difference between a boot menu that lists the new
        // kernel and one that does not mention it.
        $grub = $this->shell->run('update-grub');
        if (! $grub->ok()) {
            return StepResult::fail(
                'Refusing to reboot: update-grub failed, so the boot menu may not list the Proxmox kernel '.
                '('.$this->firstLine($kernels).'). Output: '.$this->firstLine(trim($grub->stderr.' '.$grub->stdout)).'. '.
                'Fix that on the host (usually free space in /boot), then retry.'
            );
        }

        return null;
    }

    /** Keeps a multi-line `ls` or a long apt error from swamping the run's error field. */
    private function firstLine(string $output): string
    {
        $line = trim((string) preg_split('/\R/', $output)[0]);

        return $line === '' ? '(no output)' : $line;
    }
}