CluPilotCloud/tests/Feature/Provisioning/InstanceRestartTest.php

<?php

use App\Actions\RestartInstance;
use App\Livewire\Admin\ConfirmRestartInstance;
use App\Livewire\Admin\Instances as AdminInstances;
use App\Livewire\Cloud;
use App\Livewire\ConfirmRestartCloud;
use App\Models\Host;
use App\Models\Instance;
use App\Models\Order;
use App\Models\ProvisioningRun;
use App\Models\User;
use App\Provisioning\RunRunner;
use App\Provisioning\Steps\Customer\CompleteRestart;
use App\Provisioning\Steps\Customer\ShutDownVirtualMachine;
use App\Support\ProvisioningSettings;
use Illuminate\Auth\Access\AuthorizationException;
use Illuminate\Support\Facades\Queue;
use Livewire\Livewire;

/**
 * Restarting a customer's machine — the thing the product could not do.
 *
 * ProxmoxClient had startVm and no way to stop anything, so a plan change that
 * grew a running guest wrote the new cores and memory into the VM's definition,
 * set `restart_required_since`, and left the flag to be cleared by a resize step
 * that happened to find the machine stopped — which nothing could arrange. A
 * paid upgrade could stay unreached for the life of the contract, and the cloud
 * page's "Neu starten" was a toast.
 *
 * These tests are about the two halves of the fix: a shutdown that is a request
 * and never a power cut, and a flag that is cleared by the machine coming back
 * rather than by anybody pressing anything.
 */

/**
 * A live instance whose contract has been grown but whose guest is still
 * running on what it booted with — exactly the state a plan change leaves.
 */
function restartableInstance(array $instanceAttributes = []): array
{
    $services = fakeServices();
    $pve = $services['pve'];

    $host = Host::factory()->active()->create(['datacenter' => 'fsn', 'node' => 'pve']);
    $order = Order::factory()->withSubscription()->create(['datacenter' => 'fsn', 'plan' => 'business']);

    $instance = Instance::factory()->create(array_merge([
        'order_id' => $order->id,
        'customer_id' => $order->customer_id,
        'host_id' => $host->id,
        'vmid' => 101,
        'guest_ip' => '10.20.0.7',
        'status' => 'active',
        // What the contract now says. The guest below is running on less.
        'cores' => 8,
        'ram_mb' => 16384,
        'restart_required_since' => now()->subDays(3),
    ], $instanceAttributes));

    // The machine as it actually is: booted small, then given a bigger
    // definition it has not read yet.
    $pve->setCloudInit('pve', 101, ['cores' => 2, 'memory' => 4096]);
    $pve->startVm('pve', 101);
    $pve->setCloudInit('pve', 101, ['cores' => 8, 'memory' => 16384]);

    // The address the guest comes back on. The restart pipeline reads it — a
    // cold boot is the one event that can move a DHCP lease — and would poll for
    // it until the step's deadline if the fake answered nothing.
    $pve->guestScript('hostname -I', 0, '10.20.0.7');

    return compact('services', 'pve', 'host', 'order', 'instance');
}

/**
 * Advance a run until it settles, clearing the backoff between steps.
 *
 * next_attempt_at is what makes a poll a wait; a test that did not clear it
 * would simply watch the runner decline to do anything.
 */
function driveRun(ProvisioningRun $run, int $limit = 25): ProvisioningRun
{
    $runner = app(RunRunner::class);

    for ($i = 0; $i < $limit; $i++) {
        $run->refresh();

        if (in_array($run->status, [ProvisioningRun::STATUS_COMPLETED, ProvisioningRun::STATUS_FAILED], true)) {
            break;
        }

        $run->forceFill(['next_attempt_at' => null])->save();
        $runner->advance($run);
    }

    return $run->refresh();
}

it('asks the guest to shut down, brings it back, and only then stops saying a restart is due', function () {
    Queue::fake();
    ['pve' => $pve, 'instance' => $instance] = restartableInstance();
    $this->actingAs(admin(), 'operator');

    // Before: the definition is big, the running machine is not — which is what
    // "Neustart erforderlich" has been telling the customer for three days.
    expect($pve->vmStatus('pve', 101)['cpus'])->toBe(2)
        ->and($instance->restartIsPending())->toBeTrue();

    $run = app(RestartInstance::class)($instance);
    expect($run)->not->toBeNull();

    // Pressing the button does not clear it. Nothing has happened to the
    // machine yet, and a flag that clears on intent is a flag that lies.
    expect($instance->fresh()->restartIsPending())->toBeTrue();

    driveRun($run);

    expect($run->fresh()->status)->toBe(ProvisioningRun::STATUS_COMPLETED)
        // The guest was ASKED, and it was asked exactly once.
        ->and($pve->shutdownCalls)->toHaveCount(1)
        ->and($pve->shutdownCalls[0]['vmid'])->toBe(101)
        // It came back — and came back on the definition it had been given,
        // which is the only thing that makes the upgrade real.
        ->and($pve->vmStatus('pve', 101)['status'])->toBe('running')
        ->and($pve->vmStatus('pve', 101)['cpus'])->toBe(8)
        ->and((int) ($pve->vmStatus('pve', 101)['maxmem'] / 1048576))->toBe(16384)
        ->and($instance->fresh()->restartIsPending())->toBeFalse();
});

it('never pulls the plug on a guest that will not shut down', function () {
    Queue::fake();
    ['pve' => $pve, 'instance' => $instance, 'order' => $order] = restartableInstance();
    $order->update(['status' => 'active']);
    $this->actingAs(admin(), 'operator');

    // A guest that ignores ACPI — a wedged handler, or a machine busy with
    // something it will not be interrupted in.
    $pve->shutdownIgnored = true;

    $run = app(RestartInstance::class)($instance);

    driveRun($run);
    // Past the step's grace period but inside the runner's own step timeout, so
    // the step's deliberate failure fires rather than a generic retry.
    $this->travel(11)->minutes();
    $run = driveRun($run);

    expect($run->status)->toBe(ProvisioningRun::STATUS_FAILED)
        ->and($run->error)->toBe('shutdown_timeout')
        // The machine is still running. Nextcloud is a database and this
        // product has no power cut to escalate to — the run fails and a person
        // decides what happens next.
        ->and($pve->runningVmids)->toContain(101)
        ->and($pve->vmStatus('pve', 101)['status'])->toBe('running')
        // And nothing has been declared finished on the strength of a failure.
        ->and($instance->fresh()->restartIsPending())->toBeTrue()
        // A maintenance pipeline failing must not release a paid, running
        // customer — RunRunner's subject hook only fires for the pipeline that
        // BUILDS the subject, and a restart is not that one.
        ->and($order->fresh()->status)->toBe('active')
        ->and($instance->fresh()->status)->toBe('active');
});

it('refuses to call a restart done when the machine came back smaller than it was sold', function () {
    Queue::fake();
    ['pve' => $pve, 'instance' => $instance, 'order' => $order] = restartableInstance();

    // The machine went round but the bigger definition never took — a config
    // write that silently failed, a host that could not honour it.
    $pve->shutdownVm('pve', 101, 600);
    $pve->vmConfig[101] = ['cores' => 2, 'memory' => 4096];
    $pve->startVm('pve', 101);

    $run = ProvisioningRun::factory()->create([
        'subject_type' => Order::class, 'subject_id' => $order->id, 'pipeline' => 'restart',
        'context' => ['instance_id' => $instance->id, 'node' => 'pve', 'vmid' => 101],
    ]);

    $result = app(CompleteRestart::class)->execute($run);

    expect($result->type)->toBe('fail')
        ->and($result->reason)->toContain('2 statt 8 vCPU')
        ->and($instance->fresh()->restartIsPending())->toBeTrue();
});

it('leaves the pending restart alone until the very last step of the run', function () {
    Queue::fake();
    ['instance' => $instance] = restartableInstance();
    $this->actingAs(admin(), 'operator');

    $run = app(RestartInstance::class)($instance);
    $runner = app(RunRunner::class);

    // Read off the pipeline rather than written out, so inserting a step cannot
    // quietly turn "the last one" into "the one before it" and leave this test
    // green while proving something weaker.
    $last = count((array) config('provisioning.pipelines.restart')) - 1;

    // Shutdown asked, shutdown observed, machine started, agent answered, address
    // read — and through all of it the flag stands, because none of those is the
    // machine running on what was bought.
    for ($i = 0; $i < 10; $i++) {
        $run->refresh();

        if ($run->current_step >= $last) {
            break;
        }

        $run->forceFill(['next_attempt_at' => null])->save();
        $runner->advance($run);

        expect($instance->fresh()->restartIsPending())->toBeTrue();
    }

    expect($run->refresh()->current_step)->toBe($last)
        ->and($instance->fresh()->restartIsPending())->toBeTrue();

    driveRun($run);

    expect($instance->fresh()->restartIsPending())->toBeFalse();
});

it('lets a customer restart their own machine and nobody else’s', function () {
    Queue::fake();
    ['instance' => $mine] = restartableInstance();
    ['instance' => $theirs] = restartableInstance();

    $user = User::factory()->create(['email_verified_at' => now()]);
    $mine->customer->update(['user_id' => $user->id, 'email' => $user->email]);

    $this->actingAs($user);

    // Checked in the action, not in the view. A Livewire method is reachable by
    // anyone who can POST to /livewire/update, so hiding a button proves
    // nothing at all about who may press it.
    expect(fn () => app(RestartInstance::class)($theirs))->toThrow(AuthorizationException::class);
    expect(app(RestartInstance::class)->allows($theirs))->toBeFalse()
        ->and(app(RestartInstance::class)->allows($mine))->toBeTrue();

    expect(app(RestartInstance::class)($mine))->not->toBeNull();
});

it('lets an operator with the capability restart any machine, and refuses one without it', function () {
    Queue::fake();
    ['instance' => $instance] = restartableInstance();

    $this->actingAs(operator('Read-only'), 'operator');
    expect(fn () => app(RestartInstance::class)($instance))->toThrow(AuthorizationException::class);
    expect(ProvisioningRun::query()->where('pipeline', 'restart')->count())->toBe(0);

    $this->actingAs(admin(), 'operator');
    expect(app(RestartInstance::class)($instance))->not->toBeNull()
        ->and(ProvisioningRun::query()->where('pipeline', 'restart')->count())->toBe(1);
});

it('does not start a restart while another run is already in flight for that instance', function () {
    Queue::fake();
    ['instance' => $instance, 'order' => $order] = restartableInstance();

    // A plan change still working on the same machine. Stopping the guest
    // underneath it would race the config it is about to boot.
    ProvisioningRun::factory()->create([
        'subject_type' => Order::class,
        'subject_id' => $order->id,
        'pipeline' => 'plan-change',
        'status' => ProvisioningRun::STATUS_RUNNING,
    ]);

    $this->actingAs(admin(), 'operator');

    expect(app(RestartInstance::class)($instance))->toBeNull()
        ->and(ProvisioningRun::query()->where('pipeline', 'restart')->count())->toBe(0);
});

it('refuses a machine there is nothing to restart', function () {
    Queue::fake();
    ['instance' => $instance] = restartableInstance(['status' => 'provisioning']);

    $this->actingAs(admin(), 'operator');

    // Still being built: its own run is writing this machine, and a shutdown
    // beside it would stop the guest that run is talking to.
    expect(app(RestartInstance::class)($instance))->toBeNull();
});

it('confirms in a modal on both sides and drives the real action from the page', function () {
    Queue::fake();
    ['instance' => $instance] = restartableInstance();

    $user = User::factory()->create(['email_verified_at' => now()]);
    $instance->customer->update(['user_id' => $user->id, 'email' => $user->email]);

    // Customer: the modal only says yes; Cloud::restart() does the work.
    Livewire::actingAs($user)->test(ConfirmRestartCloud::class)
        ->call('proceed')
        ->assertDispatched('cloud-restart-confirmed');

    Livewire::actingAs($user)->test(Cloud::class)
        ->dispatch('cloud-restart-confirmed')
        ->assertDispatched('notify');

    expect(ProvisioningRun::query()->where('pipeline', 'restart')->count())->toBe(1);

    // Operator: same shape, addressed by uuid (R11).
    ProvisioningRun::query()->delete();

    Livewire::actingAs(admin(), 'operator')->test(ConfirmRestartInstance::class, ['uuid' => $instance->uuid])
        ->call('proceed')
        ->assertDispatched('instance-restart-confirmed');

    Livewire::actingAs(admin(), 'operator')->test(AdminInstances::class)
        ->dispatch('instance-restart-confirmed', uuid: $instance->uuid);

    expect(ProvisioningRun::query()->where('pipeline', 'restart')->count())->toBe(1);
});

it('refuses the console action to an operator who does not hold the capability', function () {
    Queue::fake();
    ['instance' => $instance] = restartableInstance();

    Livewire::actingAs(operator('Read-only'), 'operator')->test(AdminInstances::class)
        ->dispatch('instance-restart-confirmed', uuid: $instance->uuid)
        ->assertForbidden();

    expect(ProvisioningRun::query()->where('pipeline', 'restart')->count())->toBe(0);
});

it('follows the guest when a restart brings it back on another address', function () {
    // cloud-init hands every guest ip=dhcp, so the address is a lease and not a
    // property of the machine. The router file on the serving host names it as the
    // backend — so a machine that comes back somewhere else, with nothing
    // rewriting that file, is a customer's cloud answering 502 for the rest of its
    // life. Nothing in the product re-read the address at all, and the route guard
    // compared only the hostname list, so neither half could notice.
    Queue::fake();
    ['pve' => $pve, 'instance' => $instance] = restartableInstance([
        // What the router was last told, and where the machine actually is now.
        'routed_backend' => '10.20.0.7',
        'route_written' => true,
        'routed_hostnames' => ['berger.'.ProvisioningSettings::dnsZone()],
        'subdomain' => 'berger',
        'cert_ok' => true,
    ]);
    $pve->guestScript('hostname -I', 0, '10.20.0.44 fe80::1');
    $this->actingAs(admin(), 'operator');

    driveRun(app(RestartInstance::class)($instance));

    expect($instance->fresh()->guest_ip)->toBe('10.20.0.44')
        // An address run, because writing the router is that pipeline's job. It is
        // allowed to start beside the restart because the guard asks whether the
        // run in flight carries out the address steps, and a restart carries
        // neither of them.
        ->and(ProvisioningRun::query()->where('pipeline', 'address')->count())->toBe(1);

    driveRun(ProvisioningRun::query()->where('pipeline', 'address')->sole());

    expect($instance->fresh()->routed_backend)->toBe('10.20.0.44');
});

it('gives the shutdown step nothing to do when the guest is already stopped', function () {
    Queue::fake();
    ['pve' => $pve, 'instance' => $instance, 'order' => $order] = restartableInstance();

    $pve->runningVmids = [];

    $run = ProvisioningRun::factory()->create([
        'subject_type' => Order::class, 'subject_id' => $order->id, 'pipeline' => 'restart',
        'context' => ['instance_id' => $instance->id, 'node' => 'pve', 'vmid' => 101],
    ]);

    expect(app(ShutDownVirtualMachine::class)->execute($run)->type)->toBe('advance')
        ->and($pve->shutdownCalls)->toBe([]);
});