active()->create(['datacenter' => 'fsn', 'node' => 'pve']); $order = Order::factory()->withSubscription()->create(['datacenter' => 'fsn', 'plan' => 'business']); $instance = Instance::factory()->create(array_merge([ 'order_id' => $order->id, 'customer_id' => $order->customer_id, 'host_id' => $host->id, 'vmid' => 101, 'guest_ip' => '10.20.0.7', 'status' => 'active', // What the contract now says. The guest below is running on less. 'cores' => 8, 'ram_mb' => 16384, 'restart_required_since' => now()->subDays(3), ], $instanceAttributes)); // The machine as it actually is: booted small, then given a bigger // definition it has not read yet. $pve->setCloudInit('pve', 101, ['cores' => 2, 'memory' => 4096]); $pve->startVm('pve', 101); $pve->setCloudInit('pve', 101, ['cores' => 8, 'memory' => 16384]); // The address the guest comes back on. The restart pipeline reads it — a // cold boot is the one event that can move a DHCP lease — and would poll for // it until the step's deadline if the fake answered nothing. $pve->guestScript('hostname -I', 0, '10.20.0.7'); return compact('services', 'pve', 'host', 'order', 'instance'); } /** * Advance a run until it settles, clearing the backoff between steps. * * next_attempt_at is what makes a poll a wait; a test that did not clear it * would simply watch the runner decline to do anything. */ function driveRun(ProvisioningRun $run, int $limit = 25): ProvisioningRun { $runner = app(RunRunner::class); for ($i = 0; $i < $limit; $i++) { $run->refresh(); if (in_array($run->status, [ProvisioningRun::STATUS_COMPLETED, ProvisioningRun::STATUS_FAILED], true)) { break; } $run->forceFill(['next_attempt_at' => null])->save(); $runner->advance($run); } return $run->refresh(); } it('asks the guest to shut down, brings it back, and only then stops saying a restart is due', function () { Queue::fake(); ['pve' => $pve, 'instance' => $instance] = restartableInstance(); $this->actingAs(admin(), 'operator'); // Before: the definition is big, the running machine is not — which is what // "Neustart erforderlich" has been telling the customer for three days. expect($pve->vmStatus('pve', 101)['cpus'])->toBe(2) ->and($instance->restartIsPending())->toBeTrue(); $run = app(RestartInstance::class)($instance); expect($run)->not->toBeNull(); // Pressing the button does not clear it. Nothing has happened to the // machine yet, and a flag that clears on intent is a flag that lies. expect($instance->fresh()->restartIsPending())->toBeTrue(); driveRun($run); expect($run->fresh()->status)->toBe(ProvisioningRun::STATUS_COMPLETED) // The guest was ASKED, and it was asked exactly once. ->and($pve->shutdownCalls)->toHaveCount(1) ->and($pve->shutdownCalls[0]['vmid'])->toBe(101) // It came back — and came back on the definition it had been given, // which is the only thing that makes the upgrade real. ->and($pve->vmStatus('pve', 101)['status'])->toBe('running') ->and($pve->vmStatus('pve', 101)['cpus'])->toBe(8) ->and((int) ($pve->vmStatus('pve', 101)['maxmem'] / 1048576))->toBe(16384) ->and($instance->fresh()->restartIsPending())->toBeFalse(); }); it('never pulls the plug on a guest that will not shut down', function () { Queue::fake(); ['pve' => $pve, 'instance' => $instance, 'order' => $order] = restartableInstance(); $order->update(['status' => 'active']); $this->actingAs(admin(), 'operator'); // A guest that ignores ACPI — a wedged handler, or a machine busy with // something it will not be interrupted in. $pve->shutdownIgnored = true; $run = app(RestartInstance::class)($instance); driveRun($run); // Past the step's grace period but inside the runner's own step timeout, so // the step's deliberate failure fires rather than a generic retry. $this->travel(11)->minutes(); $run = driveRun($run); expect($run->status)->toBe(ProvisioningRun::STATUS_FAILED) ->and($run->error)->toBe('shutdown_timeout') // The machine is still running. Nextcloud is a database and this // product has no power cut to escalate to — the run fails and a person // decides what happens next. ->and($pve->runningVmids)->toContain(101) ->and($pve->vmStatus('pve', 101)['status'])->toBe('running') // And nothing has been declared finished on the strength of a failure. ->and($instance->fresh()->restartIsPending())->toBeTrue() // A maintenance pipeline failing must not release a paid, running // customer — RunRunner's subject hook only fires for the pipeline that // BUILDS the subject, and a restart is not that one. ->and($order->fresh()->status)->toBe('active') ->and($instance->fresh()->status)->toBe('active'); }); it('refuses to call a restart done when the machine came back smaller than it was sold', function () { Queue::fake(); ['pve' => $pve, 'instance' => $instance, 'order' => $order] = restartableInstance(); // The machine went round but the bigger definition never took — a config // write that silently failed, a host that could not honour it. $pve->shutdownVm('pve', 101, 600); $pve->vmConfig[101] = ['cores' => 2, 'memory' => 4096]; $pve->startVm('pve', 101); $run = ProvisioningRun::factory()->create([ 'subject_type' => Order::class, 'subject_id' => $order->id, 'pipeline' => 'restart', 'context' => ['instance_id' => $instance->id, 'node' => 'pve', 'vmid' => 101], ]); $result = app(CompleteRestart::class)->execute($run); expect($result->type)->toBe('fail') ->and($result->reason)->toContain('2 statt 8 vCPU') ->and($instance->fresh()->restartIsPending())->toBeTrue(); }); it('leaves the pending restart alone until the very last step of the run', function () { Queue::fake(); ['instance' => $instance] = restartableInstance(); $this->actingAs(admin(), 'operator'); $run = app(RestartInstance::class)($instance); $runner = app(RunRunner::class); // Read off the pipeline rather than written out, so inserting a step cannot // quietly turn "the last one" into "the one before it" and leave this test // green while proving something weaker. $last = count((array) config('provisioning.pipelines.restart')) - 1; // Shutdown asked, shutdown observed, machine started, agent answered, address // read — and through all of it the flag stands, because none of those is the // machine running on what was bought. for ($i = 0; $i < 10; $i++) { $run->refresh(); if ($run->current_step >= $last) { break; } $run->forceFill(['next_attempt_at' => null])->save(); $runner->advance($run); expect($instance->fresh()->restartIsPending())->toBeTrue(); } expect($run->refresh()->current_step)->toBe($last) ->and($instance->fresh()->restartIsPending())->toBeTrue(); driveRun($run); expect($instance->fresh()->restartIsPending())->toBeFalse(); }); it('lets a customer restart their own machine and nobody else’s', function () { Queue::fake(); ['instance' => $mine] = restartableInstance(); ['instance' => $theirs] = restartableInstance(); $user = User::factory()->create(['email_verified_at' => now()]); $mine->customer->update(['user_id' => $user->id, 'email' => $user->email]); $this->actingAs($user); // Checked in the action, not in the view. A Livewire method is reachable by // anyone who can POST to /livewire/update, so hiding a button proves // nothing at all about who may press it. expect(fn () => app(RestartInstance::class)($theirs))->toThrow(AuthorizationException::class); expect(app(RestartInstance::class)->allows($theirs))->toBeFalse() ->and(app(RestartInstance::class)->allows($mine))->toBeTrue(); expect(app(RestartInstance::class)($mine))->not->toBeNull(); }); it('lets an operator with the capability restart any machine, and refuses one without it', function () { Queue::fake(); ['instance' => $instance] = restartableInstance(); $this->actingAs(operator('Read-only'), 'operator'); expect(fn () => app(RestartInstance::class)($instance))->toThrow(AuthorizationException::class); expect(ProvisioningRun::query()->where('pipeline', 'restart')->count())->toBe(0); $this->actingAs(admin(), 'operator'); expect(app(RestartInstance::class)($instance))->not->toBeNull() ->and(ProvisioningRun::query()->where('pipeline', 'restart')->count())->toBe(1); }); it('does not start a restart while another run is already in flight for that instance', function () { Queue::fake(); ['instance' => $instance, 'order' => $order] = restartableInstance(); // A plan change still working on the same machine. Stopping the guest // underneath it would race the config it is about to boot. ProvisioningRun::factory()->create([ 'subject_type' => Order::class, 'subject_id' => $order->id, 'pipeline' => 'plan-change', 'status' => ProvisioningRun::STATUS_RUNNING, ]); $this->actingAs(admin(), 'operator'); expect(app(RestartInstance::class)($instance))->toBeNull() ->and(ProvisioningRun::query()->where('pipeline', 'restart')->count())->toBe(0); }); it('refuses a machine there is nothing to restart', function () { Queue::fake(); ['instance' => $instance] = restartableInstance(['status' => 'provisioning']); $this->actingAs(admin(), 'operator'); // Still being built: its own run is writing this machine, and a shutdown // beside it would stop the guest that run is talking to. expect(app(RestartInstance::class)($instance))->toBeNull(); }); it('confirms in a modal on both sides and drives the real action from the page', function () { Queue::fake(); ['instance' => $instance] = restartableInstance(); $user = User::factory()->create(['email_verified_at' => now()]); $instance->customer->update(['user_id' => $user->id, 'email' => $user->email]); // Customer: the modal only says yes; Cloud::restart() does the work. Livewire::actingAs($user)->test(ConfirmRestartCloud::class) ->call('proceed') ->assertDispatched('cloud-restart-confirmed'); Livewire::actingAs($user)->test(Cloud::class) ->dispatch('cloud-restart-confirmed') ->assertDispatched('notify'); expect(ProvisioningRun::query()->where('pipeline', 'restart')->count())->toBe(1); // Operator: same shape, addressed by uuid (R11). ProvisioningRun::query()->delete(); Livewire::actingAs(admin(), 'operator')->test(ConfirmRestartInstance::class, ['uuid' => $instance->uuid]) ->call('proceed') ->assertDispatched('instance-restart-confirmed'); Livewire::actingAs(admin(), 'operator')->test(AdminInstances::class) ->dispatch('instance-restart-confirmed', uuid: $instance->uuid); expect(ProvisioningRun::query()->where('pipeline', 'restart')->count())->toBe(1); }); it('refuses the console action to an operator who does not hold the capability', function () { Queue::fake(); ['instance' => $instance] = restartableInstance(); Livewire::actingAs(operator('Read-only'), 'operator')->test(AdminInstances::class) ->dispatch('instance-restart-confirmed', uuid: $instance->uuid) ->assertForbidden(); expect(ProvisioningRun::query()->where('pipeline', 'restart')->count())->toBe(0); }); it('follows the guest when a restart brings it back on another address', function () { // cloud-init hands every guest ip=dhcp, so the address is a lease and not a // property of the machine. The router file on the serving host names it as the // backend — so a machine that comes back somewhere else, with nothing // rewriting that file, is a customer's cloud answering 502 for the rest of its // life. Nothing in the product re-read the address at all, and the route guard // compared only the hostname list, so neither half could notice. Queue::fake(); ['pve' => $pve, 'instance' => $instance] = restartableInstance([ // What the router was last told, and where the machine actually is now. 'routed_backend' => '10.20.0.7', 'route_written' => true, 'routed_hostnames' => ['berger.'.ProvisioningSettings::dnsZone()], 'subdomain' => 'berger', 'cert_ok' => true, ]); $pve->guestScript('hostname -I', 0, '10.20.0.44 fe80::1'); $this->actingAs(admin(), 'operator'); driveRun(app(RestartInstance::class)($instance)); expect($instance->fresh()->guest_ip)->toBe('10.20.0.44') // An address run, because writing the router is that pipeline's job. It is // allowed to start beside the restart because the guard asks whether the // run in flight carries out the address steps, and a restart carries // neither of them. ->and(ProvisioningRun::query()->where('pipeline', 'address')->count())->toBe(1); driveRun(ProvisioningRun::query()->where('pipeline', 'address')->sole()); expect($instance->fresh()->routed_backend)->toBe('10.20.0.44'); }); it('gives the shutdown step nothing to do when the guest is already stopped', function () { Queue::fake(); ['pve' => $pve, 'instance' => $instance, 'order' => $order] = restartableInstance(); $pve->runningVmids = []; $run = ProvisioningRun::factory()->create([ 'subject_type' => Order::class, 'subject_id' => $order->id, 'pipeline' => 'restart', 'context' => ['instance_id' => $instance->id, 'node' => 'pve', 'vmid' => 101], ]); expect(app(ShutDownVirtualMachine::class)->execute($run)->type)->toBe('advance') ->and($pve->shutdownCalls)->toBe([]); });