From d76de5ada323aaf49de5f97381c23c36a00e7e7e Mon Sep 17 00:00:00 2001 From: nexxo Date: Wed, 29 Jul 2026 18:28:28 +0200 Subject: [PATCH] Restart a machine, enforce the quota that was sold, end a route that ended MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three places where the product said one thing and did another. **Nothing could restart a virtual machine.** ProxmoxClient had startVm and no way to stop anything, so `restart_required_since` — set by every plan change that grows a running guest — could only ever be cleared by a resize step that happened to find the machine stopped, which nothing in the product could arrange. A paid upgrade's cores and RAM could stay unreached for the life of the contract, and the cloud page's "Neu starten" was a toast. There is now a `restart` pipeline: shutdown, start, wait for the agent, confirm. A shutdown and a start rather than a reboot, because only a cold boot makes qemu read the VM definition again — a reset would take the machine round and bring it back exactly as small as it was. The shutdown is a REQUEST, bounded at ten minutes, and there is no escalation: ProxmoxClient deliberately does not expose Proxmox's forceStop, because the guest is a Nextcloud and cutting power to a database mid-write to apply a CPU change trades an inconvenience for a restore from backup. A guest that ignores ACPI for ten minutes fails the run instead, and a person decides what happens to it. `restart_required_since` is cleared by the last step, from what the RUNNING guest reports (`cpus`/`maxmem`) — not by the button, and not by the run merely finishing. A machine that comes back smaller than it was sold fails the run. Started by App\Actions\RestartInstance, which re-checks authorisation itself rather than trusting a hidden button: an operator holding the new `instances.restart` may restart anything, a portal user their own machine and nothing else. Refused while another run is in flight against the same order. Confirmed in a modal on both sides (R23), addressed by uuid (R11). **Existing instances had no storage quota.** ApplyStorageQuota reaches new and changed machines; everything built before it kept the whole disk whatever it had paid for, and nothing recorded which was which. `instances.quota_applied_gb` is now what the guest was actually told, written by the step only after the occ call succeeds — so "sold" and "enforced" are separate facts and can be compared. `clupilot:apply-quotas` sweeps the estate: one `quota` run per instance through the same step the pipelines use, skipping what it should and saying why, a no-op once an allowance is enforced, and inert under --dry-run. Deliberately NOT scheduled — a nightly sweep would quietly cover for the pipeline step the day it stopped working, which is how the original hole survived this long. **A Traefik router was never torn down.** TraefikWriter::remove() had no caller anywhere. What was missing was the moment: ConfirmCancelPackage wrote a date into `service_ends_at` and nothing ever went back to it, so every route this platform ever wrote was written for good — pointing at a guest address the host is free to reassign. `clupilot:end-due-services` (hourly) keeps that appointment through App\Actions\EndInstanceService. "Ended" means cancellation_scheduled AND `service_ends_at` passed, never the day a cancellation is scheduled: that customer has paid to the end of the term and is working in it. The DNS record goes with the route — it is in our zone, pointing at a host that serves other people, which is the shape of every subdomain takeover there has ever been. The customer's own domain is in their zone and is not ours to touch; the virtual machine is left alone, because the cancellation flow promises a data export first and destroying disks is not this change's decision. Co-Authored-By: Claude Opus 5 --- app/Actions/EndInstanceService.php | 135 +++++++ app/Actions/RestartInstance.php | 163 +++++++++ app/Console/Commands/ApplyStorageQuotas.php | 167 +++++++++ app/Console/Commands/EndDueServices.php | 74 ++++ app/Livewire/Admin/ConfirmRestartInstance.php | 48 +++ app/Livewire/Admin/Instances.php | 50 +++ app/Livewire/Cloud.php | 63 +++- app/Livewire/ConfirmRestartCloud.php | 29 ++ app/Models/Instance.php | 22 +- .../Steps/Customer/ApplyStorageQuota.php | 8 + .../Steps/Customer/CompleteRestart.php | 117 +++++++ .../Steps/Customer/ShutDownVirtualMachine.php | 107 ++++++ app/Services/Proxmox/FakeProxmoxClient.php | 57 ++- app/Services/Proxmox/HttpProxmoxClient.php | 12 + app/Services/Proxmox/ProxmoxClient.php | 18 + config/provisioning.php | 37 ++ ...250000_add_instance_restart_capability.php | 42 +++ ..._260000_add_applied_quota_to_instances.php | 39 +++ lang/de/admin.php | 2 +- lang/de/cloud.php | 7 + lang/de/provisioning.php | 2 + lang/en/admin.php | 2 +- lang/en/cloud.php | 7 + lang/en/provisioning.php | 2 + resources/views/components/ui/badge.blade.php | 3 + .../admin/confirm-restart-instance.blade.php | 17 + .../views/livewire/admin/instances.blade.php | 15 +- resources/views/livewire/cloud.blade.php | 5 +- .../livewire/confirm-restart-cloud.blade.php | 17 + routes/console.php | 15 + tests/Feature/Admin/RbacMoveTest.php | 9 +- .../Console/ApplyStorageQuotasTest.php | 154 ++++++++ .../Provisioning/EndInstanceServiceTest.php | 198 +++++++++++ .../Provisioning/InstanceRestartTest.php | 330 ++++++++++++++++++ 34 files changed, 1961 insertions(+), 12 deletions(-) create mode 100644 app/Actions/EndInstanceService.php create mode 100644 app/Actions/RestartInstance.php create mode 100644 app/Console/Commands/ApplyStorageQuotas.php create mode 100644 app/Console/Commands/EndDueServices.php create mode 100644 app/Livewire/Admin/ConfirmRestartInstance.php create mode 100644 app/Livewire/ConfirmRestartCloud.php create mode 100644 app/Provisioning/Steps/Customer/CompleteRestart.php create mode 100644 app/Provisioning/Steps/Customer/ShutDownVirtualMachine.php create mode 100644 database/migrations/2026_07_29_250000_add_instance_restart_capability.php create mode 100644 database/migrations/2026_07_29_260000_add_applied_quota_to_instances.php create mode 100644 resources/views/livewire/admin/confirm-restart-instance.blade.php create mode 100644 resources/views/livewire/confirm-restart-cloud.blade.php create mode 100644 tests/Feature/Console/ApplyStorageQuotasTest.php create mode 100644 tests/Feature/Provisioning/EndInstanceServiceTest.php create mode 100644 tests/Feature/Provisioning/InstanceRestartTest.php diff --git a/app/Actions/EndInstanceService.php b/app/Actions/EndInstanceService.php new file mode 100644 index 0000000..2f687c9 --- /dev/null +++ b/app/Actions/EndInstanceService.php @@ -0,0 +1,135 @@ +status === 'cancellation_scheduled' + && $instance->service_ends_at !== null + && $instance->service_ends_at->lessThanOrEqualTo($at ?? now()); + } + + /** + * Tear the address down and mark the instance ended. + * + * Re-entrant: removing a router file that is not there is `rm -f`, a record + * id that has already been deleted is deleted again for nothing, and an + * instance already marked `ended` is returned false straight away. A DNS + * failure is deliberately not swallowed — the record id survives on its row, + * so a retry can finish the job, and a provider that is permanently broken + * becomes a visible failure rather than a silent leak. + * + * @return bool whether this call was the one that ended it + */ + public function __invoke(Instance $instance): bool + { + if (! $this->hasEnded($instance)) { + return false; + } + + $host = $instance->host; + + if ($host !== null) { + // The host that actually serves the traffic — DNS points at it, and + // it is where ConfigureDnsAndTls wrote the file in the first place. + $this->traefik->remove($host->wg_ip ?? $host->public_ip, $instance->subdomain); + } else { + // No host left to reach: the router file went with the machine it + // lived on. Said out loud rather than passed over, because it means + // this teardown is only doing half of what it usually does. + Log::warning('Ending an instance whose host is gone — no router to remove.', [ + 'instance' => $instance->uuid, + ]); + } + + foreach ($instance->dnsRecords()->get() as $record) { + $this->dns->deleteRecord($record->record_id); + $record->delete(); + } + + $instance->update([ + 'status' => 'ended', + // The address state has to go with the address. Left as it was, a + // later run would read `route_written` plus a matching hostname list + // and decide there was nothing to write — so an instance that was + // ever revived would be announced and routed nowhere. + 'route_written' => false, + 'routed_hostnames' => null, + 'cert_ok' => false, + 'domain_cert_ok' => false, + ]); + + Log::info('Instance service ended; address withdrawn.', [ + 'instance' => $instance->uuid, + 'subdomain' => $instance->subdomain, + 'service_ended_at' => $instance->service_ends_at?->toIso8601String(), + ]); + + return true; + } +} diff --git a/app/Actions/RestartInstance.php b/app/Actions/RestartInstance.php new file mode 100644 index 0000000..6e70a1b --- /dev/null +++ b/app/Actions/RestartInstance.php @@ -0,0 +1,163 @@ +user(); + + if ($operator !== null && $operator->isActive() && $operator->can('instances.restart')) { + return true; + } + + $customer = Customer::forUser(Auth::guard('web')->user()); + + return $customer !== null && (int) $instance->customer_id === (int) $customer->id; + } + + /** + * Start a restart run for this instance, or return null when there is + * nothing to start. + * + * Null is the ordinary "no" and not a failure — a machine that is still + * being built, one whose VM has gone, one whose service has ended, or one + * that already has a run in flight all have a better reason to be left + * alone than a restart has to interrupt them. A refusal on AUTHORISATION is + * the one case that throws, because it is not an ordinary outcome and the + * caller must not be able to mistake it for one. + * + * @throws AuthorizationException + */ + public function __invoke(Instance $instance): ?ProvisioningRun + { + if (! $this->allows($instance)) { + throw new AuthorizationException; + } + + if (! $instance->hasLiveMachine()) { + return null; + } + + // The same lock ReapplyInstanceAddress takes, for the same reason: two + // requests can reach this in one instant — the customer pressing the + // button while an operator presses it for them is the obvious pair — + // and two restart runs against one machine would have the second one + // shutting the guest down while the first waits for it to come back. + // Nobody waits for the lock: a restart that lost the race has nothing + // to add, because the run that won does exactly the same thing. + $lock = Cache::lock('instance-restart:'.$instance->uuid, 30); + + if (! $lock->get()) { + return null; + } + + try { + return $this->start($instance); + } finally { + $lock->release(); + } + } + + private function start(Instance $instance): ?ProvisioningRun + { + $order = $instance->order; + + // Checked against the ORDER, because that is the subject every customer + // pipeline shares. A restart beside an unfinished build would stop the + // machine the build is still writing to; a restart beside a plan change + // would race the config PUT it is about to boot. Whatever is in flight + // gets to finish, and the customer can press the button again after. + if ($this->hasRunInFlight($order)) { + return null; + } + + $run = ProvisioningRun::create([ + 'subject_type' => Order::class, + 'subject_id' => $order->id, + 'pipeline' => 'restart', + 'status' => ProvisioningRun::STATUS_PENDING, + 'current_step' => 0, + // Everything the four steps read: instance_id is how CustomerStep + // finds the machine, node and vmid are how it reaches it. + 'context' => [ + 'instance_id' => $instance->id, + 'host_id' => $instance->host_id, + 'node' => $instance->host?->node, + 'vmid' => $instance->vmid, + 'subdomain' => $instance->subdomain, + ], + ]); + + AdvanceRunJob::dispatch($run->uuid); + + Log::info('Restarting a customer instance.', [ + 'instance' => $instance->uuid, + 'restart_pending_since' => $instance->restart_required_since?->toIso8601String(), + 'run' => $run->uuid, + ]); + + return $run; + } + + private function hasRunInFlight(Order $order): bool + { + return ProvisioningRun::query() + ->where('subject_type', Order::class) + ->where('subject_id', $order->id) + ->whereIn('status', [ + ProvisioningRun::STATUS_PENDING, + ProvisioningRun::STATUS_RUNNING, + ProvisioningRun::STATUS_WAITING, + ProvisioningRun::STATUS_PAUSED, + ]) + ->exists(); + } +} diff --git a/app/Console/Commands/ApplyStorageQuotas.php b/app/Console/Commands/ApplyStorageQuotas.php new file mode 100644 index 0000000..9d98a05 --- /dev/null +++ b/app/Console/Commands/ApplyStorageQuotas.php @@ -0,0 +1,167 @@ +option('dry-run'); + $started = 0; + + /** @var array reason => count */ + $skipped = []; + + $query = Instance::query()->with(['order', 'host']); + + if ($uuid = $this->option('instance')) { + $query->where('uuid', $uuid); + } + + foreach ($query->cursor() as $instance) { + $reason = $this->reasonToSkip($instance); + + if ($reason !== null) { + $skipped[$reason] = ($skipped[$reason] ?? 0) + 1; + + continue; + } + + $started++; + + $this->line(($dryRun ? '[dry-run] ' : '') + ."{$instance->subdomain}: {$instance->quota_gb} GB " + .'('.($instance->quota_applied_gb === null ? 'noch nie gesetzt' : "zuletzt {$instance->quota_applied_gb} GB").')'); + + if (! $dryRun) { + $this->startQuotaRun($instance); + } + } + + foreach ($skipped as $reason => $count) { + $this->line("übersprungen ({$reason}): {$count}"); + } + + $this->info($dryRun + ? "Probelauf: {$started} Instanz(en) bekämen ihr Kontingent, ".array_sum($skipped).' übersprungen. Nichts wurde geändert.' + : "{$started} Kontingent-Lauf/Läufe gestartet, ".array_sum($skipped).' übersprungen.'); + + return self::SUCCESS; + } + + /** + * Why this instance is being left alone, or null when it is not. + * + * The reason is the report — a sweep that says "skipped 34" and nothing else + * is a sweep nobody can act on, because "34 machines have no allowance + * enforced" and "34 machines are already fine" look identical from here. + */ + private function reasonToSkip(Instance $instance): ?string + { + // Nothing remote work can reach: a reservation with no VM, a failed + // build, an ended service. The same rule every other maintenance action + // asks — see Instance::hasLiveMachine(). + if (! $instance->hasLiveMachine()) { + return 'keine erreichbare Maschine'; + } + + // No allowance on the row at all. Writing "0 GB" would lock the customer + // out of their own files, which is a far worse answer than leaving what + // is there and letting somebody notice — the same decision the step + // itself makes, made here too so the machine is not visited for nothing. + if ((int) $instance->quota_gb <= 0) { + return 'kein Kontingent hinterlegt'; + } + + if ($instance->quotaIsEnforced()) { + return 'bereits gesetzt'; + } + + // Whatever is in flight against this order gets to finish. A quota run + // beside an unfinished build would race the same occ calls, and the + // build applies the quota itself anyway. + if ($this->hasRunInFlight($instance)) { + return 'anderer Lauf aktiv'; + } + + return null; + } + + private function startQuotaRun(Instance $instance): void + { + $run = ProvisioningRun::create([ + // The instance's own purchase order, as every customer pipeline + // uses — that is what CustomerStep::order() and ::instance() + // resolve from, and what the in-flight check above looks at. + 'subject_type' => Order::class, + 'subject_id' => $instance->order_id, + 'pipeline' => 'quota', + 'status' => ProvisioningRun::STATUS_PENDING, + 'current_step' => 0, + 'context' => [ + 'instance_id' => $instance->id, + 'host_id' => $instance->host_id, + 'node' => $instance->host?->node, + 'vmid' => $instance->vmid, + 'subdomain' => $instance->subdomain, + ], + ]); + + AdvanceRunJob::dispatch($run->uuid); + } + + private function hasRunInFlight(Instance $instance): bool + { + return ProvisioningRun::query() + ->where('subject_type', Order::class) + ->where('subject_id', $instance->order_id) + ->whereIn('status', [ + ProvisioningRun::STATUS_PENDING, + ProvisioningRun::STATUS_RUNNING, + ProvisioningRun::STATUS_WAITING, + ProvisioningRun::STATUS_PAUSED, + ]) + ->exists(); + } +} diff --git a/app/Console/Commands/EndDueServices.php b/app/Console/Commands/EndDueServices.php new file mode 100644 index 0000000..c4bb27a --- /dev/null +++ b/app/Console/Commands/EndDueServices.php @@ -0,0 +1,74 @@ +option('dry-run'); + $ended = 0; + + // Narrowed in the query for the sake of the database, decided by + // hasEnded() for the sake of the rule: the query is an index, not a + // second opinion. + $due = Instance::query() + ->with(['host', 'dnsRecords']) + ->where('status', 'cancellation_scheduled') + ->whereNotNull('service_ends_at') + ->where('service_ends_at', '<=', now()) + ->cursor(); + + foreach ($due as $instance) { + if (! $end->hasEnded($instance)) { + continue; + } + + $this->line(($dryRun ? '[dry-run] ' : '') + ."{$instance->subdomain}: Laufzeit endete am " + .$instance->service_ends_at->local()->isoFormat('LLL')); + + if ($dryRun || $end($instance)) { + $ended++; + } + } + + $this->info($dryRun + ? "Probelauf: {$ended} Dienst(e) würden beendet. Nichts wurde geändert." + : "{$ended} Dienst(e) beendet."); + + return self::SUCCESS; + } +} diff --git a/app/Livewire/Admin/ConfirmRestartInstance.php b/app/Livewire/Admin/ConfirmRestartInstance.php new file mode 100644 index 0000000..8ffb181 --- /dev/null +++ b/app/Livewire/Admin/ConfirmRestartInstance.php @@ -0,0 +1,48 @@ +where('uuid', $uuid)->firstOrFail(); + + $this->uuid = $uuid; + $this->address = $instance->domainIsVerified() + ? (string) $instance->custom_domain + : (string) $instance->subdomain; + } + + public function proceed(): void + { + $this->dispatch('instance-restart-confirmed', uuid: $this->uuid); + $this->closeModal(); + } + + public function render() + { + return view('livewire.admin.confirm-restart-instance'); + } +} diff --git a/app/Livewire/Admin/Instances.php b/app/Livewire/Admin/Instances.php index 708475b..5346762 100644 --- a/app/Livewire/Admin/Instances.php +++ b/app/Livewire/Admin/Instances.php @@ -2,9 +2,13 @@ namespace App\Livewire\Admin; +use App\Actions\RestartInstance; use App\Models\Instance; +use Illuminate\Auth\Access\AuthorizationException; +use Illuminate\Support\Facades\Gate; use Illuminate\Support\Facades\Lang; use Livewire\Attributes\Layout; +use Livewire\Attributes\On; use Livewire\Component; use Livewire\WithPagination; @@ -22,6 +26,42 @@ class Instances extends Component { use WithPagination; + /** + * Restart one instance, once ConfirmRestartInstance has said so. + * + * The capability is checked twice on purpose, and neither check is the + * decoration of the other. This one keeps an operator without + * `instances.restart` from driving the console component at all; the one + * inside RestartInstance is what actually guards the machine, because that + * is the door a hand-written POST to /livewire/update arrives at. Losing + * either would still leave a customer's cloud reachable by somebody who was + * only ever meant to read the list. + */ + #[On('instance-restart-confirmed')] + public function restart(string $uuid): void + { + $this->authorize('instances.restart'); + + $instance = Instance::query()->where('uuid', $uuid)->first(); + + if ($instance === null) { + return; + } + + try { + $run = app(RestartInstance::class)($instance); + } catch (AuthorizationException) { + $this->dispatch('notify', message: __('admin.restart_denied')); + + return; + } + + // Null is every ordinary refusal — no live machine, or a run already in + // flight against this order. Said out loud, because an operator who + // pressed a button and saw nothing will press it again. + $this->dispatch('notify', message: __($run === null ? 'admin.restart_busy' : 'admin.restart_started')); + } + public function render() { $instances = Instance::query() @@ -31,7 +71,17 @@ class Instances extends Component return view('livewire.admin.instances', [ 'instances' => $instances, + // Whether to draw the action column at all. An operator who may not + // restart anything should not be reading a column of buttons that + // answer them 403. + 'canRestart' => Gate::allows('instances.restart'), 'rows' => $instances->getCollection()->map(fn (Instance $i) => [ + // R11: a row's action is addressed by uuid, never the numeric id. + 'uuid' => $i->uuid, + // Only a machine remote work can actually reach. A reservation + // with no VM, a failed build and an ended service all have + // nothing to restart — see Instance::hasLiveMachine(). + 'live' => $i->hasLiveMachine(), // The operator list shows what is actually served — an unverified // custom domain is a plan, not an address. 'address' => $i->domainIsVerified() ? $i->custom_domain : $i->subdomain, diff --git a/app/Livewire/Cloud.php b/app/Livewire/Cloud.php index 1c1260a..6ac471d 100644 --- a/app/Livewire/Cloud.php +++ b/app/Livewire/Cloud.php @@ -2,11 +2,15 @@ namespace App\Livewire; +use App\Actions\RestartInstance; use App\Livewire\Concerns\ResolvesCustomer; +use App\Models\Instance; use App\Models\MaintenanceWindow; use App\Support\ProvisioningSettings; +use Illuminate\Auth\Access\AuthorizationException; use Illuminate\Support\Number; use Livewire\Attributes\Layout; +use Livewire\Attributes\On; use Livewire\Component; #[Layout('layouts.portal-app')] @@ -14,6 +18,56 @@ class Cloud extends Component { use ResolvesCustomer; + /** + * Restart the customer's own cloud, once ConfirmRestartCloud has said so. + * + * The modal raises the event and this method does the work, so the rule + * about who may restart what stays in one place (R23). That place is + * RestartInstance itself, not this method: the instance below is resolved + * from the signed-in customer, so it is already theirs, but the action + * re-checks anyway — a Livewire method is reachable by anyone who can POST + * to /livewire/update, and an authorisation that lives only in how a caller + * happened to look something up is one refactor away from being gone. + */ + #[On('cloud-restart-confirmed')] + public function restart(): void + { + $instance = $this->instance(); + + if ($instance === null) { + $this->dispatch('notify', message: __('dashboard.no_customer_action')); + + return; + } + + try { + $run = app(RestartInstance::class)($instance); + } catch (AuthorizationException) { + $this->dispatch('notify', message: __('cloud.restart_denied')); + + return; + } + + // Null is every ordinary "not now": a machine still being built, one + // whose service has ended, or one with a run already in flight. The + // customer gets told to try again rather than being left with a button + // that appears to have done nothing. + $this->dispatch('notify', message: __($run === null ? 'cloud.restart_busy' : 'cloud.restart_started')); + } + + /** + * The customer's service instance — the one the whole page is about. + * + * Resolved here rather than inside render() so the restart action and the + * card cannot end up talking about two different machines. + */ + private function instance(): ?Instance + { + return $this->customer()?->instances() + ->whereIn('status', ['active', 'provisioning', 'cancellation_scheduled']) + ->latest('id')->first(); + } + public function render() { $locale = app()->getLocale(); @@ -22,9 +76,7 @@ class Cloud extends Component // The card is built from the customer's real service instance, so the // maintenance badge below can be scoped to exactly that instance's host. $customer = $this->customer(); - $shown = $customer?->instances() - ->whereIn('status', ['active', 'provisioning', 'cancellation_scheduled']) - ->latest('id')->first(); + $shown = $this->instance(); $maintenance = MaintenanceWindow::forInstance($shown)->first(); // The instance carries what was actually provisioned, and the contract @@ -101,6 +153,11 @@ class Cloud extends Component // purchase, so the customer is told instead — otherwise they are // paying for cores they cannot see and have no way of finding out. 'restartPending' => $shown?->restartIsPending() ?? false, + // Whether there is a machine a restart could reach at all. The + // button is drawn either way — hiding it would leave a customer + // wondering where it went during a build — but it is only live when + // pressing it can do something. + 'canRestart' => $shown?->hasLiveMachine() ?? false, ]); } } diff --git a/app/Livewire/ConfirmRestartCloud.php b/app/Livewire/ConfirmRestartCloud.php new file mode 100644 index 0000000..2e1ecca --- /dev/null +++ b/app/Livewire/ConfirmRestartCloud.php @@ -0,0 +1,29 @@ +dispatch('cloud-restart-confirmed'); + $this->closeModal(); + } + + public function render() + { + return view('livewire.confirm-restart-cloud'); + } +} diff --git a/app/Models/Instance.php b/app/Models/Instance.php index 733ae2a..b7def82 100644 --- a/app/Models/Instance.php +++ b/app/Models/Instance.php @@ -17,7 +17,7 @@ class Instance extends Model use HasFactory, HasUuid; protected $fillable = [ - 'customer_id', 'order_id', 'host_id', 'vmid', 'guest_ip', 'plan', 'quota_gb', 'traffic_addons', 'disk_gb', + 'customer_id', 'order_id', 'host_id', 'vmid', 'guest_ip', 'plan', 'quota_gb', 'quota_applied_gb', 'traffic_addons', 'disk_gb', 'ram_mb', 'cores', 'restart_required_since', 'subdomain', 'custom_domain', 'nc_admin_ref', 'admin_password', 'credentials_acknowledged_at', 'route_written', 'routed_hostnames', 'cert_ok', 'status', 'cancel_requested_at', 'service_ends_at', @@ -46,6 +46,7 @@ class Instance extends Model 'vmid' => 'integer', 'traffic_addons' => 'integer', 'quota_gb' => 'integer', + 'quota_applied_gb' => 'integer', 'disk_gb' => 'integer', 'ram_mb' => 'integer', 'cores' => 'integer', @@ -69,6 +70,25 @@ class Instance extends Model return $this->restart_required_since !== null; } + /** + * Has this machine's Nextcloud actually been told the allowance that was + * sold? + * + * Two columns rather than one because they answer two different questions: + * `quota_gb` is what the customer bought, `quota_applied_gb` is what the + * guest was last told. They agree on every machine the quota step has + * visited since it started recording, and they disagree on exactly the two + * cases worth finding — the estate built before the step existed, and any + * machine whose package changed while the run that should have followed did + * not finish. + */ + public function quotaIsEnforced(): bool + { + return $this->quota_gb !== null + && $this->quota_gb > 0 + && (int) $this->quota_applied_gb === (int) $this->quota_gb; + } + /** * Is there a machine here that remote work can actually reach? * diff --git a/app/Provisioning/Steps/Customer/ApplyStorageQuota.php b/app/Provisioning/Steps/Customer/ApplyStorageQuota.php index 3e62361..affc2bd 100644 --- a/app/Provisioning/Steps/Customer/ApplyStorageQuota.php +++ b/app/Provisioning/Steps/Customer/ApplyStorageQuota.php @@ -59,6 +59,14 @@ class ApplyStorageQuota extends CustomerStep // same value again is a no-op and exits 0. $this->guest($pve, $run, $occ.'config:app:set files default_quota --value='.escapeshellarg($quota.' GB')); + // Recorded only AFTER the guest accepted it — guest() throws on a + // non-zero exit, so nothing below runs for a call that failed. Without + // this line there is no way to tell a machine whose allowance is + // enforced from one where the figure has only ever been a row in our + // database, which is precisely how every instance built before this step + // existed came to have the whole disk. See clupilot:apply-quotas. + $instance->update(['quota_applied_gb' => $quota]); + return StepResult::advance(); } } diff --git a/app/Provisioning/Steps/Customer/CompleteRestart.php b/app/Provisioning/Steps/Customer/CompleteRestart.php new file mode 100644 index 0000000..9db6daf --- /dev/null +++ b/app/Provisioning/Steps/Customer/CompleteRestart.php @@ -0,0 +1,117 @@ +instance($run); + + if ($instance === null) { + return StepResult::fail('no_instance'); + } + + $node = (string) $run->context('node'); + $vmid = (int) $run->context('vmid'); + $status = $this->pve->forHost($instance->host)->vmStatus($node, $vmid); + + if (($status['status'] ?? '') !== 'running') { + if ($run->started_at !== null && $run->started_at->copy()->addSeconds(self::RUNNING_DEADLINE)->isPast()) { + return StepResult::fail('restart_did_not_come_back'); + } + + return StepResult::poll(5, 'waiting for the machine to come back'); + } + + if (($short = $this->shortfall($status, $instance->cores, $instance->ram_mb)) !== null) { + return StepResult::fail('restart_applied_nothing: '.$short); + } + + // Only here. The machine is running, its guest agent has answered (the + // step before this one), and it is running on what was sold. + if ($instance->restartIsPending()) { + $instance->update(['restart_required_since' => null]); + } + + $run->events()->create([ + 'step' => $this->key(), + 'attempt' => $run->attempt, + 'outcome' => 'info', + 'message' => "Die Maschine läuft wieder mit {$instance->cores} vCPU und {$instance->ram_mb} MB RAM. " + .'Der offene Neustart ist damit erledigt.', + ]); + + return StepResult::advance(); + } + + /** + * What the running guest is short of, or null when it is not short of + * anything we can see. + * + * Compared with `>=` rather than `===`: Proxmox rounds memory and a host may + * legitimately give a guest more than was asked for, and refusing a machine + * for being too big would be a strange way to protect a customer. + * + * @param array $status + */ + private function shortfall(array $status, ?int $cores, ?int $ramMb): ?string + { + if (isset($status['cpus']) && $cores > 0 && (int) $status['cpus'] < $cores) { + return "{$status['cpus']} statt {$cores} vCPU"; + } + + if (isset($status['maxmem']) && $ramMb > 0) { + $actualMb = (int) ((int) $status['maxmem'] / 1048576); + + if ($actualMb < $ramMb) { + return "{$actualMb} statt {$ramMb} MB RAM"; + } + } + + return null; + } +} diff --git a/app/Provisioning/Steps/Customer/ShutDownVirtualMachine.php b/app/Provisioning/Steps/Customer/ShutDownVirtualMachine.php new file mode 100644 index 0000000..3423a11 --- /dev/null +++ b/app/Provisioning/Steps/Customer/ShutDownVirtualMachine.php @@ -0,0 +1,107 @@ +instance($run); + + if ($instance === null) { + return StepResult::fail('no_instance'); + } + + $node = (string) $run->context('node'); + $vmid = (int) $run->context('vmid'); + $pve = $this->pve->forHost($instance->host); + + // Idempotent, and the ordinary way this step ends: the guest is gone. + if (($pve->vmStatus($node, $vmid)['status'] ?? '') !== 'running') { + return StepResult::advance(); + } + + // Asked once, remembered on the run. Re-sending ACPI to a guest that is + // already half-way through its own shutdown achieves nothing and reads, + // in the guest's log, like a second person pressing the power button. + if ($run->context('shutdown_upid') === null) { + $run->mergeContext([ + 'shutdown_upid' => $pve->shutdownVm($node, $vmid, self::GRACE_SECONDS), + ]); + + return StepResult::poll(10, 'waiting for the guest to shut down'); + } + + // A poll does not reset started_at, so this deadline genuinely + // accumulates across the whole wait rather than restarting every time. + if ($run->started_at !== null && $run->started_at->copy()->addSeconds(self::GRACE_SECONDS)->isPast()) { + $run->events()->create([ + 'step' => $this->key(), + 'attempt' => $run->attempt, + 'outcome' => 'info', + 'message' => 'Der Gast hat sich in '.self::GRACE_SECONDS.' Sekunden nicht heruntergefahren. ' + .'Die Maschine läuft unverändert weiter — sie wird bewusst nicht hart ausgeschaltet, ' + .'weil eine Nextcloud eine laufende Datenbank ist.', + ]); + + return StepResult::fail('shutdown_timeout'); + } + + return StepResult::poll(10, 'waiting for the guest to shut down'); + } +} diff --git a/app/Services/Proxmox/FakeProxmoxClient.php b/app/Services/Proxmox/FakeProxmoxClient.php index ce30e53..3107eec 100644 --- a/app/Services/Proxmox/FakeProxmoxClient.php +++ b/app/Services/Proxmox/FakeProxmoxClient.php @@ -100,6 +100,12 @@ class FakeProxmoxClient implements ProxmoxClient { $this->cloudInitCalls[] = $vmid; $this->cloudInitParams[$vmid] = $params; + // A config PUT lands on the VM's DEFINITION, not on the qemu process + // that is already running with the old one. Kept apart from + // $bootedConfig for exactly that reason — it is the whole point of a + // pending restart, and a fake that merged the two would let a test + // "prove" a resize had taken effect on a machine that never stopped. + $this->vmConfig[$vmid] = array_merge($this->vmConfig[$vmid] ?? [], $params); } public function resizeDisk(string $node, int $vmid, string $disk, string $size): void @@ -107,13 +113,45 @@ class FakeProxmoxClient implements ProxmoxClient $this->resizeCalls[] = $vmid.':'.$disk.':'.$size; } + /** @var array> vmid => the VM's definition on disk */ + public array $vmConfig = []; + + /** @var array> vmid => the definition the running guest booted with */ + public array $bootedConfig = []; + + /** @var array every graceful shutdown that was asked for */ + public array $shutdownCalls = []; + + /** + * A guest that will not go away when it is asked to — an unresponsive + * ACPI handler, a machine busy with something it will not be interrupted + * in. Set it to prove what the product does when the polite request is + * ignored, which is the case the whole shutdown design turns on. + */ + public bool $shutdownIgnored = false; + public function startVm(string $node, int $vmid): string { $this->runningVmids[] = $vmid; + // A cold boot is where the definition becomes what the guest actually + // runs on. This is the only place $bootedConfig is filled. + $this->bootedConfig[$vmid] = $this->vmConfig[$vmid] ?? []; return 'UPID:pve:qmstart:'.$vmid; } + public function shutdownVm(string $node, int $vmid, int $timeoutSeconds): string + { + $this->shutdownCalls[] = ['vmid' => $vmid, 'timeout' => $timeoutSeconds]; + + if (! $this->shutdownIgnored) { + $this->runningVmids = array_values(array_diff($this->runningVmids, [$vmid])); + unset($this->bootedConfig[$vmid]); + } + + return 'UPID:pve:qmshutdown:'.$vmid; + } + /** Set to e.g. 'clone' to simulate a VM still locked by a running clone task. */ public ?string $vmLock = null; @@ -125,12 +163,28 @@ class FakeProxmoxClient implements ProxmoxClient public function vmStatus(string $node, int $vmid): array { - return [ + $status = [ 'status' => in_array($vmid, $this->runningVmids, true) ? 'running' : 'stopped', 'lock' => $this->vmLock, 'netin' => $this->counters[$vmid]['netin'] ?? 0, 'netout' => $this->counters[$vmid]['netout'] ?? 0, ]; + + // Only for a guest this fake has actually booted. Proxmox reports what + // the RUNNING machine has, so a test that simply put a vmid in + // $runningVmids has said nothing about its size and must not have an + // invented figure answered back to it. + $booted = $this->bootedConfig[$vmid] ?? []; + + if (isset($booted['cores'])) { + $status['cpus'] = (int) $booted['cores']; + } + + if (isset($booted['memory'])) { + $status['maxmem'] = (int) $booted['memory'] * 1048576; // Proxmox reports bytes + } + + return $status; } public function setNetworkRate(string $node, int $vmid, ?float $mbytesPerSecond): void @@ -151,6 +205,7 @@ class FakeProxmoxClient implements ProxmoxClient $this->deletedVmids[] = $vmid; $this->clonedVmids = array_values(array_diff($this->clonedVmids, [$vmid])); $this->runningVmids = array_values(array_diff($this->runningVmids, [$vmid])); + unset($this->vmConfig[$vmid], $this->bootedConfig[$vmid]); } public function guestAgentPing(string $node, int $vmid): bool diff --git a/app/Services/Proxmox/HttpProxmoxClient.php b/app/Services/Proxmox/HttpProxmoxClient.php index 430eafa..7bc7fc2 100644 --- a/app/Services/Proxmox/HttpProxmoxClient.php +++ b/app/Services/Proxmox/HttpProxmoxClient.php @@ -102,6 +102,18 @@ class HttpProxmoxClient implements ProxmoxClient ->post("/nodes/{$node}/qemu/{$vmid}/status/start")->throw()->json('data'); } + public function shutdownVm(string $node, int $vmid, int $timeoutSeconds): string + { + // forceStop is hard-coded to 0 and is not a parameter of this method: + // with it set, Proxmox destroys the qemu process once `timeout` expires, + // which is a power cut to a running database. Left at 0 the task simply + // ends in an error and the guest keeps running, which is the outcome the + // calling step is written around. + return (string) $this->http()->asForm() + ->post("/nodes/{$node}/qemu/{$vmid}/status/shutdown", ['timeout' => $timeoutSeconds, 'forceStop' => 0]) + ->throw()->json('data'); + } + public function vmStatus(string $node, int $vmid): array { return $this->http()->get("/nodes/{$node}/qemu/{$vmid}/status/current")->throw()->json('data', []); diff --git a/app/Services/Proxmox/ProxmoxClient.php b/app/Services/Proxmox/ProxmoxClient.php index 52e4405..0a341a1 100644 --- a/app/Services/Proxmox/ProxmoxClient.php +++ b/app/Services/Proxmox/ProxmoxClient.php @@ -48,6 +48,24 @@ interface ProxmoxClient /** Start a VM; returns the task UPID. */ public function startVm(string $node, int $vmid): string; + /** + * ASK the guest to shut itself down, and return the task UPID. + * + * There is deliberately no counterpart that pulls the plug. Proxmox's + * shutdown endpoint takes a `forceStop` flag that turns the request into a + * power cut once the timeout runs out, and this interface does not expose + * it: every guest we run is a Nextcloud, which is a database, and cutting + * power to one mid-write to make a CPU change take effect trades a visible + * inconvenience for a possible restore from backup. A guest that ignores + * ACPI for the whole grace period is a guest that is doing something, and + * the right answer there is a failed run an operator can look at — see + * App\Provisioning\Steps\Customer\ShutDownVirtualMachine. + * + * $timeoutSeconds is how long Proxmox itself waits for the guest before it + * gives the task up. + */ + public function shutdownVm(string $node, int $vmid, int $timeoutSeconds): string; + /** @return array ['status' => 'running'|'stopped', …] */ public function vmStatus(string $node, int $vmid): array; diff --git a/config/provisioning.php b/config/provisioning.php index a137368..2177dc9 100644 --- a/config/provisioning.php +++ b/config/provisioning.php @@ -109,6 +109,43 @@ return [ Customer\ConfigureNextcloud::class, Customer\SettlePlanServices::class, ], + + /* + | Taking a customer's machine round once, on purpose. + | + | A restart is a shutdown and a start, in that order and never a reboot: + | only a cold boot makes qemu read the VM definition again, so only a + | cold boot delivers the cores and memory a plan change wrote there. + | ResizeVirtualMachine leaves `restart_required_since` set precisely + | because nothing in the product could do this, which is how a paid + | upgrade could stay invisible forever. + | + | StartVirtualMachine and WaitForGuestAgent are the customer pipeline's + | own steps, reused rather than repeated — starting a machine and waiting + | for its agent is the same operation whichever run needs it. Started by + | App\Actions\RestartInstance, from the customer's cloud page or the + | console's instance list. + */ + 'restart' => [ + Customer\ShutDownVirtualMachine::class, + Customer\StartVirtualMachine::class, + Customer\WaitForGuestAgent::class, + Customer\CompleteRestart::class, + ], + + /* + | The storage allowance, and nothing else. + | + | One step, because that is the whole of the repair: every instance built + | before ApplyStorageQuota joined the `customer` pipeline has a quota_gb + | in its row that Nextcloud was never told about. Started one run per + | instance by `clupilot:apply-quotas`, so the sweep is retried, logged + | and visible in the console like every other piece of remote work, + | rather than a console command reaching into guests by hand. + */ + 'quota' => [ + Customer\ApplyStorageQuota::class, + ], ], // The one currency the catalogue is priced in. Plan prices carry no currency diff --git a/database/migrations/2026_07_29_250000_add_instance_restart_capability.php b/database/migrations/2026_07_29_250000_add_instance_restart_capability.php new file mode 100644 index 0000000..193be7a --- /dev/null +++ b/database/migrations/2026_07_29_250000_add_instance_restart_capability.php @@ -0,0 +1,42 @@ +forgetCachedPermissions(); + + Permission::findOrCreate('instances.restart', 'operator'); + foreach (['Owner', 'Admin', 'Support'] as $role) { + Role::findOrCreate($role, 'operator')->givePermissionTo('instances.restart'); + } + + app(PermissionRegistrar::class)->forgetCachedPermissions(); + } + + public function down(): void + { + app(PermissionRegistrar::class)->forgetCachedPermissions(); + Permission::query()->where('name', 'instances.restart')->where('guard_name', 'operator')->delete(); + app(PermissionRegistrar::class)->forgetCachedPermissions(); + } +}; diff --git a/database/migrations/2026_07_29_260000_add_applied_quota_to_instances.php b/database/migrations/2026_07_29_260000_add_applied_quota_to_instances.php new file mode 100644 index 0000000..461749f --- /dev/null +++ b/database/migrations/2026_07_29_260000_add_applied_quota_to_instances.php @@ -0,0 +1,39 @@ +unsignedInteger('quota_applied_gb')->nullable()->after('quota_gb'); + }); + } + + public function down(): void + { + Schema::table('instances', function (Blueprint $table) { + $table->dropColumn('quota_applied_gb'); + }); + } +}; diff --git a/lang/de/admin.php b/lang/de/admin.php index a267030..30ee166 100644 --- a/lang/de/admin.php +++ b/lang/de/admin.php @@ -79,7 +79,7 @@ return [ ], 'notice' => [ - 'no_capacity' => "Kein Host hat noch Platz für „:plan“ — das Paket braucht :needs GB, der freieste Host hat :largest GB. Eine bezahlte Bestellung dafür würde bei der Reservierung scheitern.", + 'no_capacity' => 'Kein Host hat noch Platz für „:plan“ — das Paket braucht :needs GB, der freieste Host hat :largest GB. Eine bezahlte Bestellung dafür würde bei der Reservierung scheitern.', 'failed_runs' => ':n fehlgeschlagene Bereitstellung(en).', 'host_error' => 'Host :host meldet einen Fehler.', 'host_silent' => 'Host :host hat sich seit über :minutes Minuten nicht gemeldet.', diff --git a/lang/de/cloud.php b/lang/de/cloud.php index 765fa84..0b2c0d0 100644 --- a/lang/de/cloud.php +++ b/lang/de/cloud.php @@ -33,6 +33,13 @@ return [ 'restart_pending_title' => 'Neustart erforderlich', 'restart_pending_body' => 'Ihr neues Paket ist aktiv. Mehr Prozessorkerne und Arbeitsspeicher werden erst beim nächsten Neustart Ihrer Cloud wirksam — Ihre Cloud wurde dafür bewusst nicht von selbst neu gestartet.', 'restart' => 'Neu starten', + 'restart_title' => 'Cloud neu starten?', + 'restart_body' => 'Ihre Cloud wird sauber heruntergefahren und wieder gestartet. Das dauert meist ein bis zwei Minuten; in dieser Zeit ist sie nicht erreichbar, und alle angemeldeten Benutzer werden getrennt. Nicht gespeicherte Arbeit in Office-Dokumenten geht dabei verloren.', + 'restart_cancel' => 'Abbrechen', + 'restart_confirm' => 'Jetzt neu starten', + 'restart_started' => 'Neustart wurde gestartet. Ihre Cloud ist in wenigen Minuten wieder erreichbar.', + 'restart_busy' => 'Gerade läuft schon eine Arbeit an Ihrer Cloud. Bitte versuchen Sie es in einigen Minuten erneut.', + 'restart_denied' => 'Für diese Aktion fehlt die Berechtigung.', 'snapshot' => 'Snapshot erstellen', 'logs' => 'Protokolle', 'action_toast' => 'Aktion im Prototyp nur angedeutet.', diff --git a/lang/de/provisioning.php b/lang/de/provisioning.php index 09d64bf..e5d32e4 100644 --- a/lang/de/provisioning.php +++ b/lang/de/provisioning.php @@ -20,6 +20,8 @@ return [ 'resize_vm' => 'VM-Ausstattung anpassen', 'apply_storage_quota' => 'Speicherkontingent setzen', 'settle_plan_services' => 'Backup & Monitoring abgleichen', + 'shut_down_vm' => 'VM sauber herunterfahren', + 'complete_restart' => 'Neustart abschließen', ], 'mail' => [ diff --git a/lang/en/admin.php b/lang/en/admin.php index 544612d..f287e61 100644 --- a/lang/en/admin.php +++ b/lang/en/admin.php @@ -79,7 +79,7 @@ return [ ], 'notice' => [ - 'no_capacity' => "No host has room for “:plan” any more — the package needs :needs GB and the roomiest host has :largest GB. A paid order for it would fail at reservation.", + 'no_capacity' => 'No host has room for “:plan” any more — the package needs :needs GB and the roomiest host has :largest GB. A paid order for it would fail at reservation.', 'failed_runs' => ':n failed provisioning run(s).', 'host_error' => 'Host :host reports an error.', 'host_silent' => 'Host :host has not checked in for over :minutes minutes.', diff --git a/lang/en/cloud.php b/lang/en/cloud.php index bf22320..59d5706 100644 --- a/lang/en/cloud.php +++ b/lang/en/cloud.php @@ -33,6 +33,13 @@ return [ 'restart_pending_title' => 'Restart required', 'restart_pending_body' => 'Your new plan is active. The extra CPU cores and memory only take effect the next time your cloud restarts — we deliberately did not restart it for you.', 'restart' => 'Restart', + 'restart_title' => 'Restart your cloud?', + 'restart_body' => 'Your cloud will be shut down cleanly and started again. That usually takes a minute or two, during which it is unreachable and every signed-in user is disconnected. Unsaved work in office documents will be lost.', + 'restart_cancel' => 'Cancel', + 'restart_confirm' => 'Restart now', + 'restart_started' => 'The restart has begun. Your cloud will be back in a few minutes.', + 'restart_busy' => 'Work on your cloud is already under way. Please try again in a few minutes.', + 'restart_denied' => 'You are not allowed to do that.', 'snapshot' => 'Create snapshot', 'logs' => 'Logs', 'action_toast' => 'Action is only indicated in this prototype.', diff --git a/lang/en/provisioning.php b/lang/en/provisioning.php index 3207f14..384bebf 100644 --- a/lang/en/provisioning.php +++ b/lang/en/provisioning.php @@ -20,6 +20,8 @@ return [ 'resize_vm' => 'Resize the VM', 'apply_storage_quota' => 'Apply storage quota', 'settle_plan_services' => 'Reconcile backup & monitoring', + 'shut_down_vm' => 'Shut the VM down cleanly', + 'complete_restart' => 'Finish the restart', ], 'mail' => [ diff --git a/resources/views/components/ui/badge.blade.php b/resources/views/components/ui/badge.blade.php index a66823f..c50fea2 100644 --- a/resources/views/components/ui/badge.blade.php +++ b/resources/views/components/ui/badge.blade.php @@ -10,6 +10,9 @@ 'pending' => 'info', 'suspended' => 'warning', 'cancellation_scheduled' => 'warning', + // A service that has run out. Not an error and not a health state — + // the same muted "this is over" as a closed account. + 'ended' => 'warning', 'closed' => 'warning', 'warning' => 'warning', 'failed' => 'danger', diff --git a/resources/views/livewire/admin/confirm-restart-instance.blade.php b/resources/views/livewire/admin/confirm-restart-instance.blade.php new file mode 100644 index 0000000..371c6f5 --- /dev/null +++ b/resources/views/livewire/admin/confirm-restart-instance.blade.php @@ -0,0 +1,17 @@ +
+
+ + + +
+

{{ __('admin.restart_title') }}

+

{{ __('admin.restart_body', ['address' => $address]) }}

+
+
+
+ {{ __('admin.restart_cancel') }} + + {{ __('admin.restart_confirm') }} + +
+
diff --git a/resources/views/livewire/admin/instances.blade.php b/resources/views/livewire/admin/instances.blade.php index a61384e..612a558 100644 --- a/resources/views/livewire/admin/instances.blade.php +++ b/resources/views/livewire/admin/instances.blade.php @@ -16,6 +16,9 @@ {{ __('admin.col.plan') }} {{ __('admin.col.quota') }} {{ __('admin.col.status') }} + @if ($canRestart) + {{ __('admin.col.actions') }} + @endif @@ -38,9 +41,19 @@ @endif + @if ($canRestart) + + @if ($r['live']) + + {{ __('admin.restart') }} + + @endif + + @endif @empty - {{ __('admin.instances_empty') }} + {{ __('admin.instances_empty') }} @endforelse diff --git a/resources/views/livewire/cloud.blade.php b/resources/views/livewire/cloud.blade.php index 2ef79b0..a3888fa 100644 --- a/resources/views/livewire/cloud.blade.php +++ b/resources/views/livewire/cloud.blade.php @@ -71,7 +71,10 @@
- {{ __('cloud.restart') }} + + {{ __('cloud.restart') }} + {{ __('cloud.snapshot') }} {{ __('cloud.logs') }}
diff --git a/resources/views/livewire/confirm-restart-cloud.blade.php b/resources/views/livewire/confirm-restart-cloud.blade.php new file mode 100644 index 0000000..8a6aa6e --- /dev/null +++ b/resources/views/livewire/confirm-restart-cloud.blade.php @@ -0,0 +1,17 @@ +
+
+ + + +
+

{{ __('cloud.restart_title') }}

+

{{ __('cloud.restart_body') }}

+
+
+
+ {{ __('cloud.restart_cancel') }} + + {{ __('cloud.restart_confirm') }} + +
+
diff --git a/routes/console.php b/routes/console.php index a32e74e..25f9b4c 100644 --- a/routes/console.php +++ b/routes/console.php @@ -121,3 +121,18 @@ Schedule::job(new PingHosts) Schedule::command('clupilot:apply-due-plan-changes') ->everyFifteenMinutes() ->withoutOverlapping(); + +// Keep the appointment a cancellation made. +// +// ConfirmCancelPackage writes a date and nothing ever went back to it, which is +// why TraefikWriter::remove() had no caller in the whole application: a +// cancelled instance kept its router, its certificate and its route to a guest +// address the host is free to reassign. Only instances whose paid term has +// actually run out are touched — a cancellation that is merely SCHEDULED is a +// paying customer, and this command is written around that distinction. +// +// Hourly: a term ends at a moment, and neither giving away a day of service nor +// hammering a DNS provider by the minute is a decision worth making by accident. +Schedule::command('clupilot:end-due-services') + ->hourly() + ->withoutOverlapping(); diff --git a/tests/Feature/Admin/RbacMoveTest.php b/tests/Feature/Admin/RbacMoveTest.php index 5b8d756..77f2c9b 100644 --- a/tests/Feature/Admin/RbacMoveTest.php +++ b/tests/Feature/Admin/RbacMoveTest.php @@ -12,8 +12,11 @@ use Spatie\Permission\Models\Role; it('moves every permission and role to the operator guard, leaving none behind', function () { expect(Permission::where('guard_name', 'web')->count())->toBe(0) ->and(Role::where('guard_name', 'web')->count())->toBe(0) - // 17 from the original seed, plus customers.grant_plan. - ->and(Permission::where('guard_name', 'operator')->count())->toBe(18) + // 17 from the original seed, plus customers.grant_plan and + // instances.restart — the latter created straight onto this guard, + // because since this migration `web` is where a permission goes to + // match nobody at all. + ->and(Permission::where('guard_name', 'operator')->count())->toBe(19) ->and(Role::where('guard_name', 'operator')->count())->toBe(6); }); @@ -154,7 +157,7 @@ it('preflights every customer conflict before mutating anything, listing all of // Nothing mutated: still exactly the pre-migration shape, not "moved and // then rolled back" — there is nothing here to roll back on a real // server, which is the whole reason this has to be checked up front. - expect(Permission::where('guard_name', 'web')->count())->toBe(18) + expect(Permission::where('guard_name', 'web')->count())->toBe(19) ->and(Permission::where('guard_name', 'operator')->count())->toBe(0) ->and(Role::where('guard_name', 'web')->count())->toBe(6) ->and(Role::where('guard_name', 'operator')->count())->toBe(0) diff --git a/tests/Feature/Console/ApplyStorageQuotasTest.php b/tests/Feature/Console/ApplyStorageQuotasTest.php new file mode 100644 index 0000000..e5f5d28 --- /dev/null +++ b/tests/Feature/Console/ApplyStorageQuotasTest.php @@ -0,0 +1,154 @@ +active()->create(['datacenter' => 'fsn', 'node' => 'pve']); + $order = Order::factory()->withSubscription()->create(['datacenter' => 'fsn', 'plan' => 'business']); + + return Instance::factory()->create(array_merge([ + 'order_id' => $order->id, + 'customer_id' => $order->customer_id, + 'host_id' => $host->id, + 'vmid' => 101, + 'status' => 'active', + 'quota_gb' => 500, + 'quota_applied_gb' => null, + ], $attributes)); +} + +/** Carry out every quota run the command started. */ +function runQuotaRuns(): void +{ + $runner = app(RunRunner::class); + + ProvisioningRun::query()->where('pipeline', 'quota')->get() + ->each(fn (ProvisioningRun $run) => $runner->advance($run)); +} + +it('applies the allowance to an instance that never had one, and records that it did', function () { + $services = fakeServices(); + Queue::fake(); + + $instance = unquotedInstance(); + + $this->artisan('clupilot:apply-quotas')->assertSuccessful(); + runQuotaRuns(); + + expect($services['pve']->guestRan("config:app:set files default_quota --value='500 GB'"))->toBeTrue() + ->and($instance->fresh()->quota_applied_gb)->toBe(500) + ->and($instance->fresh()->quotaIsEnforced())->toBeTrue(); +}); + +it('is a no-op the second time round', function () { + $services = fakeServices(); + Queue::fake(); + + unquotedInstance(); + + $this->artisan('clupilot:apply-quotas')->assertSuccessful(); + runQuotaRuns(); + + $callsAfterFirstSweep = count($services['pve']->guestCommands); + + $this->artisan('clupilot:apply-quotas') + ->expectsOutputToContain('bereits gesetzt') + ->assertSuccessful(); + runQuotaRuns(); + + expect(ProvisioningRun::query()->where('pipeline', 'quota')->count())->toBe(1) + ->and($services['pve']->guestCommands)->toHaveCount($callsAfterFirstSweep); +}); + +it('changes nothing at all in dry-run mode', function () { + $services = fakeServices(); + Queue::fake(); + + $instance = unquotedInstance(); + + $this->artisan('clupilot:apply-quotas', ['--dry-run' => true]) + ->expectsOutputToContain('Probelauf') + ->assertSuccessful(); + + expect(ProvisioningRun::query()->where('pipeline', 'quota')->count())->toBe(0) + ->and($services['pve']->guestCommands)->toBe([]) + ->and($instance->fresh()->quota_applied_gb)->toBeNull(); +}); + +it('skips what it has no business touching, and says which', function () { + fakeServices(); + Queue::fake(); + + // Still being built: its own run is writing this machine. + unquotedInstance(['status' => 'provisioning']); + // Nothing to reach at all. + unquotedInstance(['status' => 'failed', 'vmid' => null]); + // No allowance on the row. Writing "0 GB" would lock the customer out of + // their own files, which is worse than leaving what is there. + unquotedInstance(['quota_gb' => 0]); + // A service that has run out. + unquotedInstance(['status' => 'ended']); + + $busy = unquotedInstance(); + ProvisioningRun::factory()->create([ + 'subject_type' => Order::class, + 'subject_id' => $busy->order_id, + 'pipeline' => 'plan-change', + 'status' => ProvisioningRun::STATUS_RUNNING, + ]); + + $this->artisan('clupilot:apply-quotas') + ->expectsOutputToContain('keine erreichbare Maschine') + ->expectsOutputToContain('kein Kontingent hinterlegt') + ->expectsOutputToContain('anderer Lauf aktiv') + ->assertSuccessful(); + + expect(ProvisioningRun::query()->where('pipeline', 'quota')->count())->toBe(0); +}); + +it('re-applies an allowance that has changed since it was last enforced', function () { + $services = fakeServices(); + Queue::fake(); + + // A downgrade landed on the row but the run that should have followed it + // never finished — the customer is paying for less and still holds more. + $instance = unquotedInstance(['quota_gb' => 200, 'quota_applied_gb' => 500]); + + $this->artisan('clupilot:apply-quotas')->assertSuccessful(); + runQuotaRuns(); + + expect($services['pve']->guestRan("config:app:set files default_quota --value='200 GB'"))->toBeTrue() + ->and($instance->fresh()->quota_applied_gb)->toBe(200); +}); + +it('repairs one named instance when asked to', function () { + fakeServices(); + Queue::fake(); + + $wanted = unquotedInstance(); + unquotedInstance(); + + $this->artisan('clupilot:apply-quotas', ['--instance' => $wanted->uuid])->assertSuccessful(); + + expect(ProvisioningRun::query()->where('pipeline', 'quota')->count())->toBe(1) + ->and(ProvisioningRun::query()->where('pipeline', 'quota')->first()->context('instance_id')) + ->toBe($wanted->id); +}); diff --git a/tests/Feature/Provisioning/EndInstanceServiceTest.php b/tests/Feature/Provisioning/EndInstanceServiceTest.php new file mode 100644 index 0000000..4b54289 --- /dev/null +++ b/tests/Feature/Provisioning/EndInstanceServiceTest.php @@ -0,0 +1,198 @@ +active()->create(['datacenter' => 'fsn', 'node' => 'pve', 'public_ip' => '203.0.113.9']); + $order = Order::factory()->withSubscription()->create(['datacenter' => 'fsn', 'plan' => 'business']); + + $instance = Instance::factory()->create(array_merge([ + 'order_id' => $order->id, + 'customer_id' => $order->customer_id, + 'host_id' => $host->id, + 'vmid' => 101, + 'guest_ip' => '10.20.0.7', + 'subdomain' => $subdomain, + 'status' => 'active', + 'route_written' => true, + 'routed_hostnames' => [$subdomain.'.'.ProvisioningSettings::dnsZone()], + 'cert_ok' => true, + ], $attributes)); + + $recordId = $services['dns']->upsertRecord($subdomain.'.'.ProvisioningSettings::dnsZone(), 'A', $host->public_ip); + $instance->dnsRecords()->create([ + 'provider' => 'hetzner', + 'record_id' => $recordId, + 'fqdn' => $subdomain.'.'.ProvisioningSettings::dnsZone(), + 'type' => 'A', + 'value' => $host->public_ip, + ]); + + $services['traefik']->write($host->wg_ip ?? $host->public_ip, $subdomain, $instance->routed_hostnames, $instance->guest_ip); + + return compact('services', 'host', 'order', 'instance'); +} + +it('takes the router and the DNS record down when the paid term has run out', function () { + ['services' => $services, 'instance' => $instance] = routedInstance([ + 'status' => 'cancellation_scheduled', + 'cancel_requested_at' => now()->subMonth(), + 'service_ends_at' => now()->subHour(), + ]); + + expect($services['traefik']->routes)->toHaveKey('berger'); + + $this->artisan('clupilot:end-due-services')->assertSuccessful(); + + expect($services['traefik']->routes)->not->toHaveKey('berger') + // The A record goes with the route: it is in OUR zone, it points at a + // host that goes on serving other customers, and a name we own that + // resolves to a live machine with nothing of its owner's behind it is + // the shape of every subdomain takeover there has ever been. + ->and($services['dns']->records)->toBe([]) + ->and($instance->dnsRecords()->count())->toBe(0) + ->and($instance->fresh()->status)->toBe('ended') + // The address state goes too, so a revived instance is not announced + // and routed nowhere by a step that thinks it has already written. + ->and($instance->fresh()->route_written)->toBeFalse() + ->and($instance->fresh()->routed_hostnames)->toBeNull() + ->and($instance->fresh()->cert_ok)->toBeFalse(); +}); + +it('leaves a scheduled cancellation exactly where it is until the day arrives', function () { + ['services' => $services, 'instance' => $instance] = routedInstance([ + 'status' => 'cancellation_scheduled', + 'cancel_requested_at' => now(), + 'service_ends_at' => now()->addWeeks(3), + ]); + + $this->artisan('clupilot:end-due-services')->assertSuccessful(); + + // They have paid to the end of the term and are working in it this + // afternoon. Pulling the address the day somebody schedules a cancellation + // would be a far worse fault than the leak this closes. + expect($services['traefik']->routes)->toHaveKey('berger') + ->and($services['dns']->records)->not->toBe([]) + ->and($instance->fresh()->status)->toBe('cancellation_scheduled') + ->and($instance->fresh()->route_written)->toBeTrue(); +}); + +it('does not touch an active instance, however old', function () { + ['services' => $services, 'instance' => $instance] = routedInstance([ + 'created_at' => now()->subYears(2), + ]); + + $this->artisan('clupilot:end-due-services')->assertSuccessful(); + + expect($services['traefik']->routes)->toHaveKey('berger') + ->and($instance->fresh()->status)->toBe('active'); +}); + +it('leaves a cancellation with no end date alone rather than inventing one', function () { + ['services' => $services, 'instance' => $instance] = routedInstance([ + 'status' => 'cancellation_scheduled', + 'cancel_requested_at' => now()->subMonth(), + 'service_ends_at' => null, + ]); + + $this->artisan('clupilot:end-due-services')->assertSuccessful(); + + // A broken record, not an expired one. Guessing an end date for it would + // invent the very moment this whole thing is written to respect. + expect($services['traefik']->routes)->toHaveKey('berger') + ->and($instance->fresh()->status)->toBe('cancellation_scheduled'); +}); + +it('changes nothing in dry-run mode', function () { + ['services' => $services, 'instance' => $instance] = routedInstance([ + 'status' => 'cancellation_scheduled', + 'service_ends_at' => now()->subDay(), + ]); + + $this->artisan('clupilot:end-due-services', ['--dry-run' => true]) + ->expectsOutputToContain('Probelauf') + ->assertSuccessful(); + + expect($services['traefik']->routes)->toHaveKey('berger') + ->and($services['dns']->records)->not->toBe([]) + ->and($instance->fresh()->status)->toBe('cancellation_scheduled'); +}); + +it('is safe to run twice', function () { + ['services' => $services, 'instance' => $instance] = routedInstance([ + 'status' => 'cancellation_scheduled', + 'service_ends_at' => now()->subDay(), + ]); + + $this->artisan('clupilot:end-due-services')->assertSuccessful(); + $this->artisan('clupilot:end-due-services')->assertSuccessful(); + + expect($instance->fresh()->status)->toBe('ended') + ->and($services['traefik']->routes)->toBe([]); +}); + +it('carries the whole way from the customer cancelling to the address going away', function () { + ['services' => $services, 'instance' => $instance] = routedInstance(); + + $user = User::factory()->create(['email_verified_at' => now()]); + Customer::query()->whereKey($instance->customer_id)->update(['user_id' => $user->id, 'email' => $user->email]); + + Livewire::actingAs($user)->test(ConfirmCancelPackage::class) + ->set('confirmName', $instance->subdomain) + ->call('cancelPackage'); + + $instance->refresh(); + + expect($instance->status)->toBe('cancellation_scheduled') + ->and($instance->service_ends_at)->not->toBeNull(); + + // Still theirs, still served, for the whole of the term they bought. + $this->artisan('clupilot:end-due-services')->assertSuccessful(); + expect($services['traefik']->routes)->toHaveKey('berger'); + + // And gone the moment it is genuinely over. + $this->travelTo($instance->service_ends_at->copy()->addMinute()); + $this->artisan('clupilot:end-due-services')->assertSuccessful(); + + expect($services['traefik']->routes)->toBe([]) + ->and($instance->fresh()->status)->toBe('ended'); +}); + +it('refuses to end an instance whose term has not run out, whoever asks', function () { + ['instance' => $instance] = routedInstance([ + 'status' => 'cancellation_scheduled', + 'service_ends_at' => now()->addMonth(), + ]); + + // The rule lives on the action, not in the command's query — anything that + // ever ends an instance has to ask the same question. + expect(app(EndInstanceService::class)->hasEnded($instance))->toBeFalse() + ->and(app(EndInstanceService::class)($instance))->toBeFalse() + ->and($instance->fresh()->status)->toBe('cancellation_scheduled'); +}); diff --git a/tests/Feature/Provisioning/InstanceRestartTest.php b/tests/Feature/Provisioning/InstanceRestartTest.php new file mode 100644 index 0000000..656673f --- /dev/null +++ b/tests/Feature/Provisioning/InstanceRestartTest.php @@ -0,0 +1,330 @@ +active()->create(['datacenter' => 'fsn', 'node' => 'pve']); + $order = Order::factory()->withSubscription()->create(['datacenter' => 'fsn', 'plan' => 'business']); + + $instance = Instance::factory()->create(array_merge([ + 'order_id' => $order->id, + 'customer_id' => $order->customer_id, + 'host_id' => $host->id, + 'vmid' => 101, + 'guest_ip' => '10.20.0.7', + 'status' => 'active', + // What the contract now says. The guest below is running on less. + 'cores' => 8, + 'ram_mb' => 16384, + 'restart_required_since' => now()->subDays(3), + ], $instanceAttributes)); + + // The machine as it actually is: booted small, then given a bigger + // definition it has not read yet. + $pve->setCloudInit('pve', 101, ['cores' => 2, 'memory' => 4096]); + $pve->startVm('pve', 101); + $pve->setCloudInit('pve', 101, ['cores' => 8, 'memory' => 16384]); + + return compact('services', 'pve', 'host', 'order', 'instance'); +} + +/** + * Advance a run until it settles, clearing the backoff between steps. + * + * next_attempt_at is what makes a poll a wait; a test that did not clear it + * would simply watch the runner decline to do anything. + */ +function driveRun(ProvisioningRun $run, int $limit = 25): ProvisioningRun +{ + $runner = app(RunRunner::class); + + for ($i = 0; $i < $limit; $i++) { + $run->refresh(); + + if (in_array($run->status, [ProvisioningRun::STATUS_COMPLETED, ProvisioningRun::STATUS_FAILED], true)) { + break; + } + + $run->forceFill(['next_attempt_at' => null])->save(); + $runner->advance($run); + } + + return $run->refresh(); +} + +it('asks the guest to shut down, brings it back, and only then stops saying a restart is due', function () { + Queue::fake(); + ['pve' => $pve, 'instance' => $instance] = restartableInstance(); + $this->actingAs(admin(), 'operator'); + + // Before: the definition is big, the running machine is not — which is what + // "Neustart erforderlich" has been telling the customer for three days. + expect($pve->vmStatus('pve', 101)['cpus'])->toBe(2) + ->and($instance->restartIsPending())->toBeTrue(); + + $run = app(RestartInstance::class)($instance); + expect($run)->not->toBeNull(); + + // Pressing the button does not clear it. Nothing has happened to the + // machine yet, and a flag that clears on intent is a flag that lies. + expect($instance->fresh()->restartIsPending())->toBeTrue(); + + driveRun($run); + + expect($run->fresh()->status)->toBe(ProvisioningRun::STATUS_COMPLETED) + // The guest was ASKED, and it was asked exactly once. + ->and($pve->shutdownCalls)->toHaveCount(1) + ->and($pve->shutdownCalls[0]['vmid'])->toBe(101) + // It came back — and came back on the definition it had been given, + // which is the only thing that makes the upgrade real. + ->and($pve->vmStatus('pve', 101)['status'])->toBe('running') + ->and($pve->vmStatus('pve', 101)['cpus'])->toBe(8) + ->and((int) ($pve->vmStatus('pve', 101)['maxmem'] / 1048576))->toBe(16384) + ->and($instance->fresh()->restartIsPending())->toBeFalse(); +}); + +it('never pulls the plug on a guest that will not shut down', function () { + Queue::fake(); + ['pve' => $pve, 'instance' => $instance, 'order' => $order] = restartableInstance(); + $order->update(['status' => 'active']); + $this->actingAs(admin(), 'operator'); + + // A guest that ignores ACPI — a wedged handler, or a machine busy with + // something it will not be interrupted in. + $pve->shutdownIgnored = true; + + $run = app(RestartInstance::class)($instance); + + driveRun($run); + // Past the step's grace period but inside the runner's own step timeout, so + // the step's deliberate failure fires rather than a generic retry. + $this->travel(11)->minutes(); + $run = driveRun($run); + + expect($run->status)->toBe(ProvisioningRun::STATUS_FAILED) + ->and($run->error)->toBe('shutdown_timeout') + // The machine is still running. Nextcloud is a database and this + // product has no power cut to escalate to — the run fails and a person + // decides what happens next. + ->and($pve->runningVmids)->toContain(101) + ->and($pve->vmStatus('pve', 101)['status'])->toBe('running') + // And nothing has been declared finished on the strength of a failure. + ->and($instance->fresh()->restartIsPending())->toBeTrue() + // A maintenance pipeline failing must not release a paid, running + // customer — RunRunner's subject hook only fires for the pipeline that + // BUILDS the subject, and a restart is not that one. + ->and($order->fresh()->status)->toBe('active') + ->and($instance->fresh()->status)->toBe('active'); +}); + +it('refuses to call a restart done when the machine came back smaller than it was sold', function () { + Queue::fake(); + ['pve' => $pve, 'instance' => $instance, 'order' => $order] = restartableInstance(); + + // The machine went round but the bigger definition never took — a config + // write that silently failed, a host that could not honour it. + $pve->shutdownVm('pve', 101, 600); + $pve->vmConfig[101] = ['cores' => 2, 'memory' => 4096]; + $pve->startVm('pve', 101); + + $run = ProvisioningRun::factory()->create([ + 'subject_type' => Order::class, 'subject_id' => $order->id, 'pipeline' => 'restart', + 'context' => ['instance_id' => $instance->id, 'node' => 'pve', 'vmid' => 101], + ]); + + $result = app(CompleteRestart::class)->execute($run); + + expect($result->type)->toBe('fail') + ->and($result->reason)->toContain('2 statt 8 vCPU') + ->and($instance->fresh()->restartIsPending())->toBeTrue(); +}); + +it('leaves the pending restart alone until the very last step of the run', function () { + Queue::fake(); + ['instance' => $instance] = restartableInstance(); + $this->actingAs(admin(), 'operator'); + + $run = app(RestartInstance::class)($instance); + $runner = app(RunRunner::class); + + // Shutdown asked, shutdown observed, machine started, agent answered — and + // through all of it the flag stands, because none of those is the machine + // running on what was bought. + for ($i = 0; $i < 6; $i++) { + $run->refresh(); + + if ($run->current_step >= 3) { + break; + } + + $run->forceFill(['next_attempt_at' => null])->save(); + $runner->advance($run); + + expect($instance->fresh()->restartIsPending())->toBeTrue(); + } + + expect($run->refresh()->current_step)->toBe(3) + ->and($instance->fresh()->restartIsPending())->toBeTrue(); + + driveRun($run); + + expect($instance->fresh()->restartIsPending())->toBeFalse(); +}); + +it('lets a customer restart their own machine and nobody else’s', function () { + Queue::fake(); + ['instance' => $mine] = restartableInstance(); + ['instance' => $theirs] = restartableInstance(); + + $user = User::factory()->create(['email_verified_at' => now()]); + $mine->customer->update(['user_id' => $user->id, 'email' => $user->email]); + + $this->actingAs($user); + + // Checked in the action, not in the view. A Livewire method is reachable by + // anyone who can POST to /livewire/update, so hiding a button proves + // nothing at all about who may press it. + expect(fn () => app(RestartInstance::class)($theirs))->toThrow(AuthorizationException::class); + expect(app(RestartInstance::class)->allows($theirs))->toBeFalse() + ->and(app(RestartInstance::class)->allows($mine))->toBeTrue(); + + expect(app(RestartInstance::class)($mine))->not->toBeNull(); +}); + +it('lets an operator with the capability restart any machine, and refuses one without it', function () { + Queue::fake(); + ['instance' => $instance] = restartableInstance(); + + $this->actingAs(operator('Read-only'), 'operator'); + expect(fn () => app(RestartInstance::class)($instance))->toThrow(AuthorizationException::class); + expect(ProvisioningRun::query()->where('pipeline', 'restart')->count())->toBe(0); + + $this->actingAs(admin(), 'operator'); + expect(app(RestartInstance::class)($instance))->not->toBeNull() + ->and(ProvisioningRun::query()->where('pipeline', 'restart')->count())->toBe(1); +}); + +it('does not start a restart while another run is already in flight for that instance', function () { + Queue::fake(); + ['instance' => $instance, 'order' => $order] = restartableInstance(); + + // A plan change still working on the same machine. Stopping the guest + // underneath it would race the config it is about to boot. + ProvisioningRun::factory()->create([ + 'subject_type' => Order::class, + 'subject_id' => $order->id, + 'pipeline' => 'plan-change', + 'status' => ProvisioningRun::STATUS_RUNNING, + ]); + + $this->actingAs(admin(), 'operator'); + + expect(app(RestartInstance::class)($instance))->toBeNull() + ->and(ProvisioningRun::query()->where('pipeline', 'restart')->count())->toBe(0); +}); + +it('refuses a machine there is nothing to restart', function () { + Queue::fake(); + ['instance' => $instance] = restartableInstance(['status' => 'provisioning']); + + $this->actingAs(admin(), 'operator'); + + // Still being built: its own run is writing this machine, and a shutdown + // beside it would stop the guest that run is talking to. + expect(app(RestartInstance::class)($instance))->toBeNull(); +}); + +it('confirms in a modal on both sides and drives the real action from the page', function () { + Queue::fake(); + ['instance' => $instance] = restartableInstance(); + + $user = User::factory()->create(['email_verified_at' => now()]); + $instance->customer->update(['user_id' => $user->id, 'email' => $user->email]); + + // Customer: the modal only says yes; Cloud::restart() does the work. + Livewire::actingAs($user)->test(ConfirmRestartCloud::class) + ->call('proceed') + ->assertDispatched('cloud-restart-confirmed'); + + Livewire::actingAs($user)->test(Cloud::class) + ->dispatch('cloud-restart-confirmed') + ->assertDispatched('notify'); + + expect(ProvisioningRun::query()->where('pipeline', 'restart')->count())->toBe(1); + + // Operator: same shape, addressed by uuid (R11). + ProvisioningRun::query()->delete(); + + Livewire::actingAs(admin(), 'operator')->test(ConfirmRestartInstance::class, ['uuid' => $instance->uuid]) + ->call('proceed') + ->assertDispatched('instance-restart-confirmed'); + + Livewire::actingAs(admin(), 'operator')->test(AdminInstances::class) + ->dispatch('instance-restart-confirmed', uuid: $instance->uuid); + + expect(ProvisioningRun::query()->where('pipeline', 'restart')->count())->toBe(1); +}); + +it('refuses the console action to an operator who does not hold the capability', function () { + Queue::fake(); + ['instance' => $instance] = restartableInstance(); + + Livewire::actingAs(operator('Read-only'), 'operator')->test(AdminInstances::class) + ->dispatch('instance-restart-confirmed', uuid: $instance->uuid) + ->assertForbidden(); + + expect(ProvisioningRun::query()->where('pipeline', 'restart')->count())->toBe(0); +}); + +it('gives the shutdown step nothing to do when the guest is already stopped', function () { + Queue::fake(); + ['pve' => $pve, 'instance' => $instance, 'order' => $order] = restartableInstance(); + + $pve->runningVmids = []; + + $run = ProvisioningRun::factory()->create([ + 'subject_type' => Order::class, 'subject_id' => $order->id, 'pipeline' => 'restart', + 'context' => ['instance_id' => $instance->id, 'node' => 'pve', 'vmid' => 101], + ]); + + expect(app(ShutDownVirtualMachine::class)->execute($run)->type)->toBe('advance') + ->and($pve->shutdownCalls)->toBe([]); +});