diff --git a/app/Actions/EndInstanceService.php b/app/Actions/EndInstanceService.php new file mode 100644 index 0000000..2f687c9 --- /dev/null +++ b/app/Actions/EndInstanceService.php @@ -0,0 +1,135 @@ +status === 'cancellation_scheduled' + && $instance->service_ends_at !== null + && $instance->service_ends_at->lessThanOrEqualTo($at ?? now()); + } + + /** + * Tear the address down and mark the instance ended. + * + * Re-entrant: removing a router file that is not there is `rm -f`, a record + * id that has already been deleted is deleted again for nothing, and an + * instance already marked `ended` is returned false straight away. A DNS + * failure is deliberately not swallowed — the record id survives on its row, + * so a retry can finish the job, and a provider that is permanently broken + * becomes a visible failure rather than a silent leak. + * + * @return bool whether this call was the one that ended it + */ + public function __invoke(Instance $instance): bool + { + if (! $this->hasEnded($instance)) { + return false; + } + + $host = $instance->host; + + if ($host !== null) { + // The host that actually serves the traffic — DNS points at it, and + // it is where ConfigureDnsAndTls wrote the file in the first place. + $this->traefik->remove($host->wg_ip ?? $host->public_ip, $instance->subdomain); + } else { + // No host left to reach: the router file went with the machine it + // lived on. Said out loud rather than passed over, because it means + // this teardown is only doing half of what it usually does. + Log::warning('Ending an instance whose host is gone — no router to remove.', [ + 'instance' => $instance->uuid, + ]); + } + + foreach ($instance->dnsRecords()->get() as $record) { + $this->dns->deleteRecord($record->record_id); + $record->delete(); + } + + $instance->update([ + 'status' => 'ended', + // The address state has to go with the address. Left as it was, a + // later run would read `route_written` plus a matching hostname list + // and decide there was nothing to write — so an instance that was + // ever revived would be announced and routed nowhere. + 'route_written' => false, + 'routed_hostnames' => null, + 'cert_ok' => false, + 'domain_cert_ok' => false, + ]); + + Log::info('Instance service ended; address withdrawn.', [ + 'instance' => $instance->uuid, + 'subdomain' => $instance->subdomain, + 'service_ended_at' => $instance->service_ends_at?->toIso8601String(), + ]); + + return true; + } +} diff --git a/app/Actions/RestartInstance.php b/app/Actions/RestartInstance.php new file mode 100644 index 0000000..6e70a1b --- /dev/null +++ b/app/Actions/RestartInstance.php @@ -0,0 +1,163 @@ +user(); + + if ($operator !== null && $operator->isActive() && $operator->can('instances.restart')) { + return true; + } + + $customer = Customer::forUser(Auth::guard('web')->user()); + + return $customer !== null && (int) $instance->customer_id === (int) $customer->id; + } + + /** + * Start a restart run for this instance, or return null when there is + * nothing to start. + * + * Null is the ordinary "no" and not a failure — a machine that is still + * being built, one whose VM has gone, one whose service has ended, or one + * that already has a run in flight all have a better reason to be left + * alone than a restart has to interrupt them. A refusal on AUTHORISATION is + * the one case that throws, because it is not an ordinary outcome and the + * caller must not be able to mistake it for one. + * + * @throws AuthorizationException + */ + public function __invoke(Instance $instance): ?ProvisioningRun + { + if (! $this->allows($instance)) { + throw new AuthorizationException; + } + + if (! $instance->hasLiveMachine()) { + return null; + } + + // The same lock ReapplyInstanceAddress takes, for the same reason: two + // requests can reach this in one instant — the customer pressing the + // button while an operator presses it for them is the obvious pair — + // and two restart runs against one machine would have the second one + // shutting the guest down while the first waits for it to come back. + // Nobody waits for the lock: a restart that lost the race has nothing + // to add, because the run that won does exactly the same thing. + $lock = Cache::lock('instance-restart:'.$instance->uuid, 30); + + if (! $lock->get()) { + return null; + } + + try { + return $this->start($instance); + } finally { + $lock->release(); + } + } + + private function start(Instance $instance): ?ProvisioningRun + { + $order = $instance->order; + + // Checked against the ORDER, because that is the subject every customer + // pipeline shares. A restart beside an unfinished build would stop the + // machine the build is still writing to; a restart beside a plan change + // would race the config PUT it is about to boot. Whatever is in flight + // gets to finish, and the customer can press the button again after. + if ($this->hasRunInFlight($order)) { + return null; + } + + $run = ProvisioningRun::create([ + 'subject_type' => Order::class, + 'subject_id' => $order->id, + 'pipeline' => 'restart', + 'status' => ProvisioningRun::STATUS_PENDING, + 'current_step' => 0, + // Everything the four steps read: instance_id is how CustomerStep + // finds the machine, node and vmid are how it reaches it. + 'context' => [ + 'instance_id' => $instance->id, + 'host_id' => $instance->host_id, + 'node' => $instance->host?->node, + 'vmid' => $instance->vmid, + 'subdomain' => $instance->subdomain, + ], + ]); + + AdvanceRunJob::dispatch($run->uuid); + + Log::info('Restarting a customer instance.', [ + 'instance' => $instance->uuid, + 'restart_pending_since' => $instance->restart_required_since?->toIso8601String(), + 'run' => $run->uuid, + ]); + + return $run; + } + + private function hasRunInFlight(Order $order): bool + { + return ProvisioningRun::query() + ->where('subject_type', Order::class) + ->where('subject_id', $order->id) + ->whereIn('status', [ + ProvisioningRun::STATUS_PENDING, + ProvisioningRun::STATUS_RUNNING, + ProvisioningRun::STATUS_WAITING, + ProvisioningRun::STATUS_PAUSED, + ]) + ->exists(); + } +} diff --git a/app/Console/Commands/ApplyStorageQuotas.php b/app/Console/Commands/ApplyStorageQuotas.php new file mode 100644 index 0000000..9d98a05 --- /dev/null +++ b/app/Console/Commands/ApplyStorageQuotas.php @@ -0,0 +1,167 @@ +option('dry-run'); + $started = 0; + + /** @var array reason => count */ + $skipped = []; + + $query = Instance::query()->with(['order', 'host']); + + if ($uuid = $this->option('instance')) { + $query->where('uuid', $uuid); + } + + foreach ($query->cursor() as $instance) { + $reason = $this->reasonToSkip($instance); + + if ($reason !== null) { + $skipped[$reason] = ($skipped[$reason] ?? 0) + 1; + + continue; + } + + $started++; + + $this->line(($dryRun ? '[dry-run] ' : '') + ."{$instance->subdomain}: {$instance->quota_gb} GB " + .'('.($instance->quota_applied_gb === null ? 'noch nie gesetzt' : "zuletzt {$instance->quota_applied_gb} GB").')'); + + if (! $dryRun) { + $this->startQuotaRun($instance); + } + } + + foreach ($skipped as $reason => $count) { + $this->line("übersprungen ({$reason}): {$count}"); + } + + $this->info($dryRun + ? "Probelauf: {$started} Instanz(en) bekämen ihr Kontingent, ".array_sum($skipped).' übersprungen. Nichts wurde geändert.' + : "{$started} Kontingent-Lauf/Läufe gestartet, ".array_sum($skipped).' übersprungen.'); + + return self::SUCCESS; + } + + /** + * Why this instance is being left alone, or null when it is not. + * + * The reason is the report — a sweep that says "skipped 34" and nothing else + * is a sweep nobody can act on, because "34 machines have no allowance + * enforced" and "34 machines are already fine" look identical from here. + */ + private function reasonToSkip(Instance $instance): ?string + { + // Nothing remote work can reach: a reservation with no VM, a failed + // build, an ended service. The same rule every other maintenance action + // asks — see Instance::hasLiveMachine(). + if (! $instance->hasLiveMachine()) { + return 'keine erreichbare Maschine'; + } + + // No allowance on the row at all. Writing "0 GB" would lock the customer + // out of their own files, which is a far worse answer than leaving what + // is there and letting somebody notice — the same decision the step + // itself makes, made here too so the machine is not visited for nothing. + if ((int) $instance->quota_gb <= 0) { + return 'kein Kontingent hinterlegt'; + } + + if ($instance->quotaIsEnforced()) { + return 'bereits gesetzt'; + } + + // Whatever is in flight against this order gets to finish. A quota run + // beside an unfinished build would race the same occ calls, and the + // build applies the quota itself anyway. + if ($this->hasRunInFlight($instance)) { + return 'anderer Lauf aktiv'; + } + + return null; + } + + private function startQuotaRun(Instance $instance): void + { + $run = ProvisioningRun::create([ + // The instance's own purchase order, as every customer pipeline + // uses — that is what CustomerStep::order() and ::instance() + // resolve from, and what the in-flight check above looks at. + 'subject_type' => Order::class, + 'subject_id' => $instance->order_id, + 'pipeline' => 'quota', + 'status' => ProvisioningRun::STATUS_PENDING, + 'current_step' => 0, + 'context' => [ + 'instance_id' => $instance->id, + 'host_id' => $instance->host_id, + 'node' => $instance->host?->node, + 'vmid' => $instance->vmid, + 'subdomain' => $instance->subdomain, + ], + ]); + + AdvanceRunJob::dispatch($run->uuid); + } + + private function hasRunInFlight(Instance $instance): bool + { + return ProvisioningRun::query() + ->where('subject_type', Order::class) + ->where('subject_id', $instance->order_id) + ->whereIn('status', [ + ProvisioningRun::STATUS_PENDING, + ProvisioningRun::STATUS_RUNNING, + ProvisioningRun::STATUS_WAITING, + ProvisioningRun::STATUS_PAUSED, + ]) + ->exists(); + } +} diff --git a/app/Console/Commands/EndDueServices.php b/app/Console/Commands/EndDueServices.php new file mode 100644 index 0000000..c4bb27a --- /dev/null +++ b/app/Console/Commands/EndDueServices.php @@ -0,0 +1,74 @@ +option('dry-run'); + $ended = 0; + + // Narrowed in the query for the sake of the database, decided by + // hasEnded() for the sake of the rule: the query is an index, not a + // second opinion. + $due = Instance::query() + ->with(['host', 'dnsRecords']) + ->where('status', 'cancellation_scheduled') + ->whereNotNull('service_ends_at') + ->where('service_ends_at', '<=', now()) + ->cursor(); + + foreach ($due as $instance) { + if (! $end->hasEnded($instance)) { + continue; + } + + $this->line(($dryRun ? '[dry-run] ' : '') + ."{$instance->subdomain}: Laufzeit endete am " + .$instance->service_ends_at->local()->isoFormat('LLL')); + + if ($dryRun || $end($instance)) { + $ended++; + } + } + + $this->info($dryRun + ? "Probelauf: {$ended} Dienst(e) würden beendet. Nichts wurde geändert." + : "{$ended} Dienst(e) beendet."); + + return self::SUCCESS; + } +} diff --git a/app/Livewire/Admin/ConfirmRestartInstance.php b/app/Livewire/Admin/ConfirmRestartInstance.php new file mode 100644 index 0000000..8ffb181 --- /dev/null +++ b/app/Livewire/Admin/ConfirmRestartInstance.php @@ -0,0 +1,48 @@ +where('uuid', $uuid)->firstOrFail(); + + $this->uuid = $uuid; + $this->address = $instance->domainIsVerified() + ? (string) $instance->custom_domain + : (string) $instance->subdomain; + } + + public function proceed(): void + { + $this->dispatch('instance-restart-confirmed', uuid: $this->uuid); + $this->closeModal(); + } + + public function render() + { + return view('livewire.admin.confirm-restart-instance'); + } +} diff --git a/app/Livewire/Admin/Instances.php b/app/Livewire/Admin/Instances.php index 708475b..5346762 100644 --- a/app/Livewire/Admin/Instances.php +++ b/app/Livewire/Admin/Instances.php @@ -2,9 +2,13 @@ namespace App\Livewire\Admin; +use App\Actions\RestartInstance; use App\Models\Instance; +use Illuminate\Auth\Access\AuthorizationException; +use Illuminate\Support\Facades\Gate; use Illuminate\Support\Facades\Lang; use Livewire\Attributes\Layout; +use Livewire\Attributes\On; use Livewire\Component; use Livewire\WithPagination; @@ -22,6 +26,42 @@ class Instances extends Component { use WithPagination; + /** + * Restart one instance, once ConfirmRestartInstance has said so. + * + * The capability is checked twice on purpose, and neither check is the + * decoration of the other. This one keeps an operator without + * `instances.restart` from driving the console component at all; the one + * inside RestartInstance is what actually guards the machine, because that + * is the door a hand-written POST to /livewire/update arrives at. Losing + * either would still leave a customer's cloud reachable by somebody who was + * only ever meant to read the list. + */ + #[On('instance-restart-confirmed')] + public function restart(string $uuid): void + { + $this->authorize('instances.restart'); + + $instance = Instance::query()->where('uuid', $uuid)->first(); + + if ($instance === null) { + return; + } + + try { + $run = app(RestartInstance::class)($instance); + } catch (AuthorizationException) { + $this->dispatch('notify', message: __('admin.restart_denied')); + + return; + } + + // Null is every ordinary refusal — no live machine, or a run already in + // flight against this order. Said out loud, because an operator who + // pressed a button and saw nothing will press it again. + $this->dispatch('notify', message: __($run === null ? 'admin.restart_busy' : 'admin.restart_started')); + } + public function render() { $instances = Instance::query() @@ -31,7 +71,17 @@ class Instances extends Component return view('livewire.admin.instances', [ 'instances' => $instances, + // Whether to draw the action column at all. An operator who may not + // restart anything should not be reading a column of buttons that + // answer them 403. + 'canRestart' => Gate::allows('instances.restart'), 'rows' => $instances->getCollection()->map(fn (Instance $i) => [ + // R11: a row's action is addressed by uuid, never the numeric id. + 'uuid' => $i->uuid, + // Only a machine remote work can actually reach. A reservation + // with no VM, a failed build and an ended service all have + // nothing to restart — see Instance::hasLiveMachine(). + 'live' => $i->hasLiveMachine(), // The operator list shows what is actually served — an unverified // custom domain is a plan, not an address. 'address' => $i->domainIsVerified() ? $i->custom_domain : $i->subdomain, diff --git a/app/Livewire/Cloud.php b/app/Livewire/Cloud.php index 1c1260a..6ac471d 100644 --- a/app/Livewire/Cloud.php +++ b/app/Livewire/Cloud.php @@ -2,11 +2,15 @@ namespace App\Livewire; +use App\Actions\RestartInstance; use App\Livewire\Concerns\ResolvesCustomer; +use App\Models\Instance; use App\Models\MaintenanceWindow; use App\Support\ProvisioningSettings; +use Illuminate\Auth\Access\AuthorizationException; use Illuminate\Support\Number; use Livewire\Attributes\Layout; +use Livewire\Attributes\On; use Livewire\Component; #[Layout('layouts.portal-app')] @@ -14,6 +18,56 @@ class Cloud extends Component { use ResolvesCustomer; + /** + * Restart the customer's own cloud, once ConfirmRestartCloud has said so. + * + * The modal raises the event and this method does the work, so the rule + * about who may restart what stays in one place (R23). That place is + * RestartInstance itself, not this method: the instance below is resolved + * from the signed-in customer, so it is already theirs, but the action + * re-checks anyway — a Livewire method is reachable by anyone who can POST + * to /livewire/update, and an authorisation that lives only in how a caller + * happened to look something up is one refactor away from being gone. + */ + #[On('cloud-restart-confirmed')] + public function restart(): void + { + $instance = $this->instance(); + + if ($instance === null) { + $this->dispatch('notify', message: __('dashboard.no_customer_action')); + + return; + } + + try { + $run = app(RestartInstance::class)($instance); + } catch (AuthorizationException) { + $this->dispatch('notify', message: __('cloud.restart_denied')); + + return; + } + + // Null is every ordinary "not now": a machine still being built, one + // whose service has ended, or one with a run already in flight. The + // customer gets told to try again rather than being left with a button + // that appears to have done nothing. + $this->dispatch('notify', message: __($run === null ? 'cloud.restart_busy' : 'cloud.restart_started')); + } + + /** + * The customer's service instance — the one the whole page is about. + * + * Resolved here rather than inside render() so the restart action and the + * card cannot end up talking about two different machines. + */ + private function instance(): ?Instance + { + return $this->customer()?->instances() + ->whereIn('status', ['active', 'provisioning', 'cancellation_scheduled']) + ->latest('id')->first(); + } + public function render() { $locale = app()->getLocale(); @@ -22,9 +76,7 @@ class Cloud extends Component // The card is built from the customer's real service instance, so the // maintenance badge below can be scoped to exactly that instance's host. $customer = $this->customer(); - $shown = $customer?->instances() - ->whereIn('status', ['active', 'provisioning', 'cancellation_scheduled']) - ->latest('id')->first(); + $shown = $this->instance(); $maintenance = MaintenanceWindow::forInstance($shown)->first(); // The instance carries what was actually provisioned, and the contract @@ -101,6 +153,11 @@ class Cloud extends Component // purchase, so the customer is told instead — otherwise they are // paying for cores they cannot see and have no way of finding out. 'restartPending' => $shown?->restartIsPending() ?? false, + // Whether there is a machine a restart could reach at all. The + // button is drawn either way — hiding it would leave a customer + // wondering where it went during a build — but it is only live when + // pressing it can do something. + 'canRestart' => $shown?->hasLiveMachine() ?? false, ]); } } diff --git a/app/Livewire/ConfirmRestartCloud.php b/app/Livewire/ConfirmRestartCloud.php new file mode 100644 index 0000000..2e1ecca --- /dev/null +++ b/app/Livewire/ConfirmRestartCloud.php @@ -0,0 +1,29 @@ +dispatch('cloud-restart-confirmed'); + $this->closeModal(); + } + + public function render() + { + return view('livewire.confirm-restart-cloud'); + } +} diff --git a/app/Models/Instance.php b/app/Models/Instance.php index 733ae2a..b7def82 100644 --- a/app/Models/Instance.php +++ b/app/Models/Instance.php @@ -17,7 +17,7 @@ class Instance extends Model use HasFactory, HasUuid; protected $fillable = [ - 'customer_id', 'order_id', 'host_id', 'vmid', 'guest_ip', 'plan', 'quota_gb', 'traffic_addons', 'disk_gb', + 'customer_id', 'order_id', 'host_id', 'vmid', 'guest_ip', 'plan', 'quota_gb', 'quota_applied_gb', 'traffic_addons', 'disk_gb', 'ram_mb', 'cores', 'restart_required_since', 'subdomain', 'custom_domain', 'nc_admin_ref', 'admin_password', 'credentials_acknowledged_at', 'route_written', 'routed_hostnames', 'cert_ok', 'status', 'cancel_requested_at', 'service_ends_at', @@ -46,6 +46,7 @@ class Instance extends Model 'vmid' => 'integer', 'traffic_addons' => 'integer', 'quota_gb' => 'integer', + 'quota_applied_gb' => 'integer', 'disk_gb' => 'integer', 'ram_mb' => 'integer', 'cores' => 'integer', @@ -69,6 +70,25 @@ class Instance extends Model return $this->restart_required_since !== null; } + /** + * Has this machine's Nextcloud actually been told the allowance that was + * sold? + * + * Two columns rather than one because they answer two different questions: + * `quota_gb` is what the customer bought, `quota_applied_gb` is what the + * guest was last told. They agree on every machine the quota step has + * visited since it started recording, and they disagree on exactly the two + * cases worth finding — the estate built before the step existed, and any + * machine whose package changed while the run that should have followed did + * not finish. + */ + public function quotaIsEnforced(): bool + { + return $this->quota_gb !== null + && $this->quota_gb > 0 + && (int) $this->quota_applied_gb === (int) $this->quota_gb; + } + /** * Is there a machine here that remote work can actually reach? * diff --git a/app/Provisioning/Steps/Customer/ApplyStorageQuota.php b/app/Provisioning/Steps/Customer/ApplyStorageQuota.php index 3e62361..affc2bd 100644 --- a/app/Provisioning/Steps/Customer/ApplyStorageQuota.php +++ b/app/Provisioning/Steps/Customer/ApplyStorageQuota.php @@ -59,6 +59,14 @@ class ApplyStorageQuota extends CustomerStep // same value again is a no-op and exits 0. $this->guest($pve, $run, $occ.'config:app:set files default_quota --value='.escapeshellarg($quota.' GB')); + // Recorded only AFTER the guest accepted it — guest() throws on a + // non-zero exit, so nothing below runs for a call that failed. Without + // this line there is no way to tell a machine whose allowance is + // enforced from one where the figure has only ever been a row in our + // database, which is precisely how every instance built before this step + // existed came to have the whole disk. See clupilot:apply-quotas. + $instance->update(['quota_applied_gb' => $quota]); + return StepResult::advance(); } } diff --git a/app/Provisioning/Steps/Customer/CompleteRestart.php b/app/Provisioning/Steps/Customer/CompleteRestart.php new file mode 100644 index 0000000..9db6daf --- /dev/null +++ b/app/Provisioning/Steps/Customer/CompleteRestart.php @@ -0,0 +1,117 @@ +instance($run); + + if ($instance === null) { + return StepResult::fail('no_instance'); + } + + $node = (string) $run->context('node'); + $vmid = (int) $run->context('vmid'); + $status = $this->pve->forHost($instance->host)->vmStatus($node, $vmid); + + if (($status['status'] ?? '') !== 'running') { + if ($run->started_at !== null && $run->started_at->copy()->addSeconds(self::RUNNING_DEADLINE)->isPast()) { + return StepResult::fail('restart_did_not_come_back'); + } + + return StepResult::poll(5, 'waiting for the machine to come back'); + } + + if (($short = $this->shortfall($status, $instance->cores, $instance->ram_mb)) !== null) { + return StepResult::fail('restart_applied_nothing: '.$short); + } + + // Only here. The machine is running, its guest agent has answered (the + // step before this one), and it is running on what was sold. + if ($instance->restartIsPending()) { + $instance->update(['restart_required_since' => null]); + } + + $run->events()->create([ + 'step' => $this->key(), + 'attempt' => $run->attempt, + 'outcome' => 'info', + 'message' => "Die Maschine läuft wieder mit {$instance->cores} vCPU und {$instance->ram_mb} MB RAM. " + .'Der offene Neustart ist damit erledigt.', + ]); + + return StepResult::advance(); + } + + /** + * What the running guest is short of, or null when it is not short of + * anything we can see. + * + * Compared with `>=` rather than `===`: Proxmox rounds memory and a host may + * legitimately give a guest more than was asked for, and refusing a machine + * for being too big would be a strange way to protect a customer. + * + * @param array $status + */ + private function shortfall(array $status, ?int $cores, ?int $ramMb): ?string + { + if (isset($status['cpus']) && $cores > 0 && (int) $status['cpus'] < $cores) { + return "{$status['cpus']} statt {$cores} vCPU"; + } + + if (isset($status['maxmem']) && $ramMb > 0) { + $actualMb = (int) ((int) $status['maxmem'] / 1048576); + + if ($actualMb < $ramMb) { + return "{$actualMb} statt {$ramMb} MB RAM"; + } + } + + return null; + } +} diff --git a/app/Provisioning/Steps/Customer/ShutDownVirtualMachine.php b/app/Provisioning/Steps/Customer/ShutDownVirtualMachine.php new file mode 100644 index 0000000..3423a11 --- /dev/null +++ b/app/Provisioning/Steps/Customer/ShutDownVirtualMachine.php @@ -0,0 +1,107 @@ +instance($run); + + if ($instance === null) { + return StepResult::fail('no_instance'); + } + + $node = (string) $run->context('node'); + $vmid = (int) $run->context('vmid'); + $pve = $this->pve->forHost($instance->host); + + // Idempotent, and the ordinary way this step ends: the guest is gone. + if (($pve->vmStatus($node, $vmid)['status'] ?? '') !== 'running') { + return StepResult::advance(); + } + + // Asked once, remembered on the run. Re-sending ACPI to a guest that is + // already half-way through its own shutdown achieves nothing and reads, + // in the guest's log, like a second person pressing the power button. + if ($run->context('shutdown_upid') === null) { + $run->mergeContext([ + 'shutdown_upid' => $pve->shutdownVm($node, $vmid, self::GRACE_SECONDS), + ]); + + return StepResult::poll(10, 'waiting for the guest to shut down'); + } + + // A poll does not reset started_at, so this deadline genuinely + // accumulates across the whole wait rather than restarting every time. + if ($run->started_at !== null && $run->started_at->copy()->addSeconds(self::GRACE_SECONDS)->isPast()) { + $run->events()->create([ + 'step' => $this->key(), + 'attempt' => $run->attempt, + 'outcome' => 'info', + 'message' => 'Der Gast hat sich in '.self::GRACE_SECONDS.' Sekunden nicht heruntergefahren. ' + .'Die Maschine läuft unverändert weiter — sie wird bewusst nicht hart ausgeschaltet, ' + .'weil eine Nextcloud eine laufende Datenbank ist.', + ]); + + return StepResult::fail('shutdown_timeout'); + } + + return StepResult::poll(10, 'waiting for the guest to shut down'); + } +} diff --git a/app/Services/Proxmox/FakeProxmoxClient.php b/app/Services/Proxmox/FakeProxmoxClient.php index ce30e53..3107eec 100644 --- a/app/Services/Proxmox/FakeProxmoxClient.php +++ b/app/Services/Proxmox/FakeProxmoxClient.php @@ -100,6 +100,12 @@ class FakeProxmoxClient implements ProxmoxClient { $this->cloudInitCalls[] = $vmid; $this->cloudInitParams[$vmid] = $params; + // A config PUT lands on the VM's DEFINITION, not on the qemu process + // that is already running with the old one. Kept apart from + // $bootedConfig for exactly that reason — it is the whole point of a + // pending restart, and a fake that merged the two would let a test + // "prove" a resize had taken effect on a machine that never stopped. + $this->vmConfig[$vmid] = array_merge($this->vmConfig[$vmid] ?? [], $params); } public function resizeDisk(string $node, int $vmid, string $disk, string $size): void @@ -107,13 +113,45 @@ class FakeProxmoxClient implements ProxmoxClient $this->resizeCalls[] = $vmid.':'.$disk.':'.$size; } + /** @var array> vmid => the VM's definition on disk */ + public array $vmConfig = []; + + /** @var array> vmid => the definition the running guest booted with */ + public array $bootedConfig = []; + + /** @var array every graceful shutdown that was asked for */ + public array $shutdownCalls = []; + + /** + * A guest that will not go away when it is asked to — an unresponsive + * ACPI handler, a machine busy with something it will not be interrupted + * in. Set it to prove what the product does when the polite request is + * ignored, which is the case the whole shutdown design turns on. + */ + public bool $shutdownIgnored = false; + public function startVm(string $node, int $vmid): string { $this->runningVmids[] = $vmid; + // A cold boot is where the definition becomes what the guest actually + // runs on. This is the only place $bootedConfig is filled. + $this->bootedConfig[$vmid] = $this->vmConfig[$vmid] ?? []; return 'UPID:pve:qmstart:'.$vmid; } + public function shutdownVm(string $node, int $vmid, int $timeoutSeconds): string + { + $this->shutdownCalls[] = ['vmid' => $vmid, 'timeout' => $timeoutSeconds]; + + if (! $this->shutdownIgnored) { + $this->runningVmids = array_values(array_diff($this->runningVmids, [$vmid])); + unset($this->bootedConfig[$vmid]); + } + + return 'UPID:pve:qmshutdown:'.$vmid; + } + /** Set to e.g. 'clone' to simulate a VM still locked by a running clone task. */ public ?string $vmLock = null; @@ -125,12 +163,28 @@ class FakeProxmoxClient implements ProxmoxClient public function vmStatus(string $node, int $vmid): array { - return [ + $status = [ 'status' => in_array($vmid, $this->runningVmids, true) ? 'running' : 'stopped', 'lock' => $this->vmLock, 'netin' => $this->counters[$vmid]['netin'] ?? 0, 'netout' => $this->counters[$vmid]['netout'] ?? 0, ]; + + // Only for a guest this fake has actually booted. Proxmox reports what + // the RUNNING machine has, so a test that simply put a vmid in + // $runningVmids has said nothing about its size and must not have an + // invented figure answered back to it. + $booted = $this->bootedConfig[$vmid] ?? []; + + if (isset($booted['cores'])) { + $status['cpus'] = (int) $booted['cores']; + } + + if (isset($booted['memory'])) { + $status['maxmem'] = (int) $booted['memory'] * 1048576; // Proxmox reports bytes + } + + return $status; } public function setNetworkRate(string $node, int $vmid, ?float $mbytesPerSecond): void @@ -151,6 +205,7 @@ class FakeProxmoxClient implements ProxmoxClient $this->deletedVmids[] = $vmid; $this->clonedVmids = array_values(array_diff($this->clonedVmids, [$vmid])); $this->runningVmids = array_values(array_diff($this->runningVmids, [$vmid])); + unset($this->vmConfig[$vmid], $this->bootedConfig[$vmid]); } public function guestAgentPing(string $node, int $vmid): bool diff --git a/app/Services/Proxmox/HttpProxmoxClient.php b/app/Services/Proxmox/HttpProxmoxClient.php index 430eafa..7bc7fc2 100644 --- a/app/Services/Proxmox/HttpProxmoxClient.php +++ b/app/Services/Proxmox/HttpProxmoxClient.php @@ -102,6 +102,18 @@ class HttpProxmoxClient implements ProxmoxClient ->post("/nodes/{$node}/qemu/{$vmid}/status/start")->throw()->json('data'); } + public function shutdownVm(string $node, int $vmid, int $timeoutSeconds): string + { + // forceStop is hard-coded to 0 and is not a parameter of this method: + // with it set, Proxmox destroys the qemu process once `timeout` expires, + // which is a power cut to a running database. Left at 0 the task simply + // ends in an error and the guest keeps running, which is the outcome the + // calling step is written around. + return (string) $this->http()->asForm() + ->post("/nodes/{$node}/qemu/{$vmid}/status/shutdown", ['timeout' => $timeoutSeconds, 'forceStop' => 0]) + ->throw()->json('data'); + } + public function vmStatus(string $node, int $vmid): array { return $this->http()->get("/nodes/{$node}/qemu/{$vmid}/status/current")->throw()->json('data', []); diff --git a/app/Services/Proxmox/ProxmoxClient.php b/app/Services/Proxmox/ProxmoxClient.php index 52e4405..0a341a1 100644 --- a/app/Services/Proxmox/ProxmoxClient.php +++ b/app/Services/Proxmox/ProxmoxClient.php @@ -48,6 +48,24 @@ interface ProxmoxClient /** Start a VM; returns the task UPID. */ public function startVm(string $node, int $vmid): string; + /** + * ASK the guest to shut itself down, and return the task UPID. + * + * There is deliberately no counterpart that pulls the plug. Proxmox's + * shutdown endpoint takes a `forceStop` flag that turns the request into a + * power cut once the timeout runs out, and this interface does not expose + * it: every guest we run is a Nextcloud, which is a database, and cutting + * power to one mid-write to make a CPU change take effect trades a visible + * inconvenience for a possible restore from backup. A guest that ignores + * ACPI for the whole grace period is a guest that is doing something, and + * the right answer there is a failed run an operator can look at — see + * App\Provisioning\Steps\Customer\ShutDownVirtualMachine. + * + * $timeoutSeconds is how long Proxmox itself waits for the guest before it + * gives the task up. + */ + public function shutdownVm(string $node, int $vmid, int $timeoutSeconds): string; + /** @return array ['status' => 'running'|'stopped', …] */ public function vmStatus(string $node, int $vmid): array; diff --git a/config/provisioning.php b/config/provisioning.php index a137368..2177dc9 100644 --- a/config/provisioning.php +++ b/config/provisioning.php @@ -109,6 +109,43 @@ return [ Customer\ConfigureNextcloud::class, Customer\SettlePlanServices::class, ], + + /* + | Taking a customer's machine round once, on purpose. + | + | A restart is a shutdown and a start, in that order and never a reboot: + | only a cold boot makes qemu read the VM definition again, so only a + | cold boot delivers the cores and memory a plan change wrote there. + | ResizeVirtualMachine leaves `restart_required_since` set precisely + | because nothing in the product could do this, which is how a paid + | upgrade could stay invisible forever. + | + | StartVirtualMachine and WaitForGuestAgent are the customer pipeline's + | own steps, reused rather than repeated — starting a machine and waiting + | for its agent is the same operation whichever run needs it. Started by + | App\Actions\RestartInstance, from the customer's cloud page or the + | console's instance list. + */ + 'restart' => [ + Customer\ShutDownVirtualMachine::class, + Customer\StartVirtualMachine::class, + Customer\WaitForGuestAgent::class, + Customer\CompleteRestart::class, + ], + + /* + | The storage allowance, and nothing else. + | + | One step, because that is the whole of the repair: every instance built + | before ApplyStorageQuota joined the `customer` pipeline has a quota_gb + | in its row that Nextcloud was never told about. Started one run per + | instance by `clupilot:apply-quotas`, so the sweep is retried, logged + | and visible in the console like every other piece of remote work, + | rather than a console command reaching into guests by hand. + */ + 'quota' => [ + Customer\ApplyStorageQuota::class, + ], ], // The one currency the catalogue is priced in. Plan prices carry no currency diff --git a/database/migrations/2026_07_29_250000_add_instance_restart_capability.php b/database/migrations/2026_07_29_250000_add_instance_restart_capability.php new file mode 100644 index 0000000..193be7a --- /dev/null +++ b/database/migrations/2026_07_29_250000_add_instance_restart_capability.php @@ -0,0 +1,42 @@ +forgetCachedPermissions(); + + Permission::findOrCreate('instances.restart', 'operator'); + foreach (['Owner', 'Admin', 'Support'] as $role) { + Role::findOrCreate($role, 'operator')->givePermissionTo('instances.restart'); + } + + app(PermissionRegistrar::class)->forgetCachedPermissions(); + } + + public function down(): void + { + app(PermissionRegistrar::class)->forgetCachedPermissions(); + Permission::query()->where('name', 'instances.restart')->where('guard_name', 'operator')->delete(); + app(PermissionRegistrar::class)->forgetCachedPermissions(); + } +}; diff --git a/database/migrations/2026_07_29_260000_add_applied_quota_to_instances.php b/database/migrations/2026_07_29_260000_add_applied_quota_to_instances.php new file mode 100644 index 0000000..461749f --- /dev/null +++ b/database/migrations/2026_07_29_260000_add_applied_quota_to_instances.php @@ -0,0 +1,39 @@ +unsignedInteger('quota_applied_gb')->nullable()->after('quota_gb'); + }); + } + + public function down(): void + { + Schema::table('instances', function (Blueprint $table) { + $table->dropColumn('quota_applied_gb'); + }); + } +}; diff --git a/lang/de/admin.php b/lang/de/admin.php index a267030..30ee166 100644 --- a/lang/de/admin.php +++ b/lang/de/admin.php @@ -79,7 +79,7 @@ return [ ], 'notice' => [ - 'no_capacity' => "Kein Host hat noch Platz für „:plan“ — das Paket braucht :needs GB, der freieste Host hat :largest GB. Eine bezahlte Bestellung dafür würde bei der Reservierung scheitern.", + 'no_capacity' => 'Kein Host hat noch Platz für „:plan“ — das Paket braucht :needs GB, der freieste Host hat :largest GB. Eine bezahlte Bestellung dafür würde bei der Reservierung scheitern.', 'failed_runs' => ':n fehlgeschlagene Bereitstellung(en).', 'host_error' => 'Host :host meldet einen Fehler.', 'host_silent' => 'Host :host hat sich seit über :minutes Minuten nicht gemeldet.', diff --git a/lang/de/cloud.php b/lang/de/cloud.php index 765fa84..0b2c0d0 100644 --- a/lang/de/cloud.php +++ b/lang/de/cloud.php @@ -33,6 +33,13 @@ return [ 'restart_pending_title' => 'Neustart erforderlich', 'restart_pending_body' => 'Ihr neues Paket ist aktiv. Mehr Prozessorkerne und Arbeitsspeicher werden erst beim nächsten Neustart Ihrer Cloud wirksam — Ihre Cloud wurde dafür bewusst nicht von selbst neu gestartet.', 'restart' => 'Neu starten', + 'restart_title' => 'Cloud neu starten?', + 'restart_body' => 'Ihre Cloud wird sauber heruntergefahren und wieder gestartet. Das dauert meist ein bis zwei Minuten; in dieser Zeit ist sie nicht erreichbar, und alle angemeldeten Benutzer werden getrennt. Nicht gespeicherte Arbeit in Office-Dokumenten geht dabei verloren.', + 'restart_cancel' => 'Abbrechen', + 'restart_confirm' => 'Jetzt neu starten', + 'restart_started' => 'Neustart wurde gestartet. Ihre Cloud ist in wenigen Minuten wieder erreichbar.', + 'restart_busy' => 'Gerade läuft schon eine Arbeit an Ihrer Cloud. Bitte versuchen Sie es in einigen Minuten erneut.', + 'restart_denied' => 'Für diese Aktion fehlt die Berechtigung.', 'snapshot' => 'Snapshot erstellen', 'logs' => 'Protokolle', 'action_toast' => 'Aktion im Prototyp nur angedeutet.', diff --git a/lang/de/provisioning.php b/lang/de/provisioning.php index 09d64bf..e5d32e4 100644 --- a/lang/de/provisioning.php +++ b/lang/de/provisioning.php @@ -20,6 +20,8 @@ return [ 'resize_vm' => 'VM-Ausstattung anpassen', 'apply_storage_quota' => 'Speicherkontingent setzen', 'settle_plan_services' => 'Backup & Monitoring abgleichen', + 'shut_down_vm' => 'VM sauber herunterfahren', + 'complete_restart' => 'Neustart abschließen', ], 'mail' => [ diff --git a/lang/en/admin.php b/lang/en/admin.php index 544612d..f287e61 100644 --- a/lang/en/admin.php +++ b/lang/en/admin.php @@ -79,7 +79,7 @@ return [ ], 'notice' => [ - 'no_capacity' => "No host has room for “:plan” any more — the package needs :needs GB and the roomiest host has :largest GB. A paid order for it would fail at reservation.", + 'no_capacity' => 'No host has room for “:plan” any more — the package needs :needs GB and the roomiest host has :largest GB. A paid order for it would fail at reservation.', 'failed_runs' => ':n failed provisioning run(s).', 'host_error' => 'Host :host reports an error.', 'host_silent' => 'Host :host has not checked in for over :minutes minutes.', diff --git a/lang/en/cloud.php b/lang/en/cloud.php index bf22320..59d5706 100644 --- a/lang/en/cloud.php +++ b/lang/en/cloud.php @@ -33,6 +33,13 @@ return [ 'restart_pending_title' => 'Restart required', 'restart_pending_body' => 'Your new plan is active. The extra CPU cores and memory only take effect the next time your cloud restarts — we deliberately did not restart it for you.', 'restart' => 'Restart', + 'restart_title' => 'Restart your cloud?', + 'restart_body' => 'Your cloud will be shut down cleanly and started again. That usually takes a minute or two, during which it is unreachable and every signed-in user is disconnected. Unsaved work in office documents will be lost.', + 'restart_cancel' => 'Cancel', + 'restart_confirm' => 'Restart now', + 'restart_started' => 'The restart has begun. Your cloud will be back in a few minutes.', + 'restart_busy' => 'Work on your cloud is already under way. Please try again in a few minutes.', + 'restart_denied' => 'You are not allowed to do that.', 'snapshot' => 'Create snapshot', 'logs' => 'Logs', 'action_toast' => 'Action is only indicated in this prototype.', diff --git a/lang/en/provisioning.php b/lang/en/provisioning.php index 3207f14..384bebf 100644 --- a/lang/en/provisioning.php +++ b/lang/en/provisioning.php @@ -20,6 +20,8 @@ return [ 'resize_vm' => 'Resize the VM', 'apply_storage_quota' => 'Apply storage quota', 'settle_plan_services' => 'Reconcile backup & monitoring', + 'shut_down_vm' => 'Shut the VM down cleanly', + 'complete_restart' => 'Finish the restart', ], 'mail' => [ diff --git a/resources/views/components/ui/badge.blade.php b/resources/views/components/ui/badge.blade.php index a66823f..c50fea2 100644 --- a/resources/views/components/ui/badge.blade.php +++ b/resources/views/components/ui/badge.blade.php @@ -10,6 +10,9 @@ 'pending' => 'info', 'suspended' => 'warning', 'cancellation_scheduled' => 'warning', + // A service that has run out. Not an error and not a health state — + // the same muted "this is over" as a closed account. + 'ended' => 'warning', 'closed' => 'warning', 'warning' => 'warning', 'failed' => 'danger', diff --git a/resources/views/livewire/admin/confirm-restart-instance.blade.php b/resources/views/livewire/admin/confirm-restart-instance.blade.php new file mode 100644 index 0000000..371c6f5 --- /dev/null +++ b/resources/views/livewire/admin/confirm-restart-instance.blade.php @@ -0,0 +1,17 @@ +
+
+ + + +
+

{{ __('admin.restart_title') }}

+

{{ __('admin.restart_body', ['address' => $address]) }}

+
+
+
+ {{ __('admin.restart_cancel') }} + + {{ __('admin.restart_confirm') }} + +
+
diff --git a/resources/views/livewire/admin/instances.blade.php b/resources/views/livewire/admin/instances.blade.php index a61384e..612a558 100644 --- a/resources/views/livewire/admin/instances.blade.php +++ b/resources/views/livewire/admin/instances.blade.php @@ -16,6 +16,9 @@ {{ __('admin.col.plan') }} {{ __('admin.col.quota') }} {{ __('admin.col.status') }} + @if ($canRestart) + {{ __('admin.col.actions') }} + @endif @@ -38,9 +41,19 @@ @endif + @if ($canRestart) + + @if ($r['live']) + + {{ __('admin.restart') }} + + @endif + + @endif @empty - {{ __('admin.instances_empty') }} + {{ __('admin.instances_empty') }} @endforelse diff --git a/resources/views/livewire/cloud.blade.php b/resources/views/livewire/cloud.blade.php index 2ef79b0..a3888fa 100644 --- a/resources/views/livewire/cloud.blade.php +++ b/resources/views/livewire/cloud.blade.php @@ -71,7 +71,10 @@
- {{ __('cloud.restart') }} + + {{ __('cloud.restart') }} + {{ __('cloud.snapshot') }} {{ __('cloud.logs') }}
diff --git a/resources/views/livewire/confirm-restart-cloud.blade.php b/resources/views/livewire/confirm-restart-cloud.blade.php new file mode 100644 index 0000000..8a6aa6e --- /dev/null +++ b/resources/views/livewire/confirm-restart-cloud.blade.php @@ -0,0 +1,17 @@ +
+
+ + + +
+

{{ __('cloud.restart_title') }}

+

{{ __('cloud.restart_body') }}

+
+
+
+ {{ __('cloud.restart_cancel') }} + + {{ __('cloud.restart_confirm') }} + +
+
diff --git a/routes/console.php b/routes/console.php index a32e74e..25f9b4c 100644 --- a/routes/console.php +++ b/routes/console.php @@ -121,3 +121,18 @@ Schedule::job(new PingHosts) Schedule::command('clupilot:apply-due-plan-changes') ->everyFifteenMinutes() ->withoutOverlapping(); + +// Keep the appointment a cancellation made. +// +// ConfirmCancelPackage writes a date and nothing ever went back to it, which is +// why TraefikWriter::remove() had no caller in the whole application: a +// cancelled instance kept its router, its certificate and its route to a guest +// address the host is free to reassign. Only instances whose paid term has +// actually run out are touched — a cancellation that is merely SCHEDULED is a +// paying customer, and this command is written around that distinction. +// +// Hourly: a term ends at a moment, and neither giving away a day of service nor +// hammering a DNS provider by the minute is a decision worth making by accident. +Schedule::command('clupilot:end-due-services') + ->hourly() + ->withoutOverlapping(); diff --git a/tests/Feature/Admin/RbacMoveTest.php b/tests/Feature/Admin/RbacMoveTest.php index 5b8d756..77f2c9b 100644 --- a/tests/Feature/Admin/RbacMoveTest.php +++ b/tests/Feature/Admin/RbacMoveTest.php @@ -12,8 +12,11 @@ use Spatie\Permission\Models\Role; it('moves every permission and role to the operator guard, leaving none behind', function () { expect(Permission::where('guard_name', 'web')->count())->toBe(0) ->and(Role::where('guard_name', 'web')->count())->toBe(0) - // 17 from the original seed, plus customers.grant_plan. - ->and(Permission::where('guard_name', 'operator')->count())->toBe(18) + // 17 from the original seed, plus customers.grant_plan and + // instances.restart — the latter created straight onto this guard, + // because since this migration `web` is where a permission goes to + // match nobody at all. + ->and(Permission::where('guard_name', 'operator')->count())->toBe(19) ->and(Role::where('guard_name', 'operator')->count())->toBe(6); }); @@ -154,7 +157,7 @@ it('preflights every customer conflict before mutating anything, listing all of // Nothing mutated: still exactly the pre-migration shape, not "moved and // then rolled back" — there is nothing here to roll back on a real // server, which is the whole reason this has to be checked up front. - expect(Permission::where('guard_name', 'web')->count())->toBe(18) + expect(Permission::where('guard_name', 'web')->count())->toBe(19) ->and(Permission::where('guard_name', 'operator')->count())->toBe(0) ->and(Role::where('guard_name', 'web')->count())->toBe(6) ->and(Role::where('guard_name', 'operator')->count())->toBe(0) diff --git a/tests/Feature/Console/ApplyStorageQuotasTest.php b/tests/Feature/Console/ApplyStorageQuotasTest.php new file mode 100644 index 0000000..e5f5d28 --- /dev/null +++ b/tests/Feature/Console/ApplyStorageQuotasTest.php @@ -0,0 +1,154 @@ +active()->create(['datacenter' => 'fsn', 'node' => 'pve']); + $order = Order::factory()->withSubscription()->create(['datacenter' => 'fsn', 'plan' => 'business']); + + return Instance::factory()->create(array_merge([ + 'order_id' => $order->id, + 'customer_id' => $order->customer_id, + 'host_id' => $host->id, + 'vmid' => 101, + 'status' => 'active', + 'quota_gb' => 500, + 'quota_applied_gb' => null, + ], $attributes)); +} + +/** Carry out every quota run the command started. */ +function runQuotaRuns(): void +{ + $runner = app(RunRunner::class); + + ProvisioningRun::query()->where('pipeline', 'quota')->get() + ->each(fn (ProvisioningRun $run) => $runner->advance($run)); +} + +it('applies the allowance to an instance that never had one, and records that it did', function () { + $services = fakeServices(); + Queue::fake(); + + $instance = unquotedInstance(); + + $this->artisan('clupilot:apply-quotas')->assertSuccessful(); + runQuotaRuns(); + + expect($services['pve']->guestRan("config:app:set files default_quota --value='500 GB'"))->toBeTrue() + ->and($instance->fresh()->quota_applied_gb)->toBe(500) + ->and($instance->fresh()->quotaIsEnforced())->toBeTrue(); +}); + +it('is a no-op the second time round', function () { + $services = fakeServices(); + Queue::fake(); + + unquotedInstance(); + + $this->artisan('clupilot:apply-quotas')->assertSuccessful(); + runQuotaRuns(); + + $callsAfterFirstSweep = count($services['pve']->guestCommands); + + $this->artisan('clupilot:apply-quotas') + ->expectsOutputToContain('bereits gesetzt') + ->assertSuccessful(); + runQuotaRuns(); + + expect(ProvisioningRun::query()->where('pipeline', 'quota')->count())->toBe(1) + ->and($services['pve']->guestCommands)->toHaveCount($callsAfterFirstSweep); +}); + +it('changes nothing at all in dry-run mode', function () { + $services = fakeServices(); + Queue::fake(); + + $instance = unquotedInstance(); + + $this->artisan('clupilot:apply-quotas', ['--dry-run' => true]) + ->expectsOutputToContain('Probelauf') + ->assertSuccessful(); + + expect(ProvisioningRun::query()->where('pipeline', 'quota')->count())->toBe(0) + ->and($services['pve']->guestCommands)->toBe([]) + ->and($instance->fresh()->quota_applied_gb)->toBeNull(); +}); + +it('skips what it has no business touching, and says which', function () { + fakeServices(); + Queue::fake(); + + // Still being built: its own run is writing this machine. + unquotedInstance(['status' => 'provisioning']); + // Nothing to reach at all. + unquotedInstance(['status' => 'failed', 'vmid' => null]); + // No allowance on the row. Writing "0 GB" would lock the customer out of + // their own files, which is worse than leaving what is there. + unquotedInstance(['quota_gb' => 0]); + // A service that has run out. + unquotedInstance(['status' => 'ended']); + + $busy = unquotedInstance(); + ProvisioningRun::factory()->create([ + 'subject_type' => Order::class, + 'subject_id' => $busy->order_id, + 'pipeline' => 'plan-change', + 'status' => ProvisioningRun::STATUS_RUNNING, + ]); + + $this->artisan('clupilot:apply-quotas') + ->expectsOutputToContain('keine erreichbare Maschine') + ->expectsOutputToContain('kein Kontingent hinterlegt') + ->expectsOutputToContain('anderer Lauf aktiv') + ->assertSuccessful(); + + expect(ProvisioningRun::query()->where('pipeline', 'quota')->count())->toBe(0); +}); + +it('re-applies an allowance that has changed since it was last enforced', function () { + $services = fakeServices(); + Queue::fake(); + + // A downgrade landed on the row but the run that should have followed it + // never finished — the customer is paying for less and still holds more. + $instance = unquotedInstance(['quota_gb' => 200, 'quota_applied_gb' => 500]); + + $this->artisan('clupilot:apply-quotas')->assertSuccessful(); + runQuotaRuns(); + + expect($services['pve']->guestRan("config:app:set files default_quota --value='200 GB'"))->toBeTrue() + ->and($instance->fresh()->quota_applied_gb)->toBe(200); +}); + +it('repairs one named instance when asked to', function () { + fakeServices(); + Queue::fake(); + + $wanted = unquotedInstance(); + unquotedInstance(); + + $this->artisan('clupilot:apply-quotas', ['--instance' => $wanted->uuid])->assertSuccessful(); + + expect(ProvisioningRun::query()->where('pipeline', 'quota')->count())->toBe(1) + ->and(ProvisioningRun::query()->where('pipeline', 'quota')->first()->context('instance_id')) + ->toBe($wanted->id); +}); diff --git a/tests/Feature/Provisioning/EndInstanceServiceTest.php b/tests/Feature/Provisioning/EndInstanceServiceTest.php new file mode 100644 index 0000000..4b54289 --- /dev/null +++ b/tests/Feature/Provisioning/EndInstanceServiceTest.php @@ -0,0 +1,198 @@ +active()->create(['datacenter' => 'fsn', 'node' => 'pve', 'public_ip' => '203.0.113.9']); + $order = Order::factory()->withSubscription()->create(['datacenter' => 'fsn', 'plan' => 'business']); + + $instance = Instance::factory()->create(array_merge([ + 'order_id' => $order->id, + 'customer_id' => $order->customer_id, + 'host_id' => $host->id, + 'vmid' => 101, + 'guest_ip' => '10.20.0.7', + 'subdomain' => $subdomain, + 'status' => 'active', + 'route_written' => true, + 'routed_hostnames' => [$subdomain.'.'.ProvisioningSettings::dnsZone()], + 'cert_ok' => true, + ], $attributes)); + + $recordId = $services['dns']->upsertRecord($subdomain.'.'.ProvisioningSettings::dnsZone(), 'A', $host->public_ip); + $instance->dnsRecords()->create([ + 'provider' => 'hetzner', + 'record_id' => $recordId, + 'fqdn' => $subdomain.'.'.ProvisioningSettings::dnsZone(), + 'type' => 'A', + 'value' => $host->public_ip, + ]); + + $services['traefik']->write($host->wg_ip ?? $host->public_ip, $subdomain, $instance->routed_hostnames, $instance->guest_ip); + + return compact('services', 'host', 'order', 'instance'); +} + +it('takes the router and the DNS record down when the paid term has run out', function () { + ['services' => $services, 'instance' => $instance] = routedInstance([ + 'status' => 'cancellation_scheduled', + 'cancel_requested_at' => now()->subMonth(), + 'service_ends_at' => now()->subHour(), + ]); + + expect($services['traefik']->routes)->toHaveKey('berger'); + + $this->artisan('clupilot:end-due-services')->assertSuccessful(); + + expect($services['traefik']->routes)->not->toHaveKey('berger') + // The A record goes with the route: it is in OUR zone, it points at a + // host that goes on serving other customers, and a name we own that + // resolves to a live machine with nothing of its owner's behind it is + // the shape of every subdomain takeover there has ever been. + ->and($services['dns']->records)->toBe([]) + ->and($instance->dnsRecords()->count())->toBe(0) + ->and($instance->fresh()->status)->toBe('ended') + // The address state goes too, so a revived instance is not announced + // and routed nowhere by a step that thinks it has already written. + ->and($instance->fresh()->route_written)->toBeFalse() + ->and($instance->fresh()->routed_hostnames)->toBeNull() + ->and($instance->fresh()->cert_ok)->toBeFalse(); +}); + +it('leaves a scheduled cancellation exactly where it is until the day arrives', function () { + ['services' => $services, 'instance' => $instance] = routedInstance([ + 'status' => 'cancellation_scheduled', + 'cancel_requested_at' => now(), + 'service_ends_at' => now()->addWeeks(3), + ]); + + $this->artisan('clupilot:end-due-services')->assertSuccessful(); + + // They have paid to the end of the term and are working in it this + // afternoon. Pulling the address the day somebody schedules a cancellation + // would be a far worse fault than the leak this closes. + expect($services['traefik']->routes)->toHaveKey('berger') + ->and($services['dns']->records)->not->toBe([]) + ->and($instance->fresh()->status)->toBe('cancellation_scheduled') + ->and($instance->fresh()->route_written)->toBeTrue(); +}); + +it('does not touch an active instance, however old', function () { + ['services' => $services, 'instance' => $instance] = routedInstance([ + 'created_at' => now()->subYears(2), + ]); + + $this->artisan('clupilot:end-due-services')->assertSuccessful(); + + expect($services['traefik']->routes)->toHaveKey('berger') + ->and($instance->fresh()->status)->toBe('active'); +}); + +it('leaves a cancellation with no end date alone rather than inventing one', function () { + ['services' => $services, 'instance' => $instance] = routedInstance([ + 'status' => 'cancellation_scheduled', + 'cancel_requested_at' => now()->subMonth(), + 'service_ends_at' => null, + ]); + + $this->artisan('clupilot:end-due-services')->assertSuccessful(); + + // A broken record, not an expired one. Guessing an end date for it would + // invent the very moment this whole thing is written to respect. + expect($services['traefik']->routes)->toHaveKey('berger') + ->and($instance->fresh()->status)->toBe('cancellation_scheduled'); +}); + +it('changes nothing in dry-run mode', function () { + ['services' => $services, 'instance' => $instance] = routedInstance([ + 'status' => 'cancellation_scheduled', + 'service_ends_at' => now()->subDay(), + ]); + + $this->artisan('clupilot:end-due-services', ['--dry-run' => true]) + ->expectsOutputToContain('Probelauf') + ->assertSuccessful(); + + expect($services['traefik']->routes)->toHaveKey('berger') + ->and($services['dns']->records)->not->toBe([]) + ->and($instance->fresh()->status)->toBe('cancellation_scheduled'); +}); + +it('is safe to run twice', function () { + ['services' => $services, 'instance' => $instance] = routedInstance([ + 'status' => 'cancellation_scheduled', + 'service_ends_at' => now()->subDay(), + ]); + + $this->artisan('clupilot:end-due-services')->assertSuccessful(); + $this->artisan('clupilot:end-due-services')->assertSuccessful(); + + expect($instance->fresh()->status)->toBe('ended') + ->and($services['traefik']->routes)->toBe([]); +}); + +it('carries the whole way from the customer cancelling to the address going away', function () { + ['services' => $services, 'instance' => $instance] = routedInstance(); + + $user = User::factory()->create(['email_verified_at' => now()]); + Customer::query()->whereKey($instance->customer_id)->update(['user_id' => $user->id, 'email' => $user->email]); + + Livewire::actingAs($user)->test(ConfirmCancelPackage::class) + ->set('confirmName', $instance->subdomain) + ->call('cancelPackage'); + + $instance->refresh(); + + expect($instance->status)->toBe('cancellation_scheduled') + ->and($instance->service_ends_at)->not->toBeNull(); + + // Still theirs, still served, for the whole of the term they bought. + $this->artisan('clupilot:end-due-services')->assertSuccessful(); + expect($services['traefik']->routes)->toHaveKey('berger'); + + // And gone the moment it is genuinely over. + $this->travelTo($instance->service_ends_at->copy()->addMinute()); + $this->artisan('clupilot:end-due-services')->assertSuccessful(); + + expect($services['traefik']->routes)->toBe([]) + ->and($instance->fresh()->status)->toBe('ended'); +}); + +it('refuses to end an instance whose term has not run out, whoever asks', function () { + ['instance' => $instance] = routedInstance([ + 'status' => 'cancellation_scheduled', + 'service_ends_at' => now()->addMonth(), + ]); + + // The rule lives on the action, not in the command's query — anything that + // ever ends an instance has to ask the same question. + expect(app(EndInstanceService::class)->hasEnded($instance))->toBeFalse() + ->and(app(EndInstanceService::class)($instance))->toBeFalse() + ->and($instance->fresh()->status)->toBe('cancellation_scheduled'); +}); diff --git a/tests/Feature/Provisioning/InstanceRestartTest.php b/tests/Feature/Provisioning/InstanceRestartTest.php new file mode 100644 index 0000000..656673f --- /dev/null +++ b/tests/Feature/Provisioning/InstanceRestartTest.php @@ -0,0 +1,330 @@ +active()->create(['datacenter' => 'fsn', 'node' => 'pve']); + $order = Order::factory()->withSubscription()->create(['datacenter' => 'fsn', 'plan' => 'business']); + + $instance = Instance::factory()->create(array_merge([ + 'order_id' => $order->id, + 'customer_id' => $order->customer_id, + 'host_id' => $host->id, + 'vmid' => 101, + 'guest_ip' => '10.20.0.7', + 'status' => 'active', + // What the contract now says. The guest below is running on less. + 'cores' => 8, + 'ram_mb' => 16384, + 'restart_required_since' => now()->subDays(3), + ], $instanceAttributes)); + + // The machine as it actually is: booted small, then given a bigger + // definition it has not read yet. + $pve->setCloudInit('pve', 101, ['cores' => 2, 'memory' => 4096]); + $pve->startVm('pve', 101); + $pve->setCloudInit('pve', 101, ['cores' => 8, 'memory' => 16384]); + + return compact('services', 'pve', 'host', 'order', 'instance'); +} + +/** + * Advance a run until it settles, clearing the backoff between steps. + * + * next_attempt_at is what makes a poll a wait; a test that did not clear it + * would simply watch the runner decline to do anything. + */ +function driveRun(ProvisioningRun $run, int $limit = 25): ProvisioningRun +{ + $runner = app(RunRunner::class); + + for ($i = 0; $i < $limit; $i++) { + $run->refresh(); + + if (in_array($run->status, [ProvisioningRun::STATUS_COMPLETED, ProvisioningRun::STATUS_FAILED], true)) { + break; + } + + $run->forceFill(['next_attempt_at' => null])->save(); + $runner->advance($run); + } + + return $run->refresh(); +} + +it('asks the guest to shut down, brings it back, and only then stops saying a restart is due', function () { + Queue::fake(); + ['pve' => $pve, 'instance' => $instance] = restartableInstance(); + $this->actingAs(admin(), 'operator'); + + // Before: the definition is big, the running machine is not — which is what + // "Neustart erforderlich" has been telling the customer for three days. + expect($pve->vmStatus('pve', 101)['cpus'])->toBe(2) + ->and($instance->restartIsPending())->toBeTrue(); + + $run = app(RestartInstance::class)($instance); + expect($run)->not->toBeNull(); + + // Pressing the button does not clear it. Nothing has happened to the + // machine yet, and a flag that clears on intent is a flag that lies. + expect($instance->fresh()->restartIsPending())->toBeTrue(); + + driveRun($run); + + expect($run->fresh()->status)->toBe(ProvisioningRun::STATUS_COMPLETED) + // The guest was ASKED, and it was asked exactly once. + ->and($pve->shutdownCalls)->toHaveCount(1) + ->and($pve->shutdownCalls[0]['vmid'])->toBe(101) + // It came back — and came back on the definition it had been given, + // which is the only thing that makes the upgrade real. + ->and($pve->vmStatus('pve', 101)['status'])->toBe('running') + ->and($pve->vmStatus('pve', 101)['cpus'])->toBe(8) + ->and((int) ($pve->vmStatus('pve', 101)['maxmem'] / 1048576))->toBe(16384) + ->and($instance->fresh()->restartIsPending())->toBeFalse(); +}); + +it('never pulls the plug on a guest that will not shut down', function () { + Queue::fake(); + ['pve' => $pve, 'instance' => $instance, 'order' => $order] = restartableInstance(); + $order->update(['status' => 'active']); + $this->actingAs(admin(), 'operator'); + + // A guest that ignores ACPI — a wedged handler, or a machine busy with + // something it will not be interrupted in. + $pve->shutdownIgnored = true; + + $run = app(RestartInstance::class)($instance); + + driveRun($run); + // Past the step's grace period but inside the runner's own step timeout, so + // the step's deliberate failure fires rather than a generic retry. + $this->travel(11)->minutes(); + $run = driveRun($run); + + expect($run->status)->toBe(ProvisioningRun::STATUS_FAILED) + ->and($run->error)->toBe('shutdown_timeout') + // The machine is still running. Nextcloud is a database and this + // product has no power cut to escalate to — the run fails and a person + // decides what happens next. + ->and($pve->runningVmids)->toContain(101) + ->and($pve->vmStatus('pve', 101)['status'])->toBe('running') + // And nothing has been declared finished on the strength of a failure. + ->and($instance->fresh()->restartIsPending())->toBeTrue() + // A maintenance pipeline failing must not release a paid, running + // customer — RunRunner's subject hook only fires for the pipeline that + // BUILDS the subject, and a restart is not that one. + ->and($order->fresh()->status)->toBe('active') + ->and($instance->fresh()->status)->toBe('active'); +}); + +it('refuses to call a restart done when the machine came back smaller than it was sold', function () { + Queue::fake(); + ['pve' => $pve, 'instance' => $instance, 'order' => $order] = restartableInstance(); + + // The machine went round but the bigger definition never took — a config + // write that silently failed, a host that could not honour it. + $pve->shutdownVm('pve', 101, 600); + $pve->vmConfig[101] = ['cores' => 2, 'memory' => 4096]; + $pve->startVm('pve', 101); + + $run = ProvisioningRun::factory()->create([ + 'subject_type' => Order::class, 'subject_id' => $order->id, 'pipeline' => 'restart', + 'context' => ['instance_id' => $instance->id, 'node' => 'pve', 'vmid' => 101], + ]); + + $result = app(CompleteRestart::class)->execute($run); + + expect($result->type)->toBe('fail') + ->and($result->reason)->toContain('2 statt 8 vCPU') + ->and($instance->fresh()->restartIsPending())->toBeTrue(); +}); + +it('leaves the pending restart alone until the very last step of the run', function () { + Queue::fake(); + ['instance' => $instance] = restartableInstance(); + $this->actingAs(admin(), 'operator'); + + $run = app(RestartInstance::class)($instance); + $runner = app(RunRunner::class); + + // Shutdown asked, shutdown observed, machine started, agent answered — and + // through all of it the flag stands, because none of those is the machine + // running on what was bought. + for ($i = 0; $i < 6; $i++) { + $run->refresh(); + + if ($run->current_step >= 3) { + break; + } + + $run->forceFill(['next_attempt_at' => null])->save(); + $runner->advance($run); + + expect($instance->fresh()->restartIsPending())->toBeTrue(); + } + + expect($run->refresh()->current_step)->toBe(3) + ->and($instance->fresh()->restartIsPending())->toBeTrue(); + + driveRun($run); + + expect($instance->fresh()->restartIsPending())->toBeFalse(); +}); + +it('lets a customer restart their own machine and nobody else’s', function () { + Queue::fake(); + ['instance' => $mine] = restartableInstance(); + ['instance' => $theirs] = restartableInstance(); + + $user = User::factory()->create(['email_verified_at' => now()]); + $mine->customer->update(['user_id' => $user->id, 'email' => $user->email]); + + $this->actingAs($user); + + // Checked in the action, not in the view. A Livewire method is reachable by + // anyone who can POST to /livewire/update, so hiding a button proves + // nothing at all about who may press it. + expect(fn () => app(RestartInstance::class)($theirs))->toThrow(AuthorizationException::class); + expect(app(RestartInstance::class)->allows($theirs))->toBeFalse() + ->and(app(RestartInstance::class)->allows($mine))->toBeTrue(); + + expect(app(RestartInstance::class)($mine))->not->toBeNull(); +}); + +it('lets an operator with the capability restart any machine, and refuses one without it', function () { + Queue::fake(); + ['instance' => $instance] = restartableInstance(); + + $this->actingAs(operator('Read-only'), 'operator'); + expect(fn () => app(RestartInstance::class)($instance))->toThrow(AuthorizationException::class); + expect(ProvisioningRun::query()->where('pipeline', 'restart')->count())->toBe(0); + + $this->actingAs(admin(), 'operator'); + expect(app(RestartInstance::class)($instance))->not->toBeNull() + ->and(ProvisioningRun::query()->where('pipeline', 'restart')->count())->toBe(1); +}); + +it('does not start a restart while another run is already in flight for that instance', function () { + Queue::fake(); + ['instance' => $instance, 'order' => $order] = restartableInstance(); + + // A plan change still working on the same machine. Stopping the guest + // underneath it would race the config it is about to boot. + ProvisioningRun::factory()->create([ + 'subject_type' => Order::class, + 'subject_id' => $order->id, + 'pipeline' => 'plan-change', + 'status' => ProvisioningRun::STATUS_RUNNING, + ]); + + $this->actingAs(admin(), 'operator'); + + expect(app(RestartInstance::class)($instance))->toBeNull() + ->and(ProvisioningRun::query()->where('pipeline', 'restart')->count())->toBe(0); +}); + +it('refuses a machine there is nothing to restart', function () { + Queue::fake(); + ['instance' => $instance] = restartableInstance(['status' => 'provisioning']); + + $this->actingAs(admin(), 'operator'); + + // Still being built: its own run is writing this machine, and a shutdown + // beside it would stop the guest that run is talking to. + expect(app(RestartInstance::class)($instance))->toBeNull(); +}); + +it('confirms in a modal on both sides and drives the real action from the page', function () { + Queue::fake(); + ['instance' => $instance] = restartableInstance(); + + $user = User::factory()->create(['email_verified_at' => now()]); + $instance->customer->update(['user_id' => $user->id, 'email' => $user->email]); + + // Customer: the modal only says yes; Cloud::restart() does the work. + Livewire::actingAs($user)->test(ConfirmRestartCloud::class) + ->call('proceed') + ->assertDispatched('cloud-restart-confirmed'); + + Livewire::actingAs($user)->test(Cloud::class) + ->dispatch('cloud-restart-confirmed') + ->assertDispatched('notify'); + + expect(ProvisioningRun::query()->where('pipeline', 'restart')->count())->toBe(1); + + // Operator: same shape, addressed by uuid (R11). + ProvisioningRun::query()->delete(); + + Livewire::actingAs(admin(), 'operator')->test(ConfirmRestartInstance::class, ['uuid' => $instance->uuid]) + ->call('proceed') + ->assertDispatched('instance-restart-confirmed'); + + Livewire::actingAs(admin(), 'operator')->test(AdminInstances::class) + ->dispatch('instance-restart-confirmed', uuid: $instance->uuid); + + expect(ProvisioningRun::query()->where('pipeline', 'restart')->count())->toBe(1); +}); + +it('refuses the console action to an operator who does not hold the capability', function () { + Queue::fake(); + ['instance' => $instance] = restartableInstance(); + + Livewire::actingAs(operator('Read-only'), 'operator')->test(AdminInstances::class) + ->dispatch('instance-restart-confirmed', uuid: $instance->uuid) + ->assertForbidden(); + + expect(ProvisioningRun::query()->where('pipeline', 'restart')->count())->toBe(0); +}); + +it('gives the shutdown step nothing to do when the guest is already stopped', function () { + Queue::fake(); + ['pve' => $pve, 'instance' => $instance, 'order' => $order] = restartableInstance(); + + $pve->runningVmids = []; + + $run = ProvisioningRun::factory()->create([ + 'subject_type' => Order::class, 'subject_id' => $order->id, 'pipeline' => 'restart', + 'context' => ['instance_id' => $instance->id, 'node' => 'pve', 'vmid' => 101], + ]); + + expect(app(ShutDownVirtualMachine::class)->execute($run)->type)->toBe('advance') + ->and($pve->shutdownCalls)->toBe([]); +});