withSubscription()->create($attrs); return ProvisioningRun::factory()->create([ 'subject_type' => Order::class, 'subject_id' => $order->id, 'pipeline' => 'customer', 'context' => $context, ]); } /** A run with a host + reserved instance + full VM context. */ function reservedRun(array $extraContext = [], array $instanceAttrs = []): array { $host = Host::factory()->active()->create(['datacenter' => 'fsn', 'node' => 'pve']); $order = Order::factory()->withSubscription()->create(['datacenter' => 'fsn', 'plan' => 'start']); $instance = Instance::factory()->create(array_merge([ 'order_id' => $order->id, 'customer_id' => $order->customer_id, 'host_id' => $host->id, 'vmid' => 101, 'status' => 'provisioning', ], $instanceAttrs)); $run = ProvisioningRun::factory()->create([ 'subject_type' => Order::class, 'subject_id' => $order->id, 'pipeline' => 'customer', 'context' => array_merge([ 'instance_id' => $instance->id, 'host_id' => $host->id, 'node' => 'pve', 'vmid' => 101, 'subdomain' => $instance->subdomain, 'plan' => 'start', ], $extraContext), ]); return compact('host', 'order', 'instance', 'run'); } // 1. ValidateOrder it('validates a paid order and moves it to provisioning', function () { $run = orderRun(['status' => 'paid', 'plan' => 'start']); expect(app(ValidateOrder::class)->execute($run)->type)->toBe('advance') ->and($run->subject->fresh()->status)->toBe('provisioning'); }); it('fails validation for an unpaid order, and for a paid one with no contract', function () { // Named for what the step actually checks. There is no plan check in // ValidateOrder at all — an unknown plan fails because OpenSubscription could // not price it and so froze no contract, which is a different sentence and the // one that matters: nothing is built from today's catalogue. Asserting the // reason is what keeps the two apart; on ->type alone this passed either way. $unpaid = app(ValidateOrder::class)->execute(orderRun(['status' => 'pending'])); expect($unpaid->type)->toBe('fail') ->and($unpaid->reason)->toBe('order_not_paid'); $ghost = app(ValidateOrder::class)->execute(orderRun(['status' => 'paid', 'plan' => 'ghost'])); expect($ghost->type)->toBe('fail') ->and($ghost->reason)->toBe('no_subscription'); }); // 2. ReserveResources it('reserves an instance on a datacenter host and records it once', function () { fakeServices(); Host::factory()->active()->create(['datacenter' => 'fsn', 'node' => 'pve', 'total_gb' => 1000]); $run = orderRun(['status' => 'provisioning', 'plan' => 'start', 'datacenter' => 'fsn']); expect(app(ReserveResources::class)->execute($run)->type)->toBe('advance'); $run->refresh(); $instance = Instance::query()->where('order_id', $run->subject_id)->first(); expect($instance)->not->toBeNull() ->and($run->context('instance_id'))->toBe($instance->id) ->and($run->context('node'))->toBe('pve'); // Re-run: idempotent (no second instance, one breadcrumb). app(ReserveResources::class)->execute($run->fresh()); expect(Instance::query()->where('order_id', $run->subject_id)->count())->toBe(1) ->and(RunResource::where('run_id', $run->id)->where('kind', 'instance_id')->count())->toBe(1); }); it('parks reservation when no host has capacity, rather than failing a paid order', function () { // This used to fail the run. The customer has paid by the time it runs, and // "no host has room" is a purchasing decision, not a fault of theirs — // failing it turned a sale into an incident and left somebody holding money // against nothing. It waits for a machine now and only fails after // PARK_DAYS. The queue and the deadline are covered in // tests/Feature/Admin/CapacityQueueTest.php. $run = orderRun(['status' => 'provisioning', 'plan' => 'start', 'datacenter' => 'nowhere']); $result = app(ReserveResources::class)->execute($run); expect($result->type)->toBe('poll') ->and($result->reason)->toBe('awaiting_capacity'); }); // 3. CloneVirtualMachine it('clones the template, persisting vmid before polling, once', function () { $s = fakeServices(); ['run' => $run, 'instance' => $instance] = reservedRun([], ['vmid' => null]); expect(app(CloneVirtualMachine::class)->execute($run)->type)->toBe('advance'); $run->refresh(); expect($run->context('vmid'))->not->toBeNull() ->and($instance->fresh()->vmid)->toBe($run->context('vmid')) ->and(RunResource::where('run_id', $run->id)->where('kind', 'vmid')->count())->toBe(1); // Re-run: idempotent, no second clone. app(CloneVirtualMachine::class)->execute($run->fresh()); expect(count($s['pve']->clonedVmids))->toBe(1); }); it('polls while cloning and fails on a clone error', function () { $s = fakeServices(); $s['pve']->forceTaskStatus = 'running'; ['run' => $run] = reservedRun([], ['vmid' => null]); expect(app(CloneVirtualMachine::class)->execute($run)->type)->toBe('poll'); $s2 = fakeServices(); $s2['pve']->taskExitStatus = 'clone failed'; ['run' => $run2] = reservedRun([], ['vmid' => null]); expect(app(CloneVirtualMachine::class)->execute($run2)->type)->toBe('fail'); }); it('recovers a lost-task clone by waiting for the lock then re-cloning cleanly', function () { $s = fakeServices(); ['run' => $run] = reservedRun([], ['vmid' => 101]); // Crash after cloneVm() but before clone_upid was persisted: vmid reserved + // breadcrumb, no clone_upid, and the VM already exists. RunResource::create(['run_id' => $run->id, 'kind' => 'vmid', 'external_id' => '101']); $run->mergeContext(['vmid' => 101]); $s['pve']->clonedVmids[] = 101; // still cloning (locked) → poll, don't touch it $s['pve']->vmLock = 'clone'; expect(app(CloneVirtualMachine::class)->execute($run->fresh())->type)->toBe('poll'); // unlocked but task status unknown → destroy the maybe-incomplete VM + re-clone $s['pve']->vmLock = null; expect(app(CloneVirtualMachine::class)->execute($run->fresh())->type)->toBe('retry'); expect($s['pve']->deletedVmids)->toContain(101) ->and(RunResource::where('run_id', $run->id)->where('kind', 'vmid')->exists())->toBeFalse(); }); it('fails clone when the contract carries no template', function () { // This used to write config('provisioning.plans.start.template_vmid') — a key // that has not existed since the catalogue moved into the database, so the // write was a no-op and the run failed for want of a CONTRACT instead. The // case the test is named for was never exercised. // // The template comes off the frozen snapshot, so the state to build is a // contract with no blueprint on it: what a pre-catalogue subscription looks // like before the backfill fills it in. PlanCatalogue::publish() has refused // to put a version on sale without one ever since, and this is the failure // that refusal exists to keep away from a paying customer. fakeServices(); $host = Host::factory()->active()->create(['node' => 'pve']); $order = Order::factory()->create(['plan' => 'start']); $instance = Instance::factory()->create(['order_id' => $order->id, 'customer_id' => $order->customer_id, 'host_id' => $host->id]); Subscription::factory()->create([ 'customer_id' => $order->customer_id, 'order_id' => $order->id, 'instance_id' => $instance->id, 'template_vmid' => null, ]); $run = ProvisioningRun::factory()->create([ 'subject_type' => Order::class, 'subject_id' => $order->id, 'pipeline' => 'customer', 'context' => ['instance_id' => $instance->id, 'node' => 'pve'], ]); $result = app(CloneVirtualMachine::class)->execute($run); expect($result->type)->toBe('fail') ->and($result->reason)->toBe('template_missing') // It is the template that is missing and not the contract — otherwise this // is the previous test over again under a misleading name. ->and($order->subscription()->exists())->toBeTrue(); }); // 4. ConfigureCloudInit it('configures cloud-init and resizes the disk', function () { $s = fakeServices(); ['run' => $run] = reservedRun(); expect(app(ConfigureCloudInit::class)->execute($run)->type)->toBe('advance') ->and($s['pve']->cloudInitCalls)->toContain(101) ->and($s['pve']->cloudInitParams[101]['cores'])->toBe(2) // start plan ->and($s['pve']->cloudInitParams[101]['memory'])->toBe(4096) ->and($s['pve']->resizeCalls)->not->toBeEmpty(); }); // 5. StartVirtualMachine it('starts the VM and is idempotent when already running', function () { $s = fakeServices(); ['run' => $run] = reservedRun(); expect(app(StartVirtualMachine::class)->execute($run)->type)->toBe('advance') ->and($s['pve']->runningVmids)->toContain(101); // already running → advance without another start expect(app(StartVirtualMachine::class)->execute($run->fresh())->type)->toBe('advance') ->and(count($s['pve']->runningVmids))->toBe(1); }); // 6. WaitForGuestAgent it('waits for the guest agent then advances', function () { $s = fakeServices(); $s['pve']->guestAgentUp = false; ['run' => $run] = reservedRun(); expect(app(WaitForGuestAgent::class)->execute($run)->type)->toBe('poll'); $s['pve']->guestAgentUp = true; expect(app(WaitForGuestAgent::class)->execute($run->fresh())->type)->toBe('advance'); }); it('fails when the guest agent never answers before the deadline', function () { $s = fakeServices(); $s['pve']->guestAgentUp = false; ['run' => $run] = reservedRun(); $run->update(['started_at' => now()->subMinutes(10)]); expect(app(WaitForGuestAgent::class)->execute($run->fresh())->type)->toBe('fail'); }); // 7. ConfigureNetwork it('configures the guest network, captures the ip, and firewall', function () { $s = fakeServices(); $s['pve']->guestScript('hostname -I', 0, '10.20.0.7 fe80::1'); ['run' => $run, 'instance' => $instance] = reservedRun(); expect(app(ConfigureNetwork::class)->execute($run)->type)->toBe('advance') ->and($s['pve']->guestRan('set-hostname'))->toBeTrue() ->and($instance->fresh()->guest_ip)->toBe('10.20.0.7') ->and($s['pve']->firewallCalls)->toContain('101'); }); it('polls network config until the guest has an address', function () { fakeServices(); // hostname -I returns empty by default → no address yet ['run' => $run] = reservedRun(); expect(app(ConfigureNetwork::class)->execute($run)->type)->toBe('poll'); }); // 8. DeployApplicationStack it('deploys the stack, polling until healthy, with an encrypted reusable db password', function () { $s = fakeServices(); ['run' => $run] = reservedRun(); // not healthy yet → poll; the stack is deployed and the password is encrypted expect(app(DeployApplicationStack::class)->execute($run)->type)->toBe('poll') ->and($run->fresh()->context('stack_deployed'))->toBeTrue() ->and($s['pve']->guestRan('docker compose up'))->toBeTrue(); $enc = $run->fresh()->context('db_password'); expect($enc)->not->toBeNull(); // the persisted context holds only the ENCRYPTED credential, never plaintext $raw = \DB::table('provisioning_runs')->where('id', $run->id)->value('context'); expect($raw)->not->toContain(Crypt::decryptString($enc)); // healthy → advance, secret scrubbed $s['pve']->guestScript('status.php', 0, 'ok'); expect(app(DeployApplicationStack::class)->execute($run->fresh())->type)->toBe('advance') ->and($run->fresh()->context('db_password'))->toBeNull(); }); // 9. ConfigureNextcloud it('configures nextcloud via occ', function () { $s = fakeServices(); ['run' => $run] = reservedRun(); expect(app(ConfigureNextcloud::class)->execute($run)->type)->toBe('advance') ->and($s['pve']->guestRan('occ config:system:set trusted_domains'))->toBeTrue(); }); // 10. CreateCustomerAdmin it('creates the admin once, never persisting the password in plaintext', function () { $s = fakeServices(); $s['pve']->guestScript('user:info', 1); // user does not exist yet → user:add ['run' => $run, 'instance' => $instance] = reservedRun(); expect(app(CreateCustomerAdmin::class)->execute($run)->type)->toBe('advance'); $run->refresh(); expect($s['pve']->guestRan('occ user:add'))->toBeTrue() ->and($instance->fresh()->nc_admin_ref)->toBe('admin') ->and($run->context('admin_password'))->not->toBeNull() // present but encrypted ->and(Crypt::decryptString($run->context('admin_password')))->toBeString() ->and(RunResource::where('run_id', $run->id)->where('kind', 'nc_admin')->count())->toBe(1); // raw context in DB must not contain the decrypted password $rawContext = \DB::table('provisioning_runs')->where('id', $run->id)->value('context'); expect($rawContext)->not->toContain(Crypt::decryptString($run->context('admin_password'))); // idempotent $guestCountBefore = count($s['pve']->guestCommands); app(CreateCustomerAdmin::class)->execute($run->fresh()); expect(count($s['pve']->guestCommands))->toBe($guestCountBefore); }); it('resets the password when the admin user already exists (retry-safe)', function () { $s = fakeServices(); $s['pve']->guestScript('user:info', 0); // user already exists → resetpassword ['run' => $run] = reservedRun(); expect(app(CreateCustomerAdmin::class)->execute($run)->type)->toBe('advance') ->and($s['pve']->guestRan('occ user:resetpassword'))->toBeTrue() ->and($s['pve']->guestRan('occ user:add'))->toBeFalse(); }); // 11. ConfigureDnsAndTls it('creates the dns record, writes the route, and polls for the cert', function () { $s = fakeServices(); $s['traefik']->certReady = false; ['run' => $run, 'instance' => $instance] = reservedRun([], ['guest_ip' => '10.20.0.7']); expect(app(ConfigureDnsAndTls::class)->execute($run)->type)->toBe('poll'); $run->refresh(); $instance->refresh(); expect(RunResource::where('run_id', $run->id)->where('kind', 'dns_record_id')->count())->toBe(1) ->and($instance->dnsRecords()->count())->toBe(1) ->and($instance->route_written)->toBeTrue() ->and($s['traefik']->routes[$instance->subdomain])->toBe('10.20.0.7'); // cert now reachable → advance, no duplicate dns record $s['traefik']->certReady = true; expect(app(ConfigureDnsAndTls::class)->execute($run->fresh())->type)->toBe('advance') ->and($instance->fresh()->cert_ok)->toBeTrue() ->and(RunResource::where('run_id', $run->id)->where('kind', 'dns_record_id')->count())->toBe(1); }); // 12/13. Backup + Monitoring it('registers a backup job and a monitoring target once each', function () { fakeServices(); ['run' => $run, 'instance' => $instance] = reservedRun(); expect(app(RegisterBackup::class)->execute($run)->type)->toBe('advance'); app(RegisterBackup::class)->execute($run->fresh()); expect($instance->fresh()->backups()->count())->toBe(1) ->and(RunResource::where('run_id', $run->id)->where('kind', 'backup_job_id')->count())->toBe(1); expect(app(RegisterMonitoring::class)->execute($run->fresh())->type)->toBe('advance'); app(RegisterMonitoring::class)->execute($run->fresh()); expect($instance->fresh()->monitoringTargets()->count())->toBe(1) ->and(RunResource::where('run_id', $run->id)->where('kind', 'monitoring_target_id')->count())->toBe(1); }); // 14. RunAcceptanceChecks it('passes acceptance only when everything is genuinely green', function () { $s = fakeServices(); $s['pve']->guestScript('occ status', 0, '{"installed":true,"maintenance":false}'); // healthy Nextcloud // occ user:info → default exit 0 (admin usable) ['run' => $run, 'instance' => $instance, 'host' => $host] = reservedRun([], ['cert_ok' => true]); RunResource::create(['run_id' => $run->id, 'host_id' => $host->id, 'kind' => 'backup_job_id', 'external_id' => 'b']); $instance->monitoringTargets()->create(['external_id' => 'mon-1', 'url' => 'https://x/status.php', 'status' => 'up']); expect(app(RunAcceptanceChecks::class)->execute($run)->type)->toBe('advance'); // Monitoring is a secondary signal: not green → still deliver, but record it. $s['monitoring']->healthy = false; expect(app(RunAcceptanceChecks::class)->execute($run->fresh())->type)->toBe('advance') ->and($run->events()->where('outcome', 'info')->where('step', 'run_acceptance_checks')->exists())->toBeTrue(); // ...unless the operator declared monitoring required. config()->set('provisioning.monitoring.required', true); expect(app(RunAcceptanceChecks::class)->execute($run->fresh())->type)->toBe('fail'); config()->set('provisioning.monitoring.required', false); $s['monitoring']->healthy = true; // A probe that says no asks again rather than condemning a finished cloud: // certReachable() returns the same false for a connect timeout as for a // missing certificate, and this step returning fail() bypassed the retry // budget entirely — one bad second ended a working Nextcloud as a failed // order with the instance released. $s['pve']->guestScript('occ status', 0, '{"installed":false}'); $nextcloud = app(RunAcceptanceChecks::class)->execute($run->fresh()); expect($nextcloud->type)->toBe('retry') ->and($nextcloud->reason)->toBe('acceptance_failed:nextcloud'); $s['traefik']->certReady = false; $cert = app(RunAcceptanceChecks::class)->execute($run->fresh()); expect($cert->type)->toBe('retry') ->and($cert->reason)->toBe('acceptance_failed:cert'); }); it('still fails for good when what is wrong is a fact no retry can change', function () { // The two checks that stay terminal read state THIS run wrote earlier, out of // our own database. Asking again five times only delays saying so. $s = fakeServices(); $s['pve']->guestScript('occ status', 0, '{"installed":true,"maintenance":false}'); ['run' => $run, 'instance' => $instance, 'host' => $host] = reservedRun([], ['cert_ok' => true]); $instance->monitoringTargets()->create(['external_id' => 'mon-1', 'url' => 'https://x/status.php', 'status' => 'up']); // No backup breadcrumb: RegisterBackup never registered a job. $missingBackup = app(RunAcceptanceChecks::class)->execute($run->fresh()); expect($missingBackup->type)->toBe('fail') ->and($missingBackup->reason)->toBe('acceptance_failed:backup'); RunResource::create(['run_id' => $run->id, 'host_id' => $host->id, 'kind' => 'backup_job_id', 'external_id' => 'b']); // No platform certificate — which ConfigureDnsAndTls fails the run over // itself, so standing here without it is a state the pipeline cannot reach. $instance->update(['cert_ok' => false]); $missingCert = app(RunAcceptanceChecks::class)->execute($run->fresh()); expect($missingCert->type)->toBe('fail') ->and($missingCert->reason)->toBe('acceptance_failed:cert'); }); it('spends the retry budget and then fails with the reason the probe gave', function () { // The whole point of retrying instead of failing: five noes in a row is not a // blip, and the run still ends failed — named for what was wrong, not // 'step timed out'. $s = fakeServices(); $s['pve']->guestScript('occ status', 0, '{"installed":false}'); ['run' => $run, 'instance' => $instance, 'host' => $host] = reservedRun([], ['cert_ok' => true]); RunResource::create(['run_id' => $run->id, 'host_id' => $host->id, 'kind' => 'backup_job_id', 'external_id' => 'b']); $instance->monitoringTargets()->create(['external_id' => 'mon-1', 'url' => 'https://x/status.php', 'status' => 'up']); $index = array_search(RunAcceptanceChecks::class, config('provisioning.pipelines.customer'), true); $run->update(['current_step' => $index, 'status' => 'running', 'max_attempts' => 3]); $runner = app(RunRunner::class); for ($i = 0; $i < 5; $i++) { // Re-read before writing. Saving the stale model would put the attempt // count back to what it was and the loop would never reach the budget. $run = $run->fresh(); if ($run->status === 'failed') { break; } $run->forceFill(['next_attempt_at' => null, 'started_at' => now()])->save(); $runner->advance($run); } expect($run->fresh()->status)->toBe('failed') ->and($run->fresh()->error)->toBe('acceptance_failed:nextcloud'); }); // 15. CompleteProvisioning it('completes: activates instance+order, seeds onboarding, delivers credentials', function () { Notification::fake(); fakeServices(); ['run' => $run, 'instance' => $instance, 'order' => $order] = reservedRun([ 'admin_password' => Crypt::encryptString('s3cret-pw'), ]); expect(app(CompleteProvisioning::class)->execute($run)->type)->toBe('advance'); expect($instance->fresh()->status)->toBe('active') ->and($order->fresh()->status)->toBe('active') ->and($instance->fresh()->onboardingTasks()->count())->toBeGreaterThan(0) ->and($run->fresh()->context('admin_password'))->toBeNull() // The password moved onto the instance rather than the mail — held // until the customer acknowledges it on the dashboard. ->and($instance->fresh()->admin_password)->toBe('s3cret-pw') ->and($instance->fresh()->credentials_acknowledged_at)->toBeNull(); // Re-running must NOT deliver the credentials a second time, and must not // disturb the password already stored on the instance. app(CompleteProvisioning::class)->execute($run->fresh()); Notification::assertSentOnDemandTimes(CloudReady::class, 1); expect($instance->fresh()->admin_password)->toBe('s3cret-pw'); }); it('never puts the admin password in the CloudReady notification it sends', function () { Notification::fake(); fakeServices(); ['run' => $run] = reservedRun([ 'admin_password' => Crypt::encryptString('s3cret-pw'), ]); app(CompleteProvisioning::class)->execute($run); Notification::assertSentOnDemand(CloudReady::class, function (CloudReady $notification) { // Nothing about the password is even given to the notification — the // constructor only takes the instance now — so there is nothing to // render into the mail body or serialize into a queued job payload. $mail = $notification->toMail(new stdClass); $rendered = collect($mail->introLines)->implode(' '); expect($rendered)->not->toContain('s3cret-pw'); return true; }); }); // 13b. Monitoring resilience — observability must not block delivery. it('keeps provisioning when monitoring is unreachable (retry, then degrade)', function () { $s = fakeServices(); $s['monitoring']->failWith = 'connection refused'; ['run' => $run, 'instance' => $instance] = reservedRun(); // MONITORING_ATTEMPTS=2 means exactly two tries: attempt 0 retries… config()->set('provisioning.monitoring.attempts', 2); $first = app(RegisterMonitoring::class)->execute($run->fresh()); expect($first->type)->toBe('retry')->and($first->reason)->toContain('connection refused'); // …and the SECOND try (attempt 1) is the last: continue DEGRADED with a // visible event instead of failing a paid customer's provisioning. $run->update(['attempt' => 1]); expect(app(RegisterMonitoring::class)->execute($run->fresh())->type)->toBe('advance'); expect($instance->fresh()->monitoringTargets()->count())->toBe(0) ->and($run->events()->where('outcome', 'info')->where('step', 'register_monitoring')->exists())->toBeTrue(); }); it('never lets monitoring retries exhaust the run budget', function () { // A careless MONITORING_ATTEMPTS must not burn through max_attempts and fail // the run — the cap degrades on the last permitted attempt instead. config()->set('provisioning.monitoring.attempts', 999); $s = fakeServices(); $s['monitoring']->failWith = 'connection refused'; ['run' => $run] = reservedRun(); $run->update(['attempt' => $run->max_attempts - 1]); // last attempt in the budget expect(app(RegisterMonitoring::class)->execute($run->fresh())->type)->toBe('advance'); }); it('fails the run on a monitoring outage when monitoring is declared required', function () { config()->set('provisioning.monitoring.required', true); $s = fakeServices(); $s['monitoring']->failWith = 'connection refused'; ['run' => $run] = reservedRun(); expect(fn () => app(RegisterMonitoring::class)->execute($run->fresh())) ->toThrow(RuntimeException::class); });