create($attrs); return ProvisioningRun::factory()->create([ 'subject_type' => Order::class, 'subject_id' => $order->id, 'pipeline' => 'customer', 'context' => $context, ]); } /** A run with a host + reserved instance + full VM context. */ function reservedRun(array $extraContext = [], array $instanceAttrs = []): array { $host = Host::factory()->active()->create(['datacenter' => 'fsn', 'node' => 'pve']); $order = Order::factory()->create(['datacenter' => 'fsn', 'plan' => 'start']); $instance = Instance::factory()->create(array_merge([ 'order_id' => $order->id, 'customer_id' => $order->customer_id, 'host_id' => $host->id, 'vmid' => 101, 'status' => 'provisioning', ], $instanceAttrs)); $run = ProvisioningRun::factory()->create([ 'subject_type' => Order::class, 'subject_id' => $order->id, 'pipeline' => 'customer', 'context' => array_merge([ 'instance_id' => $instance->id, 'host_id' => $host->id, 'node' => 'pve', 'vmid' => 101, 'subdomain' => $instance->subdomain, 'plan' => 'start', ], $extraContext), ]); return compact('host', 'order', 'instance', 'run'); } // 1. ValidateOrder it('validates a paid order and moves it to provisioning', function () { $run = orderRun(['status' => 'paid', 'plan' => 'start']); expect(app(ValidateOrder::class)->execute($run)->type)->toBe('advance') ->and($run->subject->fresh()->status)->toBe('provisioning'); }); it('fails validation for an unpaid order or unknown plan', function () { expect(app(ValidateOrder::class)->execute(orderRun(['status' => 'pending']))->type)->toBe('fail'); expect(app(ValidateOrder::class)->execute(orderRun(['status' => 'paid', 'plan' => 'ghost']))->type)->toBe('fail'); }); // 2. ReserveResources it('reserves an instance on a datacenter host and records it once', function () { fakeServices(); Host::factory()->active()->create(['datacenter' => 'fsn', 'node' => 'pve', 'total_gb' => 1000]); $run = orderRun(['status' => 'provisioning', 'plan' => 'start', 'datacenter' => 'fsn']); expect(app(ReserveResources::class)->execute($run)->type)->toBe('advance'); $run->refresh(); $instance = Instance::query()->where('order_id', $run->subject_id)->first(); expect($instance)->not->toBeNull() ->and($run->context('instance_id'))->toBe($instance->id) ->and($run->context('node'))->toBe('pve'); // Re-run: idempotent (no second instance, one breadcrumb). app(ReserveResources::class)->execute($run->fresh()); expect(Instance::query()->where('order_id', $run->subject_id)->count())->toBe(1) ->and(RunResource::where('run_id', $run->id)->where('kind', 'instance_id')->count())->toBe(1); }); it('fails reservation when no host has capacity', function () { $run = orderRun(['status' => 'provisioning', 'plan' => 'start', 'datacenter' => 'nowhere']); expect(app(ReserveResources::class)->execute($run)->type)->toBe('fail'); }); // 3. CloneVirtualMachine it('clones the template, persisting vmid before polling, once', function () { $s = fakeServices(); ['run' => $run, 'instance' => $instance] = reservedRun([], ['vmid' => null]); expect(app(CloneVirtualMachine::class)->execute($run)->type)->toBe('advance'); $run->refresh(); expect($run->context('vmid'))->not->toBeNull() ->and($instance->fresh()->vmid)->toBe($run->context('vmid')) ->and(RunResource::where('run_id', $run->id)->where('kind', 'vmid')->count())->toBe(1); // Re-run: idempotent, no second clone. app(CloneVirtualMachine::class)->execute($run->fresh()); expect(count($s['pve']->clonedVmids))->toBe(1); }); it('polls while cloning and fails on a clone error', function () { $s = fakeServices(); $s['pve']->forceTaskStatus = 'running'; ['run' => $run] = reservedRun([], ['vmid' => null]); expect(app(CloneVirtualMachine::class)->execute($run)->type)->toBe('poll'); $s2 = fakeServices(); $s2['pve']->taskExitStatus = 'clone failed'; ['run' => $run2] = reservedRun([], ['vmid' => null]); expect(app(CloneVirtualMachine::class)->execute($run2)->type)->toBe('fail'); }); it('recovers a lost-task clone by waiting for the lock then re-cloning cleanly', function () { $s = fakeServices(); ['run' => $run] = reservedRun([], ['vmid' => 101]); // Crash after cloneVm() but before clone_upid was persisted: vmid reserved + // breadcrumb, no clone_upid, and the VM already exists. RunResource::create(['run_id' => $run->id, 'kind' => 'vmid', 'external_id' => '101']); $run->mergeContext(['vmid' => 101]); $s['pve']->clonedVmids[] = 101; // still cloning (locked) → poll, don't touch it $s['pve']->vmLock = 'clone'; expect(app(CloneVirtualMachine::class)->execute($run->fresh())->type)->toBe('poll'); // unlocked but task status unknown → destroy the maybe-incomplete VM + re-clone $s['pve']->vmLock = null; expect(app(CloneVirtualMachine::class)->execute($run->fresh())->type)->toBe('retry'); expect($s['pve']->deletedVmids)->toContain(101) ->and(RunResource::where('run_id', $run->id)->where('kind', 'vmid')->exists())->toBeFalse(); }); it('fails clone when the plan has no template', function () { fakeServices(); $host = Host::factory()->active()->create(['node' => 'pve']); $order = Order::factory()->create(['plan' => 'start']); $instance = Instance::factory()->create(['order_id' => $order->id, 'customer_id' => $order->customer_id, 'host_id' => $host->id]); $run = ProvisioningRun::factory()->create([ 'subject_type' => Order::class, 'subject_id' => $order->id, 'pipeline' => 'customer', 'context' => ['instance_id' => $instance->id, 'node' => 'pve'], ]); config()->set('provisioning.plans.start.template_vmid', 0); expect(app(CloneVirtualMachine::class)->execute($run)->type)->toBe('fail'); }); // 4. ConfigureCloudInit it('configures cloud-init and resizes the disk', function () { $s = fakeServices(); ['run' => $run] = reservedRun(); expect(app(ConfigureCloudInit::class)->execute($run)->type)->toBe('advance') ->and($s['pve']->cloudInitCalls)->toContain(101) ->and($s['pve']->cloudInitParams[101]['cores'])->toBe(2) // start plan ->and($s['pve']->cloudInitParams[101]['memory'])->toBe(4096) ->and($s['pve']->resizeCalls)->not->toBeEmpty(); }); // 5. StartVirtualMachine it('starts the VM and is idempotent when already running', function () { $s = fakeServices(); ['run' => $run] = reservedRun(); expect(app(StartVirtualMachine::class)->execute($run)->type)->toBe('advance') ->and($s['pve']->runningVmids)->toContain(101); // already running → advance without another start expect(app(StartVirtualMachine::class)->execute($run->fresh())->type)->toBe('advance') ->and(count($s['pve']->runningVmids))->toBe(1); }); // 6. WaitForGuestAgent it('waits for the guest agent then advances', function () { $s = fakeServices(); $s['pve']->guestAgentUp = false; ['run' => $run] = reservedRun(); expect(app(WaitForGuestAgent::class)->execute($run)->type)->toBe('poll'); $s['pve']->guestAgentUp = true; expect(app(WaitForGuestAgent::class)->execute($run->fresh())->type)->toBe('advance'); }); it('fails when the guest agent never answers before the deadline', function () { $s = fakeServices(); $s['pve']->guestAgentUp = false; ['run' => $run] = reservedRun(); $run->update(['started_at' => now()->subMinutes(10)]); expect(app(WaitForGuestAgent::class)->execute($run->fresh())->type)->toBe('fail'); }); // 7. ConfigureNetwork it('configures the guest network, captures the ip, and firewall', function () { $s = fakeServices(); $s['pve']->guestScript('hostname -I', 0, '10.20.0.7 fe80::1'); ['run' => $run, 'instance' => $instance] = reservedRun(); expect(app(ConfigureNetwork::class)->execute($run)->type)->toBe('advance') ->and($s['pve']->guestRan('set-hostname'))->toBeTrue() ->and($instance->fresh()->guest_ip)->toBe('10.20.0.7') ->and($s['pve']->firewallCalls)->toContain('101'); }); it('polls network config until the guest has an address', function () { fakeServices(); // hostname -I returns empty by default → no address yet ['run' => $run] = reservedRun(); expect(app(ConfigureNetwork::class)->execute($run)->type)->toBe('poll'); }); // 8. DeployApplicationStack it('deploys the stack, polling until healthy, with an encrypted reusable db password', function () { $s = fakeServices(); ['run' => $run] = reservedRun(); // not healthy yet → poll; the stack is deployed and the password is encrypted expect(app(DeployApplicationStack::class)->execute($run)->type)->toBe('poll') ->and($run->fresh()->context('stack_deployed'))->toBeTrue() ->and($s['pve']->guestRan('docker compose up'))->toBeTrue(); $enc = $run->fresh()->context('db_password'); expect($enc)->not->toBeNull(); // the persisted context holds only the ENCRYPTED credential, never plaintext $raw = \DB::table('provisioning_runs')->where('id', $run->id)->value('context'); expect($raw)->not->toContain(Crypt::decryptString($enc)); // healthy → advance, secret scrubbed $s['pve']->guestScript('status.php', 0, 'ok'); expect(app(DeployApplicationStack::class)->execute($run->fresh())->type)->toBe('advance') ->and($run->fresh()->context('db_password'))->toBeNull(); }); // 9. ConfigureNextcloud it('configures nextcloud via occ', function () { $s = fakeServices(); ['run' => $run] = reservedRun(); expect(app(ConfigureNextcloud::class)->execute($run)->type)->toBe('advance') ->and($s['pve']->guestRan('occ config:system:set trusted_domains'))->toBeTrue(); }); // 10. CreateCustomerAdmin it('creates the admin once, never persisting the password in plaintext', function () { $s = fakeServices(); $s['pve']->guestScript('user:info', 1); // user does not exist yet → user:add ['run' => $run, 'instance' => $instance] = reservedRun(); expect(app(CreateCustomerAdmin::class)->execute($run)->type)->toBe('advance'); $run->refresh(); expect($s['pve']->guestRan('occ user:add'))->toBeTrue() ->and($instance->fresh()->nc_admin_ref)->toBe('admin') ->and($run->context('admin_password'))->not->toBeNull() // present but encrypted ->and(Crypt::decryptString($run->context('admin_password')))->toBeString() ->and(RunResource::where('run_id', $run->id)->where('kind', 'nc_admin')->count())->toBe(1); // raw context in DB must not contain the decrypted password $rawContext = \DB::table('provisioning_runs')->where('id', $run->id)->value('context'); expect($rawContext)->not->toContain(Crypt::decryptString($run->context('admin_password'))); // idempotent $guestCountBefore = count($s['pve']->guestCommands); app(CreateCustomerAdmin::class)->execute($run->fresh()); expect(count($s['pve']->guestCommands))->toBe($guestCountBefore); }); it('resets the password when the admin user already exists (retry-safe)', function () { $s = fakeServices(); $s['pve']->guestScript('user:info', 0); // user already exists → resetpassword ['run' => $run] = reservedRun(); expect(app(CreateCustomerAdmin::class)->execute($run)->type)->toBe('advance') ->and($s['pve']->guestRan('occ user:resetpassword'))->toBeTrue() ->and($s['pve']->guestRan('occ user:add'))->toBeFalse(); }); // 11. ConfigureDnsAndTls it('creates the dns record, writes the route, and polls for the cert', function () { $s = fakeServices(); $s['traefik']->certReady = false; ['run' => $run, 'instance' => $instance] = reservedRun([], ['guest_ip' => '10.20.0.7']); expect(app(ConfigureDnsAndTls::class)->execute($run)->type)->toBe('poll'); $run->refresh(); $instance->refresh(); expect(RunResource::where('run_id', $run->id)->where('kind', 'dns_record_id')->count())->toBe(1) ->and($instance->dnsRecords()->count())->toBe(1) ->and($instance->route_written)->toBeTrue() ->and($s['traefik']->routes[$instance->subdomain])->toBe('10.20.0.7'); // cert now reachable → advance, no duplicate dns record $s['traefik']->certReady = true; expect(app(ConfigureDnsAndTls::class)->execute($run->fresh())->type)->toBe('advance') ->and($instance->fresh()->cert_ok)->toBeTrue() ->and(RunResource::where('run_id', $run->id)->where('kind', 'dns_record_id')->count())->toBe(1); }); // 12/13. Backup + Monitoring it('registers a backup job and a monitoring target once each', function () { fakeServices(); ['run' => $run, 'instance' => $instance] = reservedRun(); expect(app(RegisterBackup::class)->execute($run)->type)->toBe('advance'); app(RegisterBackup::class)->execute($run->fresh()); expect($instance->fresh()->backups()->count())->toBe(1) ->and(RunResource::where('run_id', $run->id)->where('kind', 'backup_job_id')->count())->toBe(1); expect(app(RegisterMonitoring::class)->execute($run->fresh())->type)->toBe('advance'); app(RegisterMonitoring::class)->execute($run->fresh()); expect($instance->fresh()->monitoringTargets()->count())->toBe(1) ->and(RunResource::where('run_id', $run->id)->where('kind', 'monitoring_target_id')->count())->toBe(1); }); // 14. RunAcceptanceChecks it('passes acceptance only when everything is genuinely green', function () { $s = fakeServices(); $s['pve']->guestScript('occ status', 0, '{"installed":true,"maintenance":false}'); // healthy Nextcloud // occ user:info → default exit 0 (admin usable) ['run' => $run, 'instance' => $instance, 'host' => $host] = reservedRun([], ['cert_ok' => true]); RunResource::create(['run_id' => $run->id, 'host_id' => $host->id, 'kind' => 'backup_job_id', 'external_id' => 'b']); $instance->monitoringTargets()->create(['external_id' => 'mon-1', 'url' => 'https://x/status.php', 'status' => 'up']); expect(app(RunAcceptanceChecks::class)->execute($run)->type)->toBe('advance'); // monitoring unhealthy → fail $s['monitoring']->healthy = false; expect(app(RunAcceptanceChecks::class)->execute($run->fresh())->type)->toBe('fail'); $s['monitoring']->healthy = true; // Nextcloud not installed → fail $s['pve']->guestScript('occ status', 0, '{"installed":false}'); expect(app(RunAcceptanceChecks::class)->execute($run->fresh())->type)->toBe('fail'); // cert not reachable → fail $s['traefik']->certReady = false; expect(app(RunAcceptanceChecks::class)->execute($run->fresh())->type)->toBe('fail'); }); // 15. CompleteProvisioning it('completes: activates instance+order, seeds onboarding, delivers credentials', function () { Notification::fake(); fakeServices(); ['run' => $run, 'instance' => $instance, 'order' => $order] = reservedRun([ 'admin_password' => Crypt::encryptString('s3cret-pw'), ]); expect(app(CompleteProvisioning::class)->execute($run)->type)->toBe('advance'); expect($instance->fresh()->status)->toBe('active') ->and($order->fresh()->status)->toBe('active') ->and($instance->fresh()->onboardingTasks()->count())->toBeGreaterThan(0) ->and($run->fresh()->context('admin_password'))->toBeNull(); // Re-running must NOT deliver the credentials a second time. app(CompleteProvisioning::class)->execute($run->fresh()); Notification::assertSentOnDemandTimes(CloudReady::class, 1); });