CluPilotCloud/tests/Feature/Provisioning/CustomerStepsTest.php

<?php

use App\Models\Host;
use App\Models\Instance;
use App\Models\Order;
use App\Models\ProvisioningRun;
use App\Models\RunResource;
use App\Notifications\CloudReady;
use App\Provisioning\Steps\Customer\CloneVirtualMachine;
use App\Provisioning\Steps\Customer\CompleteProvisioning;
use App\Provisioning\Steps\Customer\ConfigureCloudInit;
use App\Provisioning\Steps\Customer\ConfigureDnsAndTls;
use App\Provisioning\Steps\Customer\ConfigureNetwork;
use App\Provisioning\Steps\Customer\ConfigureNextcloud;
use App\Provisioning\Steps\Customer\CreateCustomerAdmin;
use App\Provisioning\Steps\Customer\DeployApplicationStack;
use App\Provisioning\Steps\Customer\RegisterBackup;
use App\Provisioning\Steps\Customer\RegisterMonitoring;
use App\Provisioning\Steps\Customer\ReserveResources;
use App\Provisioning\Steps\Customer\RunAcceptanceChecks;
use App\Provisioning\Steps\Customer\StartVirtualMachine;
use App\Provisioning\Steps\Customer\ValidateOrder;
use App\Provisioning\Steps\Customer\WaitForGuestAgent;
use Illuminate\Support\Facades\Crypt;
use Illuminate\Support\Facades\Notification;

/** A run whose order is paid but nothing is reserved yet. */
function orderRun(array $attrs = [], array $context = []): ProvisioningRun
{
    $order = Order::factory()->create($attrs);

    return ProvisioningRun::factory()->create([
        'subject_type' => Order::class,
        'subject_id' => $order->id,
        'pipeline' => 'customer',
        'context' => $context,
    ]);
}

/** A run with a host + reserved instance + full VM context. */
function reservedRun(array $extraContext = [], array $instanceAttrs = []): array
{
    $host = Host::factory()->active()->create(['datacenter' => 'fsn', 'node' => 'pve']);
    $order = Order::factory()->create(['datacenter' => 'fsn', 'plan' => 'start']);
    $instance = Instance::factory()->create(array_merge([
        'order_id' => $order->id, 'customer_id' => $order->customer_id,
        'host_id' => $host->id, 'vmid' => 101, 'status' => 'provisioning',
    ], $instanceAttrs));
    $run = ProvisioningRun::factory()->create([
        'subject_type' => Order::class, 'subject_id' => $order->id, 'pipeline' => 'customer',
        'context' => array_merge([
            'instance_id' => $instance->id, 'host_id' => $host->id, 'node' => 'pve',
            'vmid' => 101, 'subdomain' => $instance->subdomain, 'plan' => 'start',
        ], $extraContext),
    ]);

    return compact('host', 'order', 'instance', 'run');
}

// 1. ValidateOrder
it('validates a paid order and moves it to provisioning', function () {
    $run = orderRun(['status' => 'paid', 'plan' => 'start']);
    expect(app(ValidateOrder::class)->execute($run)->type)->toBe('advance')
        ->and($run->subject->fresh()->status)->toBe('provisioning');
});

it('fails validation for an unpaid order or unknown plan', function () {
    expect(app(ValidateOrder::class)->execute(orderRun(['status' => 'pending']))->type)->toBe('fail');
    expect(app(ValidateOrder::class)->execute(orderRun(['status' => 'paid', 'plan' => 'ghost']))->type)->toBe('fail');
});

// 2. ReserveResources
it('reserves an instance on a datacenter host and records it once', function () {
    fakeServices();
    Host::factory()->active()->create(['datacenter' => 'fsn', 'node' => 'pve', 'total_gb' => 1000]);
    $run = orderRun(['status' => 'provisioning', 'plan' => 'start', 'datacenter' => 'fsn']);

    expect(app(ReserveResources::class)->execute($run)->type)->toBe('advance');
    $run->refresh();
    $instance = Instance::query()->where('order_id', $run->subject_id)->first();
    expect($instance)->not->toBeNull()
        ->and($run->context('instance_id'))->toBe($instance->id)
        ->and($run->context('node'))->toBe('pve');

    // Re-run: idempotent (no second instance, one breadcrumb).
    app(ReserveResources::class)->execute($run->fresh());
    expect(Instance::query()->where('order_id', $run->subject_id)->count())->toBe(1)
        ->and(RunResource::where('run_id', $run->id)->where('kind', 'instance_id')->count())->toBe(1);
});

it('fails reservation when no host has capacity', function () {
    $run = orderRun(['status' => 'provisioning', 'plan' => 'start', 'datacenter' => 'nowhere']);
    expect(app(ReserveResources::class)->execute($run)->type)->toBe('fail');
});

// 3. CloneVirtualMachine
it('clones the template, persisting vmid before polling, once', function () {
    $s = fakeServices();
    ['run' => $run, 'instance' => $instance] = reservedRun([], ['vmid' => null]);

    expect(app(CloneVirtualMachine::class)->execute($run)->type)->toBe('advance');
    $run->refresh();
    expect($run->context('vmid'))->not->toBeNull()
        ->and($instance->fresh()->vmid)->toBe($run->context('vmid'))
        ->and(RunResource::where('run_id', $run->id)->where('kind', 'vmid')->count())->toBe(1);

    // Re-run: idempotent, no second clone.
    app(CloneVirtualMachine::class)->execute($run->fresh());
    expect(count($s['pve']->clonedVmids))->toBe(1);
});

it('polls while cloning and fails on a clone error', function () {
    $s = fakeServices();
    $s['pve']->forceTaskStatus = 'running';
    ['run' => $run] = reservedRun([], ['vmid' => null]);
    expect(app(CloneVirtualMachine::class)->execute($run)->type)->toBe('poll');

    $s2 = fakeServices();
    $s2['pve']->taskExitStatus = 'clone failed';
    ['run' => $run2] = reservedRun([], ['vmid' => null]);
    expect(app(CloneVirtualMachine::class)->execute($run2)->type)->toBe('fail');
});

it('recovers a lost-task clone by waiting for the lock then re-cloning cleanly', function () {
    $s = fakeServices();
    ['run' => $run] = reservedRun([], ['vmid' => 101]);
    // Crash after cloneVm() but before clone_upid was persisted: vmid reserved +
    // breadcrumb, no clone_upid, and the VM already exists.
    RunResource::create(['run_id' => $run->id, 'kind' => 'vmid', 'external_id' => '101']);
    $run->mergeContext(['vmid' => 101]);
    $s['pve']->clonedVmids[] = 101;

    // still cloning (locked) → poll, don't touch it
    $s['pve']->vmLock = 'clone';
    expect(app(CloneVirtualMachine::class)->execute($run->fresh())->type)->toBe('poll');

    // unlocked but task status unknown → destroy the maybe-incomplete VM + re-clone
    $s['pve']->vmLock = null;
    expect(app(CloneVirtualMachine::class)->execute($run->fresh())->type)->toBe('retry');
    expect($s['pve']->deletedVmids)->toContain(101)
        ->and(RunResource::where('run_id', $run->id)->where('kind', 'vmid')->exists())->toBeFalse();
});

it('fails clone when the plan has no template', function () {
    fakeServices();
    $host = Host::factory()->active()->create(['node' => 'pve']);
    $order = Order::factory()->create(['plan' => 'start']);
    $instance = Instance::factory()->create(['order_id' => $order->id, 'customer_id' => $order->customer_id, 'host_id' => $host->id]);
    $run = ProvisioningRun::factory()->create([
        'subject_type' => Order::class, 'subject_id' => $order->id, 'pipeline' => 'customer',
        'context' => ['instance_id' => $instance->id, 'node' => 'pve'],
    ]);
    config()->set('provisioning.plans.start.template_vmid', 0);

    expect(app(CloneVirtualMachine::class)->execute($run)->type)->toBe('fail');
});

// 4. ConfigureCloudInit
it('configures cloud-init and resizes the disk', function () {
    $s = fakeServices();
    ['run' => $run] = reservedRun();
    expect(app(ConfigureCloudInit::class)->execute($run)->type)->toBe('advance')
        ->and($s['pve']->cloudInitCalls)->toContain(101)
        ->and($s['pve']->cloudInitParams[101]['cores'])->toBe(2)      // start plan
        ->and($s['pve']->cloudInitParams[101]['memory'])->toBe(4096)
        ->and($s['pve']->resizeCalls)->not->toBeEmpty();
});

// 5. StartVirtualMachine
it('starts the VM and is idempotent when already running', function () {
    $s = fakeServices();
    ['run' => $run] = reservedRun();
    expect(app(StartVirtualMachine::class)->execute($run)->type)->toBe('advance')
        ->and($s['pve']->runningVmids)->toContain(101);

    // already running → advance without another start
    expect(app(StartVirtualMachine::class)->execute($run->fresh())->type)->toBe('advance')
        ->and(count($s['pve']->runningVmids))->toBe(1);
});

// 6. WaitForGuestAgent
it('waits for the guest agent then advances', function () {
    $s = fakeServices();
    $s['pve']->guestAgentUp = false;
    ['run' => $run] = reservedRun();
    expect(app(WaitForGuestAgent::class)->execute($run)->type)->toBe('poll');

    $s['pve']->guestAgentUp = true;
    expect(app(WaitForGuestAgent::class)->execute($run->fresh())->type)->toBe('advance');
});

it('fails when the guest agent never answers before the deadline', function () {
    $s = fakeServices();
    $s['pve']->guestAgentUp = false;
    ['run' => $run] = reservedRun();
    $run->update(['started_at' => now()->subMinutes(10)]);
    expect(app(WaitForGuestAgent::class)->execute($run->fresh())->type)->toBe('fail');
});

// 7. ConfigureNetwork
it('configures the guest network, captures the ip, and firewall', function () {
    $s = fakeServices();
    $s['pve']->guestScript('hostname -I', 0, '10.20.0.7 fe80::1');
    ['run' => $run, 'instance' => $instance] = reservedRun();
    expect(app(ConfigureNetwork::class)->execute($run)->type)->toBe('advance')
        ->and($s['pve']->guestRan('set-hostname'))->toBeTrue()
        ->and($instance->fresh()->guest_ip)->toBe('10.20.0.7')
        ->and($s['pve']->firewallCalls)->toContain('101');
});

it('polls network config until the guest has an address', function () {
    fakeServices(); // hostname -I returns empty by default → no address yet
    ['run' => $run] = reservedRun();
    expect(app(ConfigureNetwork::class)->execute($run)->type)->toBe('poll');
});

// 8. DeployApplicationStack
it('deploys the stack, polling until healthy, with an encrypted reusable db password', function () {
    $s = fakeServices();
    ['run' => $run] = reservedRun();

    // not healthy yet → poll; the stack is deployed and the password is encrypted
    expect(app(DeployApplicationStack::class)->execute($run)->type)->toBe('poll')
        ->and($run->fresh()->context('stack_deployed'))->toBeTrue()
        ->and($s['pve']->guestRan('docker compose up'))->toBeTrue();

    $enc = $run->fresh()->context('db_password');
    expect($enc)->not->toBeNull();
    // the persisted context holds only the ENCRYPTED credential, never plaintext
    $raw = \DB::table('provisioning_runs')->where('id', $run->id)->value('context');
    expect($raw)->not->toContain(Crypt::decryptString($enc));

    // healthy → advance, secret scrubbed
    $s['pve']->guestScript('status.php', 0, 'ok');
    expect(app(DeployApplicationStack::class)->execute($run->fresh())->type)->toBe('advance')
        ->and($run->fresh()->context('db_password'))->toBeNull();
});

// 9. ConfigureNextcloud
it('configures nextcloud via occ', function () {
    $s = fakeServices();
    ['run' => $run] = reservedRun();
    expect(app(ConfigureNextcloud::class)->execute($run)->type)->toBe('advance')
        ->and($s['pve']->guestRan('occ config:system:set trusted_domains'))->toBeTrue();
});

// 10. CreateCustomerAdmin
it('creates the admin once, never persisting the password in plaintext', function () {
    $s = fakeServices();
    $s['pve']->guestScript('user:info', 1); // user does not exist yet → user:add
    ['run' => $run, 'instance' => $instance] = reservedRun();

    expect(app(CreateCustomerAdmin::class)->execute($run)->type)->toBe('advance');
    $run->refresh();
    expect($s['pve']->guestRan('occ user:add'))->toBeTrue()
        ->and($instance->fresh()->nc_admin_ref)->toBe('admin')
        ->and($run->context('admin_password'))->not->toBeNull()          // present but encrypted
        ->and(Crypt::decryptString($run->context('admin_password')))->toBeString()
        ->and(RunResource::where('run_id', $run->id)->where('kind', 'nc_admin')->count())->toBe(1);

    // raw context in DB must not contain the decrypted password
    $rawContext = \DB::table('provisioning_runs')->where('id', $run->id)->value('context');
    expect($rawContext)->not->toContain(Crypt::decryptString($run->context('admin_password')));

    // idempotent
    $guestCountBefore = count($s['pve']->guestCommands);
    app(CreateCustomerAdmin::class)->execute($run->fresh());
    expect(count($s['pve']->guestCommands))->toBe($guestCountBefore);
});

it('resets the password when the admin user already exists (retry-safe)', function () {
    $s = fakeServices();
    $s['pve']->guestScript('user:info', 0); // user already exists → resetpassword
    ['run' => $run] = reservedRun();

    expect(app(CreateCustomerAdmin::class)->execute($run)->type)->toBe('advance')
        ->and($s['pve']->guestRan('occ user:resetpassword'))->toBeTrue()
        ->and($s['pve']->guestRan('occ user:add'))->toBeFalse();
});

// 11. ConfigureDnsAndTls
it('creates the dns record, writes the route, and polls for the cert', function () {
    $s = fakeServices();
    $s['traefik']->certReady = false;
    ['run' => $run, 'instance' => $instance] = reservedRun([], ['guest_ip' => '10.20.0.7']);

    expect(app(ConfigureDnsAndTls::class)->execute($run)->type)->toBe('poll');
    $run->refresh();
    $instance->refresh();
    expect(RunResource::where('run_id', $run->id)->where('kind', 'dns_record_id')->count())->toBe(1)
        ->and($instance->dnsRecords()->count())->toBe(1)
        ->and($instance->route_written)->toBeTrue()
        ->and($s['traefik']->routes[$instance->subdomain])->toBe('10.20.0.7');

    // cert now reachable → advance, no duplicate dns record
    $s['traefik']->certReady = true;
    expect(app(ConfigureDnsAndTls::class)->execute($run->fresh())->type)->toBe('advance')
        ->and($instance->fresh()->cert_ok)->toBeTrue()
        ->and(RunResource::where('run_id', $run->id)->where('kind', 'dns_record_id')->count())->toBe(1);
});

// 12/13. Backup + Monitoring
it('registers a backup job and a monitoring target once each', function () {
    fakeServices();
    ['run' => $run, 'instance' => $instance] = reservedRun();

    expect(app(RegisterBackup::class)->execute($run)->type)->toBe('advance');
    app(RegisterBackup::class)->execute($run->fresh());
    expect($instance->fresh()->backups()->count())->toBe(1)
        ->and(RunResource::where('run_id', $run->id)->where('kind', 'backup_job_id')->count())->toBe(1);

    expect(app(RegisterMonitoring::class)->execute($run->fresh())->type)->toBe('advance');
    app(RegisterMonitoring::class)->execute($run->fresh());
    expect($instance->fresh()->monitoringTargets()->count())->toBe(1)
        ->and(RunResource::where('run_id', $run->id)->where('kind', 'monitoring_target_id')->count())->toBe(1);
});

// 14. RunAcceptanceChecks
it('passes acceptance only when everything is genuinely green', function () {
    $s = fakeServices();
    $s['pve']->guestScript('occ status', 0, '{"installed":true,"maintenance":false}'); // healthy Nextcloud
    // occ user:info → default exit 0 (admin usable)
    ['run' => $run, 'instance' => $instance, 'host' => $host] = reservedRun([], ['cert_ok' => true]);
    RunResource::create(['run_id' => $run->id, 'host_id' => $host->id, 'kind' => 'backup_job_id', 'external_id' => 'b']);
    $instance->monitoringTargets()->create(['external_id' => 'mon-1', 'url' => 'https://x/status.php', 'status' => 'up']);

    expect(app(RunAcceptanceChecks::class)->execute($run)->type)->toBe('advance');

    // monitoring unhealthy → fail
    $s['monitoring']->healthy = false;
    expect(app(RunAcceptanceChecks::class)->execute($run->fresh())->type)->toBe('fail');
    $s['monitoring']->healthy = true;

    // Nextcloud not installed → fail
    $s['pve']->guestScript('occ status', 0, '{"installed":false}');
    expect(app(RunAcceptanceChecks::class)->execute($run->fresh())->type)->toBe('fail');

    // cert not reachable → fail
    $s['traefik']->certReady = false;
    expect(app(RunAcceptanceChecks::class)->execute($run->fresh())->type)->toBe('fail');
});

// 15. CompleteProvisioning
it('completes: activates instance+order, seeds onboarding, delivers credentials', function () {
    Notification::fake();
    fakeServices();
    ['run' => $run, 'instance' => $instance, 'order' => $order] = reservedRun([
        'admin_password' => Crypt::encryptString('s3cret-pw'),
    ]);

    expect(app(CompleteProvisioning::class)->execute($run)->type)->toBe('advance');
    expect($instance->fresh()->status)->toBe('active')
        ->and($order->fresh()->status)->toBe('active')
        ->and($instance->fresh()->onboardingTasks()->count())->toBeGreaterThan(0)
        ->and($run->fresh()->context('admin_password'))->toBeNull();

    // Re-running must NOT deliver the credentials a second time.
    app(CompleteProvisioning::class)->execute($run->fresh());
    Notification::assertSentOnDemandTimes(CloudReady::class, 1);
});