CluPilotCloud/tests/Feature/Provisioning/RunRunnerTest.php

<?php

use App\Models\Host;
use App\Models\ProvisioningRun;
use App\Provisioning\Events\StepAdvanced;
use App\Provisioning\Jobs\AdvanceRunJob;
use App\Provisioning\PipelineRegistry;
use App\Provisioning\RunRunner;
use Illuminate\Support\Facades\Cache;
use Illuminate\Support\Facades\Event;
use Illuminate\Support\Facades\Queue;
use Tests\Support\Steps\FakeAdvanceStep;
use Tests\Support\Steps\FakeFailStep;
use Tests\Support\Steps\FakePollStep;
use Tests\Support\Steps\FakeRetryStep;
use Tests\Support\Steps\FakeThrowStep;

function bindPipeline(array $pipelines): void
{
    app()->instance(PipelineRegistry::class, new PipelineRegistry($pipelines));
}

it('advances to the next step and queues the follow-up job', function () {
    Queue::fake();
    bindPipeline(['test' => [FakeAdvanceStep::class, FakeAdvanceStep::class]]);
    $run = ProvisioningRun::factory()->create(['pipeline' => 'test', 'current_step' => 0, 'status' => 'pending']);

    app(RunRunner::class)->advance($run);
    $run->refresh();

    expect($run->current_step)->toBe(1)
        ->and($run->status)->toBe('running')
        ->and($run->events()->where('outcome', 'advanced')->count())->toBe(1);
    Queue::assertPushed(AdvanceRunJob::class);
});

it('completes the run on the last step', function () {
    Queue::fake();
    bindPipeline(['test' => [FakeAdvanceStep::class]]);
    $run = ProvisioningRun::factory()->create(['pipeline' => 'test', 'status' => 'pending']);

    app(RunRunner::class)->advance($run);
    $run->refresh();

    expect($run->status)->toBe('completed')->and($run->finished_at)->not->toBeNull();
    Queue::assertNotPushed(AdvanceRunJob::class);
});

it('waits and increments attempt on retry', function () {
    Queue::fake();
    bindPipeline(['test' => [FakeRetryStep::class]]);
    $run = ProvisioningRun::factory()->create(['pipeline' => 'test', 'max_attempts' => 5]);

    app(RunRunner::class)->advance($run);
    $run->refresh();

    expect($run->status)->toBe('waiting')
        ->and($run->attempt)->toBe(1)
        ->and($run->next_attempt_at)->not->toBeNull();
    Queue::assertNotPushed(AdvanceRunJob::class);
});

it('fails once retries are exhausted', function () {
    bindPipeline(['test' => [FakeRetryStep::class]]);
    $run = ProvisioningRun::factory()->create(['pipeline' => 'test', 'max_attempts' => 1]);

    app(RunRunner::class)->advance($run);

    expect($run->refresh()->status)->toBe('failed');
});

it('fails immediately on a fail result', function () {
    bindPipeline(['test' => [FakeFailStep::class]]);
    $run = ProvisioningRun::factory()->create(['pipeline' => 'test']);

    app(RunRunner::class)->advance($run);
    $run->refresh();

    expect($run->status)->toBe('failed')->and($run->error)->toBe('boom');
});

it('polls without consuming the retry budget', function () {
    bindPipeline(['test' => [FakePollStep::class]]);
    $run = ProvisioningRun::factory()->create(['pipeline' => 'test', 'max_attempts' => 1]);

    app(RunRunner::class)->advance($run);
    expect($run->fresh()->status)->toBe('waiting')->and($run->fresh()->attempt)->toBe(0);

    // Make the run due again, then poll a second time: still not failed even
    // with max_attempts = 1 (polls don't consume the retry budget).
    $run->update(['next_attempt_at' => now()->subSecond()]);
    app(RunRunner::class)->advance($run->fresh());
    expect($run->fresh()->status)->toBe('waiting')->and($run->fresh()->attempt)->toBe(0);
});

it('marks a Host subject as error when the run fails', function () {
    bindPipeline(['test' => [FakeFailStep::class]]);
    $host = Host::factory()->create(['status' => 'onboarding']);
    $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'test']);

    app(RunRunner::class)->advance($run);

    expect($run->fresh()->status)->toBe('failed')
        ->and($host->fresh()->status)->toBe('error');
});

it('treats a thrown exception as a retry', function () {
    bindPipeline(['test' => [FakeThrowStep::class]]);
    $run = ProvisioningRun::factory()->create(['pipeline' => 'test', 'max_attempts' => 5]);

    app(RunRunner::class)->advance($run);
    $run->refresh();

    expect($run->status)->toBe('waiting')
        ->and($run->attempt)->toBe(1)
        ->and($run->events()->where('outcome', 'retry')->exists())->toBeTrue();
});

it('retries when a step exceeds its max duration', function () {
    bindPipeline(['test' => [FakeAdvanceStep::class]]);
    $run = ProvisioningRun::factory()->create([
        'pipeline' => 'test',
        'status' => 'running',
        'started_at' => now()->subMinutes(10),
        'max_attempts' => 5,
    ]);

    app(RunRunner::class)->advance($run);
    $run->refresh();

    expect($run->status)->toBe('waiting')
        ->and($run->events()->where('message', 'like', '%timed out%')->exists())->toBeTrue();
});

it('re-executes a step after a timeout retry instead of timing out again', function () {
    bindPipeline(['test' => [FakeAdvanceStep::class]]);
    $run = ProvisioningRun::factory()->create([
        'pipeline' => 'test', 'status' => 'running',
        'started_at' => now()->subMinutes(10), 'max_attempts' => 5,
    ]);

    app(RunRunner::class)->advance($run);          // times out -> retry (timer reset)
    expect($run->fresh()->status)->toBe('waiting');

    app(RunRunner::class)->advance($run->fresh()); // now actually executes -> completes
    expect($run->fresh()->status)->toBe('completed');
});

it('fails terminally when the pipeline step cannot be resolved', function () {
    bindPipeline(['test' => [FakeAdvanceStep::class]]);
    $run = ProvisioningRun::factory()->create(['pipeline' => 'test', 'current_step' => 9]);

    app(RunRunner::class)->advance($run);

    expect($run->fresh()->status)->toBe('failed')
        ->and($run->fresh()->events()->where('step', 'pipeline_resolution')->exists())->toBeTrue();
});

it('is a no-op while the run lock is held', function () {
    bindPipeline(['test' => [FakeAdvanceStep::class, FakeAdvanceStep::class]]);
    $run = ProvisioningRun::factory()->create(['pipeline' => 'test', 'current_step' => 0]);

    $lock = Cache::lock('run:'.$run->uuid, 120);
    expect($lock->get())->toBeTrue();

    app(RunRunner::class)->advance($run);
    expect($run->refresh()->current_step)->toBe(0);

    $lock->release();
});

it('broadcasts a StepAdvanced event', function () {
    Event::fake([StepAdvanced::class]);
    bindPipeline(['test' => [FakeAdvanceStep::class]]);
    $run = ProvisioningRun::factory()->create(['pipeline' => 'test']);

    app(RunRunner::class)->advance($run);

    Event::assertDispatched(
        StepAdvanced::class,
        fn ($e) => $e->runUuid === $run->uuid && $e->status === 'completed',
    );
});