instance(PipelineRegistry::class, new PipelineRegistry($pipelines)); } it('advances to the next step and queues the follow-up job', function () { Queue::fake(); bindPipeline(['test' => [FakeAdvanceStep::class, FakeAdvanceStep::class]]); $run = ProvisioningRun::factory()->create(['pipeline' => 'test', 'current_step' => 0, 'status' => 'pending']); app(RunRunner::class)->advance($run); $run->refresh(); expect($run->current_step)->toBe(1) ->and($run->status)->toBe('running') ->and($run->events()->where('outcome', 'advanced')->count())->toBe(1); Queue::assertPushed(AdvanceRunJob::class); }); it('completes the run on the last step', function () { Queue::fake(); bindPipeline(['test' => [FakeAdvanceStep::class]]); $run = ProvisioningRun::factory()->create(['pipeline' => 'test', 'status' => 'pending']); app(RunRunner::class)->advance($run); $run->refresh(); expect($run->status)->toBe('completed')->and($run->finished_at)->not->toBeNull(); Queue::assertNotPushed(AdvanceRunJob::class); }); it('waits and increments attempt on retry', function () { Queue::fake(); bindPipeline(['test' => [FakeRetryStep::class]]); $run = ProvisioningRun::factory()->create(['pipeline' => 'test', 'max_attempts' => 5]); app(RunRunner::class)->advance($run); $run->refresh(); expect($run->status)->toBe('waiting') ->and($run->attempt)->toBe(1) ->and($run->next_attempt_at)->not->toBeNull(); Queue::assertNotPushed(AdvanceRunJob::class); }); it('fails once retries are exhausted', function () { bindPipeline(['test' => [FakeRetryStep::class]]); $run = ProvisioningRun::factory()->create(['pipeline' => 'test', 'max_attempts' => 1]); app(RunRunner::class)->advance($run); expect($run->refresh()->status)->toBe('failed'); }); it('fails immediately on a fail result', function () { bindPipeline(['test' => [FakeFailStep::class]]); $run = ProvisioningRun::factory()->create(['pipeline' => 'test']); app(RunRunner::class)->advance($run); $run->refresh(); expect($run->status)->toBe('failed')->and($run->error)->toBe('boom'); }); it('polls without consuming the retry budget', function () { bindPipeline(['test' => [FakePollStep::class]]); $run = ProvisioningRun::factory()->create(['pipeline' => 'test', 'max_attempts' => 1]); app(RunRunner::class)->advance($run); expect($run->fresh()->status)->toBe('waiting')->and($run->fresh()->attempt)->toBe(0); // Make the run due again, then poll a second time: still not failed even // with max_attempts = 1 (polls don't consume the retry budget). $run->update(['next_attempt_at' => now()->subSecond()]); app(RunRunner::class)->advance($run->fresh()); expect($run->fresh()->status)->toBe('waiting')->and($run->fresh()->attempt)->toBe(0); }); it('keeps started_at stable across polls so step deadlines can accumulate', function () { bindPipeline(['test' => [FakePollStep::class]]); $run = ProvisioningRun::factory()->create([ 'pipeline' => 'test', 'status' => 'running', 'started_at' => now()->subMinutes(5), ]); $original = $run->started_at->timestamp; app(RunRunner::class)->advance($run); expect($run->fresh()->started_at->timestamp)->toBe($original); }); it('marks a Host subject as error when the run fails', function () { // Named 'host' rather than 'test': the failure hook belongs to the pipeline // that BUILDS the subject, and the runner now asks which one that is. bindPipeline(['host' => [FakeFailStep::class]]); $host = Host::factory()->create(['status' => 'onboarding']); $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'host']); app(RunRunner::class)->advance($run); expect($run->fresh()->status)->toBe('failed') ->and($host->fresh()->status)->toBe('error'); }); it('does not condemn the subject when a maintenance run fails', function () { // A run that maintains something already built shares its subject. If its // failure fired the same hook, a router file that could not be written // would put a live, paid-for host into 'error' — and, on the customer side, // mark the order failed and release the running instance with it. bindPipeline(['address' => [FakeFailStep::class]]); $host = Host::factory()->create(['status' => 'active']); $run = ProvisioningRun::factory()->forHost($host)->create(['pipeline' => 'address']); app(RunRunner::class)->advance($run); expect($run->fresh()->status)->toBe('failed') ->and($host->fresh()->status)->toBe('active'); }); it('treats a thrown exception as a retry', function () { bindPipeline(['test' => [FakeThrowStep::class]]); $run = ProvisioningRun::factory()->create(['pipeline' => 'test', 'max_attempts' => 5]); app(RunRunner::class)->advance($run); $run->refresh(); expect($run->status)->toBe('waiting') ->and($run->attempt)->toBe(1) ->and($run->events()->where('outcome', 'retry')->exists())->toBeTrue(); }); it('retries when a step exceeds its max duration', function () { bindPipeline(['test' => [FakeAdvanceStep::class]]); $run = ProvisioningRun::factory()->create([ 'pipeline' => 'test', 'status' => 'running', 'started_at' => now()->subMinutes(10), 'max_attempts' => 5, ]); app(RunRunner::class)->advance($run); $run->refresh(); expect($run->status)->toBe('waiting') ->and($run->events()->where('message', 'like', '%timed out%')->exists())->toBeTrue(); }); it('re-executes a step after a timeout retry instead of timing out again', function () { bindPipeline(['test' => [FakeAdvanceStep::class]]); $run = ProvisioningRun::factory()->create([ 'pipeline' => 'test', 'status' => 'running', 'started_at' => now()->subMinutes(10), 'max_attempts' => 5, ]); app(RunRunner::class)->advance($run); // times out -> retry (timer reset) expect($run->fresh()->status)->toBe('waiting'); app(RunRunner::class)->advance($run->fresh()); // now actually executes -> completes expect($run->fresh()->status)->toBe('completed'); }); it('fails terminally when the pipeline step cannot be resolved', function () { bindPipeline(['test' => [FakeAdvanceStep::class]]); $run = ProvisioningRun::factory()->create(['pipeline' => 'test', 'current_step' => 9]); app(RunRunner::class)->advance($run); expect($run->fresh()->status)->toBe('failed') ->and($run->fresh()->events()->where('step', 'pipeline_resolution')->exists())->toBeTrue(); }); it('is a no-op while the run lock is held', function () { bindPipeline(['test' => [FakeAdvanceStep::class, FakeAdvanceStep::class]]); $run = ProvisioningRun::factory()->create(['pipeline' => 'test', 'current_step' => 0]); $lock = Cache::lock('run:'.$run->uuid, 120); expect($lock->get())->toBeTrue(); app(RunRunner::class)->advance($run); expect($run->refresh()->current_step)->toBe(0); $lock->release(); }); it('broadcasts a StepAdvanced event', function () { Event::fake([StepAdvanced::class]); bindPipeline(['test' => [FakeAdvanceStep::class]]); $run = ProvisioningRun::factory()->create(['pipeline' => 'test']); app(RunRunner::class)->advance($run); Event::assertDispatched( StepAdvanced::class, fn ($e) => $e->runUuid === $run->uuid && $e->status === 'completed', ); });