From 8a2a2ee695f6d1097861233dec7ec92180d0b02e Mon Sep 17 00:00:00 2001 From: nexxo Date: Sat, 25 Jul 2026 10:29:11 +0200 Subject: [PATCH] fix(engine): reset step timer on retry so timed-out steps recover A retried step (manual or after a timeout) now gets a fresh started_at, so RunRunner no longer immediately re-detects the timeout and burns the retry budget without executing. Co-Authored-By: Claude Opus 4.8 --- app/Livewire/Admin/HostDetail.php | 1 + app/Provisioning/RunRunner.php | 2 ++ tests/Feature/Admin/HostManagementTest.php | 2 ++ tests/Feature/Provisioning/RunRunnerTest.php | 14 ++++++++++++++ 4 files changed, 19 insertions(+) diff --git a/app/Livewire/Admin/HostDetail.php b/app/Livewire/Admin/HostDetail.php index aa5432a..19c753c 100644 --- a/app/Livewire/Admin/HostDetail.php +++ b/app/Livewire/Admin/HostDetail.php @@ -36,6 +36,7 @@ class HostDetail extends Component 'status' => ProvisioningRun::STATUS_RUNNING, 'attempt' => 0, 'next_attempt_at' => now(), + 'started_at' => now(), // reset the step timer so it doesn't re-time-out instantly 'error' => null, ]); $this->host->update(['status' => 'onboarding']); diff --git a/app/Provisioning/RunRunner.php b/app/Provisioning/RunRunner.php index 2929363..b21d82e 100644 --- a/app/Provisioning/RunRunner.php +++ b/app/Provisioning/RunRunner.php @@ -114,6 +114,7 @@ class RunRunner $run->status = ProvisioningRun::STATUS_WAITING; $run->next_attempt_at = now()->addSeconds($result->afterSeconds); + $run->started_at = now(); // fresh per-attempt timer, else a timed-out step re-times-out $run->save(); $this->record($run, $step, 'retry', $result->reason); } @@ -123,6 +124,7 @@ class RunRunner { $run->status = ProvisioningRun::STATUS_WAITING; $run->next_attempt_at = now()->addSeconds($result->afterSeconds); + $run->started_at = now(); $run->save(); $this->record($run, $step, 'info', $result->reason); } diff --git a/tests/Feature/Admin/HostManagementTest.php b/tests/Feature/Admin/HostManagementTest.php index f911b3e..563770e 100644 --- a/tests/Feature/Admin/HostManagementTest.php +++ b/tests/Feature/Admin/HostManagementTest.php @@ -73,6 +73,7 @@ it('retries a failed run from the detail page', function () { $host = Host::factory()->create(['status' => 'error']); $run = ProvisioningRun::factory()->forHost($host)->create([ 'status' => 'failed', 'current_step' => 3, 'error' => 'boom', + 'started_at' => now()->subHour(), ]); Livewire::actingAs(admin())->test(HostDetail::class, ['host' => $host])->call('retry'); @@ -80,6 +81,7 @@ it('retries a failed run from the detail page', function () { $run->refresh(); expect($run->status)->toBe('running') ->and($run->error)->toBeNull() + ->and($run->started_at->greaterThan(now()->subMinute()))->toBeTrue() ->and($host->fresh()->status)->toBe('onboarding'); Queue::assertPushed(AdvanceRunJob::class); }); diff --git a/tests/Feature/Provisioning/RunRunnerTest.php b/tests/Feature/Provisioning/RunRunnerTest.php index 0645132..a047e7d 100644 --- a/tests/Feature/Provisioning/RunRunnerTest.php +++ b/tests/Feature/Provisioning/RunRunnerTest.php @@ -130,6 +130,20 @@ it('retries when a step exceeds its max duration', function () { ->and($run->events()->where('message', 'like', '%timed out%')->exists())->toBeTrue(); }); +it('re-executes a step after a timeout retry instead of timing out again', function () { + bindPipeline(['test' => [FakeAdvanceStep::class]]); + $run = ProvisioningRun::factory()->create([ + 'pipeline' => 'test', 'status' => 'running', + 'started_at' => now()->subMinutes(10), 'max_attempts' => 5, + ]); + + app(RunRunner::class)->advance($run); // times out -> retry (timer reset) + expect($run->fresh()->status)->toBe('waiting'); + + app(RunRunner::class)->advance($run->fresh()); // now actually executes -> completes + expect($run->fresh()->status)->toBe('completed'); +}); + it('is a no-op while the run lock is held', function () { bindPipeline(['test' => [FakeAdvanceStep::class, FakeAdvanceStep::class]]); $run = ProvisioningRun::factory()->create(['pipeline' => 'test', 'current_step' => 0]);