fix(engine): reset step timer on retry so timed-out steps recover

A retried step (manual or after a timeout) now gets a fresh started_at, so
RunRunner no longer immediately re-detects the timeout and burns the retry
budget without executing.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
feat/portal-design
nexxo 2026-07-25 10:29:11 +02:00
parent 62c4412623
commit 8a2a2ee695
4 changed files with 19 additions and 0 deletions

View File

@ -36,6 +36,7 @@ class HostDetail extends Component
'status' => ProvisioningRun::STATUS_RUNNING,
'attempt' => 0,
'next_attempt_at' => now(),
'started_at' => now(), // reset the step timer so it doesn't re-time-out instantly
'error' => null,
]);
$this->host->update(['status' => 'onboarding']);

View File

@ -114,6 +114,7 @@ class RunRunner
$run->status = ProvisioningRun::STATUS_WAITING;
$run->next_attempt_at = now()->addSeconds($result->afterSeconds);
$run->started_at = now(); // fresh per-attempt timer, else a timed-out step re-times-out
$run->save();
$this->record($run, $step, 'retry', $result->reason);
}
@ -123,6 +124,7 @@ class RunRunner
{
$run->status = ProvisioningRun::STATUS_WAITING;
$run->next_attempt_at = now()->addSeconds($result->afterSeconds);
$run->started_at = now();
$run->save();
$this->record($run, $step, 'info', $result->reason);
}

View File

@ -73,6 +73,7 @@ it('retries a failed run from the detail page', function () {
$host = Host::factory()->create(['status' => 'error']);
$run = ProvisioningRun::factory()->forHost($host)->create([
'status' => 'failed', 'current_step' => 3, 'error' => 'boom',
'started_at' => now()->subHour(),
]);
Livewire::actingAs(admin())->test(HostDetail::class, ['host' => $host])->call('retry');
@ -80,6 +81,7 @@ it('retries a failed run from the detail page', function () {
$run->refresh();
expect($run->status)->toBe('running')
->and($run->error)->toBeNull()
->and($run->started_at->greaterThan(now()->subMinute()))->toBeTrue()
->and($host->fresh()->status)->toBe('onboarding');
Queue::assertPushed(AdvanceRunJob::class);
});

View File

@ -130,6 +130,20 @@ it('retries when a step exceeds its max duration', function () {
->and($run->events()->where('message', 'like', '%timed out%')->exists())->toBeTrue();
});
it('re-executes a step after a timeout retry instead of timing out again', function () {
bindPipeline(['test' => [FakeAdvanceStep::class]]);
$run = ProvisioningRun::factory()->create([
'pipeline' => 'test', 'status' => 'running',
'started_at' => now()->subMinutes(10), 'max_attempts' => 5,
]);
app(RunRunner::class)->advance($run); // times out -> retry (timer reset)
expect($run->fresh()->status)->toBe('waiting');
app(RunRunner::class)->advance($run->fresh()); // now actually executes -> completes
expect($run->fresh()->status)->toBe('completed');
});
it('is a no-op while the run lock is held', function () {
bindPipeline(['test' => [FakeAdvanceStep::class, FakeAdvanceStep::class]]);
$run = ProvisioningRun::factory()->create(['pipeline' => 'test', 'current_step' => 0]);