fix(engine): honour backoff timing + don't double-count Proxmox storage

- RunRunner returns early for a waiting run whose next_attempt_at is in the
  future, so stale/duplicate jobs can't bypass backoff.
- RegisterCapacity takes the largest VM-capable datastore instead of summing
  overlapping pools (local + local-lvm), preventing placement overcommit.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
feat/portal-design
nexxo 2026-07-25 10:32:14 +02:00
parent 8a2a2ee695
commit fbe87d99d5
5 changed files with 30 additions and 3 deletions

View File

@ -45,6 +45,13 @@ class RunRunner
return;
}
// Honour backoff: a stale/duplicate job must not run a waiting run early.
if ($run->status === ProvisioningRun::STATUS_WAITING
&& $run->next_attempt_at !== null
&& $run->next_attempt_at->isFuture()) {
return;
}
// started_at marks when the current step began (drives the timeout).
if ($run->started_at === null || $run->status === ProvisioningRun::STATUS_PENDING) {
$run->started_at ??= now();

View File

@ -32,7 +32,18 @@ class RegisterCapacity extends HostStep
$cores = (int) ($status['cpuinfo']['cpus'] ?? 0);
$ramMb = (int) round((int) ($status['memory']['total'] ?? 0) / 1048576);
$totalGb = (int) round(array_sum(array_column($storage, 'total')) / 1073741824);
// Proxmox lists overlapping pools (e.g. local + local-lvm on one disk).
// Take the largest VM-capable datastore instead of summing, so shared
// backing storage is never double-counted and placement can't overcommit.
$allocatable = array_filter($storage, function ($s) {
$content = (string) ($s['content'] ?? '');
return str_contains($content, 'images') || str_contains($content, 'rootdir');
});
$pool = $allocatable ?: $storage;
$totalBytes = $pool === [] ? 0 : max(array_map(fn ($s) => (int) ($s['total'] ?? 0), $pool));
$totalGb = (int) round($totalBytes / 1073741824);
$host->update([
'cpu_cores' => $cores,

View File

@ -20,7 +20,8 @@ class FakeProxmoxClient implements ProxmoxClient
/** @var array<int, array<string, mixed>> */
public array $storage = [
['storage' => 'local-lvm', 'type' => 'lvmthin', 'total' => 1099511627776], // 1 TiB
['storage' => 'local', 'type' => 'dir', 'content' => 'iso,vztmpl,backup', 'total' => 1099511627776],
['storage' => 'local-lvm', 'type' => 'lvmthin', 'content' => 'images,rootdir', 'total' => 1099511627776], // 1 TiB
];
public function forHost(Host $host): static

View File

@ -31,6 +31,9 @@ it('drives a fresh host all the way to active (mocked)', function () {
if (in_array($run->status, ['completed', 'failed'], true)) {
break;
}
if ($run->next_attempt_at?->isFuture()) {
$run->update(['next_attempt_at' => now()]); // simulate the scheduler waking a due run
}
$runner->advance($run);
}
@ -73,6 +76,9 @@ it('does not duplicate external resources when a step re-runs after a crash', fu
// WireGuard step and replaying it — the breadcrumb must prevent duplicates.
for ($i = 0; $i < 6; $i++) {
$run->refresh();
if ($run->next_attempt_at?->isFuture()) {
$run->update(['next_attempt_at' => now()]);
}
$runner->advance($run);
}
$run->refresh();

View File

@ -86,7 +86,9 @@ it('polls without consuming the retry budget', function () {
app(RunRunner::class)->advance($run);
expect($run->fresh()->status)->toBe('waiting')->and($run->fresh()->attempt)->toBe(0);
// A second poll must not fail the run even with max_attempts = 1.
// Make the run due again, then poll a second time: still not failed even
// with max_attempts = 1 (polls don't consume the retry budget).
$run->update(['next_attempt_at' => now()->subSecond()]);
app(RunRunner::class)->advance($run->fresh());
expect($run->fresh()->status)->toBe('waiting')->and($run->fresh()->attempt)->toBe(0);
});