fix(engine): honour backoff timing + don't double-count Proxmox storage
- RunRunner returns early for a waiting run whose next_attempt_at is in the future, so stale/duplicate jobs can't bypass backoff. - RegisterCapacity takes the largest VM-capable datastore instead of summing overlapping pools (local + local-lvm), preventing placement overcommit. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>feat/portal-design
parent
8a2a2ee695
commit
fbe87d99d5
|
|
@ -45,6 +45,13 @@ class RunRunner
|
|||
return;
|
||||
}
|
||||
|
||||
// Honour backoff: a stale/duplicate job must not run a waiting run early.
|
||||
if ($run->status === ProvisioningRun::STATUS_WAITING
|
||||
&& $run->next_attempt_at !== null
|
||||
&& $run->next_attempt_at->isFuture()) {
|
||||
return;
|
||||
}
|
||||
|
||||
// started_at marks when the current step began (drives the timeout).
|
||||
if ($run->started_at === null || $run->status === ProvisioningRun::STATUS_PENDING) {
|
||||
$run->started_at ??= now();
|
||||
|
|
|
|||
|
|
@ -32,7 +32,18 @@ class RegisterCapacity extends HostStep
|
|||
|
||||
$cores = (int) ($status['cpuinfo']['cpus'] ?? 0);
|
||||
$ramMb = (int) round((int) ($status['memory']['total'] ?? 0) / 1048576);
|
||||
$totalGb = (int) round(array_sum(array_column($storage, 'total')) / 1073741824);
|
||||
|
||||
// Proxmox lists overlapping pools (e.g. local + local-lvm on one disk).
|
||||
// Take the largest VM-capable datastore instead of summing, so shared
|
||||
// backing storage is never double-counted and placement can't overcommit.
|
||||
$allocatable = array_filter($storage, function ($s) {
|
||||
$content = (string) ($s['content'] ?? '');
|
||||
|
||||
return str_contains($content, 'images') || str_contains($content, 'rootdir');
|
||||
});
|
||||
$pool = $allocatable ?: $storage;
|
||||
$totalBytes = $pool === [] ? 0 : max(array_map(fn ($s) => (int) ($s['total'] ?? 0), $pool));
|
||||
$totalGb = (int) round($totalBytes / 1073741824);
|
||||
|
||||
$host->update([
|
||||
'cpu_cores' => $cores,
|
||||
|
|
|
|||
|
|
@ -20,7 +20,8 @@ class FakeProxmoxClient implements ProxmoxClient
|
|||
|
||||
/** @var array<int, array<string, mixed>> */
|
||||
public array $storage = [
|
||||
['storage' => 'local-lvm', 'type' => 'lvmthin', 'total' => 1099511627776], // 1 TiB
|
||||
['storage' => 'local', 'type' => 'dir', 'content' => 'iso,vztmpl,backup', 'total' => 1099511627776],
|
||||
['storage' => 'local-lvm', 'type' => 'lvmthin', 'content' => 'images,rootdir', 'total' => 1099511627776], // 1 TiB
|
||||
];
|
||||
|
||||
public function forHost(Host $host): static
|
||||
|
|
|
|||
|
|
@ -31,6 +31,9 @@ it('drives a fresh host all the way to active (mocked)', function () {
|
|||
if (in_array($run->status, ['completed', 'failed'], true)) {
|
||||
break;
|
||||
}
|
||||
if ($run->next_attempt_at?->isFuture()) {
|
||||
$run->update(['next_attempt_at' => now()]); // simulate the scheduler waking a due run
|
||||
}
|
||||
$runner->advance($run);
|
||||
}
|
||||
|
||||
|
|
@ -73,6 +76,9 @@ it('does not duplicate external resources when a step re-runs after a crash', fu
|
|||
// WireGuard step and replaying it — the breadcrumb must prevent duplicates.
|
||||
for ($i = 0; $i < 6; $i++) {
|
||||
$run->refresh();
|
||||
if ($run->next_attempt_at?->isFuture()) {
|
||||
$run->update(['next_attempt_at' => now()]);
|
||||
}
|
||||
$runner->advance($run);
|
||||
}
|
||||
$run->refresh();
|
||||
|
|
|
|||
|
|
@ -86,7 +86,9 @@ it('polls without consuming the retry budget', function () {
|
|||
app(RunRunner::class)->advance($run);
|
||||
expect($run->fresh()->status)->toBe('waiting')->and($run->fresh()->attempt)->toBe(0);
|
||||
|
||||
// A second poll must not fail the run even with max_attempts = 1.
|
||||
// Make the run due again, then poll a second time: still not failed even
|
||||
// with max_attempts = 1 (polls don't consume the retry budget).
|
||||
$run->update(['next_attempt_at' => now()->subSecond()]);
|
||||
app(RunRunner::class)->advance($run->fresh());
|
||||
expect($run->fresh()->status)->toBe('waiting')->and($run->fresh()->attempt)->toBe(0);
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in New Issue