fix(engine-b): clone recovery waits for lock; register rows before breadcrumb
- Clone recovery polls until the VM lock clears instead of advancing onto a possibly-incomplete clone. - RegisterBackup/RegisterMonitoring create the local row BEFORE the run-resource breadcrumb, so a crash between them can't leave the row permanently missing. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>feat/portal-design
parent
3c5a18b5d7
commit
661b52dbe0
|
|
@ -49,9 +49,12 @@ class CloneVirtualMachine extends CustomerStep
|
|||
$upid = $run->context('clone_upid');
|
||||
if ($upid === null) {
|
||||
// Crash-recovery: if the VM already exists, a prior attempt cloned it
|
||||
// (but lost the task ref) — don't re-clone the same vmid (Proxmox rejects).
|
||||
// (but lost the task ref) — don't re-clone. Wait until the clone lock
|
||||
// clears before advancing, in case the copy is still running.
|
||||
if ($pve->vmExists($node, $vmid)) {
|
||||
return StepResult::advance();
|
||||
return empty($pve->vmStatus($node, $vmid)['lock'])
|
||||
? StepResult::advance()
|
||||
: StepResult::poll(10, 'finishing recovered clone');
|
||||
}
|
||||
|
||||
$upid = $pve->cloneVm($node, $templateVmid, $vmid, $name);
|
||||
|
|
|
|||
|
|
@ -33,13 +33,14 @@ class RegisterBackup extends CustomerStep
|
|||
$jobId = $this->pve->forHost($instance->host)
|
||||
->createBackupJob((string) $run->context('node'), (int) $instance->vmid, $schedule);
|
||||
|
||||
$this->recordResource($run, $instance->host, 'backup_job_id', $jobId);
|
||||
// firstOrCreate keyed on the deterministic job id → no duplicate row if a
|
||||
// crash re-runs this step before the breadcrumb was recorded.
|
||||
// Create the local row BEFORE the breadcrumb — the breadcrumb is the
|
||||
// short-circuit guard, so if it exists the row must already exist too.
|
||||
// firstOrCreate on the deterministic job id keeps a retry duplicate-free.
|
||||
$instance->backups()->firstOrCreate(
|
||||
['external_job_id' => $jobId],
|
||||
['schedule' => $schedule, 'status' => 'scheduled'],
|
||||
);
|
||||
$this->recordResource($run, $instance->host, 'backup_job_id', $jobId);
|
||||
|
||||
return StepResult::advance();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,11 +32,12 @@ class RegisterMonitoring extends CustomerStep
|
|||
// Register with the monitoring service [E] before recording it.
|
||||
$targetId = $this->monitoring->registerTarget($instance->subdomain, $url);
|
||||
|
||||
$this->recordResource($run, $instance->host, 'monitoring_target_id', $targetId);
|
||||
// Local row BEFORE the breadcrumb (the short-circuit guard).
|
||||
$instance->monitoringTargets()->firstOrCreate(
|
||||
['external_id' => $targetId],
|
||||
['url' => $url, 'status' => 'up'],
|
||||
);
|
||||
$this->recordResource($run, $instance->host, 'monitoring_target_id', $targetId);
|
||||
|
||||
return StepResult::advance();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -114,9 +114,15 @@ class FakeProxmoxClient implements ProxmoxClient
|
|||
return 'UPID:pve:qmstart:'.$vmid;
|
||||
}
|
||||
|
||||
/** Set to e.g. 'clone' to simulate a VM still locked by a running clone task. */
|
||||
public ?string $vmLock = null;
|
||||
|
||||
public function vmStatus(string $node, int $vmid): array
|
||||
{
|
||||
return ['status' => in_array($vmid, $this->runningVmids, true) ? 'running' : 'stopped'];
|
||||
return [
|
||||
'status' => in_array($vmid, $this->runningVmids, true) ? 'running' : 'stopped',
|
||||
'lock' => $this->vmLock,
|
||||
];
|
||||
}
|
||||
|
||||
public function vmExists(string $node, int $vmid): bool
|
||||
|
|
|
|||
|
|
@ -132,6 +132,10 @@ it('recovers a clone whose task ref was lost (no duplicate clone)', function ()
|
|||
|
||||
expect(app(CloneVirtualMachine::class)->execute($run->fresh())->type)->toBe('advance')
|
||||
->and(count($s['pve']->clonedVmids))->toBe(1); // no second clone
|
||||
|
||||
// still locked (clone copy running) → poll, don't advance onto an incomplete VM
|
||||
$s['pve']->vmLock = 'clone';
|
||||
expect(app(CloneVirtualMachine::class)->execute($run->fresh())->type)->toBe('poll');
|
||||
});
|
||||
|
||||
it('fails clone when the plan has no template', function () {
|
||||
|
|
|
|||
Loading…
Reference in New Issue