host($run); $required = PlanVersion::requiredTemplateVmids(); // An installation that takes over its first host before it publishes a // plan needs no template. The same real state VerifyVmTemplate lets // through, and inventing a requirement here would contradict it. if ($required === []) { return StepResult::advance(); } try { $missing = $this->templatesStillMissing($host, $required); if ($missing === []) { return StepResult::advance(); } } catch (Throwable $e) { // VerifyProxmoxApi passed one step ago; a hiccup here is transient. // isTemplate() throws rather than answering "absent" for anything // short of a successful listing, precisely so this branch — and not // a destructive rebuild — is where an outage lands. return StepResult::retry(15, 'could not ask Proxmox about the VM templates: '.$e->getMessage()); } $this->keyLogin($this->shell, $host); $status = $this->readStatus(); // Nothing on the host is cleaned up when a run ends, so a status file // can be older than this run. Read as this run's own, a leftover `ok` // says "the build I started finished without producing what I asked // for" — and fails a host that only needed a build to be started. That // is the ordinary case the day the catalogue gains a second template. // // The deadline in the run context is the only thing that distinguishes // them: it is written when THIS run starts a build, and cleared again // by giveUp(). if ($run->context('template_build_deadline') === null) { return $this->adoptOrStart($run, $missing, $status); } return match ($status['state']) { '' => $this->giveUp( $run, 'The template build was started but left no status behind, so it never really began. '. 'Check the free space and '.self::WORK_DIR.' on the host, then retry.' ), 'running' => $this->whileRunning($run, $status), 'ok' => $this->giveUp( $run, 'The template build reported success, but VMID '.implode(', ', $missing). ' is still not a template through the Proxmox API — so a clone would fail. '. 'Check '.self::WORK_DIR.'/build.log on the host.' ), 'failed' => $this->giveUp($run, 'The template build failed: '.($status['note'] ?: 'no reason recorded').'.'), default => $this->giveUp($run, 'The template build left an unreadable state ("'.$status['state'].'").'), }; } /** * What to do about a status this run did not write. * * Everything except a LIVE build is simply overwritten by starting a new * one: a finished or failed verdict from an earlier run answers a question * nobody asked here, and its files are replaced by the start command. * * A live build is the one exception, and it has to be, or the collision * arrives from the other side: two drivers on one build.qcow2, both calling * `qm destroy --purge` and `qm create` on the same VMIDs. It is building the * same thing this run wants, so it is adopted — with a deadline of its own, * so an adopted build cannot be polled forever either. * * @param array $missing * @param array{state:string, alive:string, phase:string, note:string} $status */ private function adoptOrStart(ProvisioningRun $run, array $missing, array $status): StepResult { if ($status['state'] === 'running' && $status['alive'] === 'yes') { $run->mergeContext([ 'template_build_deadline' => now()->addMinutes(self::DEADLINE_MINUTES)->toIso8601String(), ]); return StepResult::poll(30, 'building the VM template: '.($status['phase'] ?: 'running')); } return $this->start($run, $missing); } /** * The required VMIDs that are NOT already a real template on this node. * * A list rather than a yes/no, because that list is what gets built — and * create_proxmox_template purges a VMID before it creates it. Handing it * every required id when only one of them is missing destroys the healthy * templates on the way past, and a rebuild that then fails leaves the host * with fewer templates than it started with. * * isTemplate rather than vmExists, and that is the whole point of asking: * an abandoned build leaves an ordinary VM numbered 9000 behind. Skipping * on the strength of its existence would adopt the leftover as finished, * and the gap would surface again at the first paid clone. * * @param array $required * @return array */ private function templatesStillMissing(Host $host, array $required): array { $client = $this->pve->forHost($host); $node = $client->listNodes()[0]['node'] ?? 'pve'; return array_values(array_filter( $required, fn (int $vmid) => ! $client->isTemplate($node, $vmid), )); } /** @param array $missing */ private function start(ProvisioningRun $run, array $missing): StepResult { $this->shell->putFile(self::WORK_DIR.'/template.sh', $this->asset('lib/template.sh')); $this->shell->putFile(self::WORK_DIR.'/template-run.sh', $this->asset('lib/template-run.sh')); $this->shell->putFile(self::WORK_DIR.'/docker-compose.yml', $this->asset('assets/docker-compose.yml')); // Only what is actually missing, and only what the catalogue asks for. // Not a number baked into the script: an installation that repoints a // plan at another template would otherwise keep building the old one. $this->shell->putFile(self::WORK_DIR.'/env', implode("\n", [ '# Written by CluPilot at the start of every build. Do not edit.', 'CLUPILOT_WORK_DIR='.self::WORK_DIR, 'CLUPILOT_TEMPLATE_VMIDS="'.implode(' ', $missing).'"', '', ])); $run->mergeContext([ 'template_build_deadline' => now()->addMinutes(self::DEADLINE_MINUTES)->toIso8601String(), ]); // `setsid` puts the build in a session of its own, so the pid the script // records is also its PROCESS GROUP id — which is what makes giving up // possible later. Killing the driver alone would leave the // virt-customize it is waiting on chewing through the image. // // The pid therefore has to come from the script (`$$`), not from `$!` // here: with setsid in between, `$!` can be a process that is already // gone. Waiting for the file closes the gap that creates — `state` is // written by this command, so a poll arriving before the script had a // chance to record itself would otherwise read running-but-not-alive // and declare a healthy build dead. $this->shell->run(implode("\n", [ ': clupilot-template-start', 'W='.self::WORK_DIR, 'mkdir -p "$W"', "printf 'running' > \"\$W/state\"", "printf 'starting' > \"\$W/phase\"", ': > "$W/note"', ': > "$W/pid"', 'nohup setsid sh "$W/template-run.sh" > "$W/build.log" 2>&1 &', 'i=0; while [ "$i" -lt 15 ] && [ ! -s "$W/pid" ]; do sleep 1; i=$((i + 1)); done', ])); // No time estimate here any more. "10–20 minutes" was a guess written // before this had ever run; the first real build took under two, and a // console that predicts wrongly teaches an operator to ignore it. return StepResult::poll(30, 'building the VM template'); } /** @param array{state:string, alive:string, phase:string, note:string} $status */ private function whileRunning(ProvisioningRun $run, array $status): StepResult { if ($status['alive'] !== 'yes') { // A reboot, an OOM kill, an operator's Ctrl-C. The file says // running and always will — there is nobody left to change it. return $this->giveUp( $run, 'The template build is no longer running, but never reported a result — it was killed or the host '. 'restarted. Check '.self::WORK_DIR.'/build.log and retry.' ); } $deadline = $run->context('template_build_deadline'); if ($deadline !== null && now()->greaterThan(Carbon::parse($deadline))) { return $this->giveUp( $run, 'The template build passed its deadline of '.self::DEADLINE_MINUTES.' minutes while still in phase "'. ($status['phase'] ?: 'unknown').'". Check '.self::WORK_DIR.'/build.log on the host.' ); } return StepResult::poll(30, 'building the VM template: '.($status['phase'] ?: 'running')); } /** * Fail, and leave nothing behind that would make a retry read this verdict * again — neither the status on the host nor the deadline in the run, which * is what startOrGiveUp() reads to tell a first visit from a lost one. * * The verified cloud image is deliberately kept: it is the one expensive * thing a retry does not have to redo, and it is only ever under its final * name once its checksum has been confirmed. */ private function giveUp(ProvisioningRun $run, string $reason): StepResult { $run->forgetContext('template_build_deadline'); $tail = trim($this->shell->run(': clupilot-template-log'."\n".'tail -n 15 '.self::WORK_DIR.'/build.log 2>/dev/null')->stdout); // Stop it BEFORE the status files go, never after. Clearing them makes // the step retryable, and a retry that meets a build still running gets // two of them on one build.qcow2, both purging and recreating the same // VMIDs. The deadline path reaches here with the process very much // alive, which is exactly the case this covers. // // The whole process group (`-$P`), because the driver spends its time // waiting on a virt-customize that would otherwise carry on alone. The // cmdline guard is against a recycled pid: this file can be minutes old, // and killing a stranger's process group would be a far worse bug than // the one being handled. $this->shell->run(implode("\n", [ ': clupilot-template-reset', 'W='.self::WORK_DIR, 'P=$(cat "$W/pid" 2>/dev/null)', 'if [ -n "$P" ] && kill -0 "$P" 2>/dev/null && tr "\\0" " " < /proc/"$P"/cmdline 2>/dev/null | grep -q template-run; then', ' kill -TERM -"$P" 2>/dev/null || kill -TERM "$P" 2>/dev/null', ' i=0; while [ "$i" -lt 10 ] && kill -0 "$P" 2>/dev/null; do sleep 1; i=$((i + 1)); done', ' kill -KILL -"$P" 2>/dev/null || kill -KILL "$P" 2>/dev/null', 'fi', 'rm -f "$W/state" "$W/pid" "$W/note"', ])); return StepResult::fail($tail === '' ? $reason : $reason.' Last lines: '.$tail); } /** * The four facts about a build in flight, in one round trip. * * `alive` is computed on the host because that is the only place the answer * exists: the pid means nothing here. * * @return array{state:string, alive:string, phase:string, note:string} */ private function readStatus(): array { $out = $this->shell->run(implode("\n", [ ': clupilot-template-status', 'W='.self::WORK_DIR, 'S=$(cat "$W/state" 2>/dev/null)', 'P=$(cat "$W/pid" 2>/dev/null)', 'A=no', 'if [ -n "$P" ] && kill -0 "$P" 2>/dev/null; then A=yes; fi', "printf 'state=%s\\n' \"\$S\"", "printf 'alive=%s\\n' \"\$A\"", "printf 'phase=%s\\n' \"\$(head -1 \"\$W/phase\" 2>/dev/null)\"", "printf 'note=%s\\n' \"\$(head -1 \"\$W/note\" 2>/dev/null)\"", ]))->stdout; $status = ['state' => '', 'alive' => 'no', 'phase' => '', 'note' => '']; foreach (preg_split('/\R/', $out) ?: [] as $line) { [$key, $value] = array_pad(explode('=', $line, 2), 2, ''); if (array_key_exists($key, $status)) { $status[$key] = trim($value); } } return $status; } /** The shipped file, read from the repo — never rendered from a string in here. */ private function asset(string $relative): string { return (string) file_get_contents(base_path('deploy/bootstrap/'.$relative)); } }