345 lines
16 KiB
PHP
345 lines
16 KiB
PHP
<?php
|
||
|
||
namespace App\Provisioning\Steps\Host;
|
||
|
||
use App\Models\Host;
|
||
use App\Models\PlanVersion;
|
||
use App\Models\ProvisioningRun;
|
||
use App\Provisioning\StepResult;
|
||
use App\Services\Proxmox\ProxmoxClient;
|
||
use App\Services\Ssh\RemoteShell;
|
||
use Illuminate\Support\Carbon;
|
||
use Throwable;
|
||
|
||
/**
|
||
* Builds the golden VM template on the host, so that a takeover finishes
|
||
* without a single manual step.
|
||
*
|
||
* This is the other half of VerifyVmTemplate, which deliberately only REPORTED
|
||
* a missing template because what goes into the golden image was a product
|
||
* decision nobody had made yet. It has been made since, and it is written down
|
||
* in deploy/bootstrap/lib/template.sh — 252 lines that check three traps, each
|
||
* of which cost a paid order once.
|
||
*
|
||
* Those lines are SHIPPED, never re-implemented here. A PHP copy of the same
|
||
* checks would be two installations to keep level with every Proxmox release,
|
||
* and the second one only ever reveals itself to whoever runs it. So this step
|
||
* uploads template.sh verbatim, next to its driver and the compose file it
|
||
* bakes in, and runs it.
|
||
*
|
||
* ---------------------------------------------------------------------------
|
||
* Why it runs detached
|
||
* ---------------------------------------------------------------------------
|
||
*
|
||
* Downloading the cloud image and three virt-customize passes take ten to
|
||
* twenty minutes. The per-command SSH timeout is 2000s and the queue job's is
|
||
* 2100s: a synchronous call would be racing both, and losing that race in the
|
||
* middle of virt-customize leaves a half-built image with nobody able to say
|
||
* so. Instead the script is started with nohup and this step comes back every
|
||
* 30 seconds — each poll is its own short job, and poll() does not consume the
|
||
* retry budget.
|
||
*
|
||
* The pid is what makes that honest. `state` on its own says `running` forever
|
||
* once the script dies, because the only process that would have changed it is
|
||
* the one that is gone. So "still running" is decided by `kill -0` against the
|
||
* recorded pid, never by the word in the file — a dead build fails at once
|
||
* instead of being polled against for three quarters of an hour, and a live one
|
||
* is never started a second time (two virt-customize on one file, both calling
|
||
* `qm create 9000`).
|
||
*/
|
||
class BuildVmTemplate extends HostStep
|
||
{
|
||
/** Where the script, its assets and its status files live on the host. */
|
||
public const WORK_DIR = '/var/lib/clupilot/template-build';
|
||
|
||
/** Long enough for a slow mirror and an emulated apt; the step's own deadline is shorter. */
|
||
private const DEADLINE_MINUTES = 45;
|
||
|
||
public function __construct(private RemoteShell $shell, private ProxmoxClient $pve) {}
|
||
|
||
public function key(): string
|
||
{
|
||
return 'build_vm_template';
|
||
}
|
||
|
||
public function maxDuration(): int
|
||
{
|
||
// Above DEADLINE_MINUTES so the step's own deadline decides the failure
|
||
// and can say what went wrong, rather than the generic step timeout —
|
||
// which would additionally spend a retry. Same shape as
|
||
// RebootIntoPveKernel.
|
||
return 3600;
|
||
}
|
||
|
||
public function execute(ProvisioningRun $run): StepResult
|
||
{
|
||
$host = $this->host($run);
|
||
$required = PlanVersion::requiredTemplateVmids();
|
||
|
||
// An installation that takes over its first host before it publishes a
|
||
// plan needs no template. The same real state VerifyVmTemplate lets
|
||
// through, and inventing a requirement here would contradict it.
|
||
if ($required === []) {
|
||
return StepResult::advance();
|
||
}
|
||
|
||
try {
|
||
$missing = $this->templatesStillMissing($host, $required);
|
||
|
||
if ($missing === []) {
|
||
return StepResult::advance();
|
||
}
|
||
} catch (Throwable $e) {
|
||
// VerifyProxmoxApi passed one step ago; a hiccup here is transient.
|
||
// isTemplate() throws rather than answering "absent" for anything
|
||
// short of a successful listing, precisely so this branch — and not
|
||
// a destructive rebuild — is where an outage lands.
|
||
return StepResult::retry(15, 'could not ask Proxmox about the VM templates: '.$e->getMessage());
|
||
}
|
||
|
||
$this->keyLogin($this->shell, $host);
|
||
$status = $this->readStatus();
|
||
|
||
// Nothing on the host is cleaned up when a run ends, so a status file
|
||
// can be older than this run. Read as this run's own, a leftover `ok`
|
||
// says "the build I started finished without producing what I asked
|
||
// for" — and fails a host that only needed a build to be started. That
|
||
// is the ordinary case the day the catalogue gains a second template.
|
||
//
|
||
// The deadline in the run context is the only thing that distinguishes
|
||
// them: it is written when THIS run starts a build, and cleared again
|
||
// by giveUp().
|
||
if ($run->context('template_build_deadline') === null) {
|
||
return $this->adoptOrStart($run, $missing, $status);
|
||
}
|
||
|
||
return match ($status['state']) {
|
||
'' => $this->giveUp(
|
||
$run,
|
||
'The template build was started but left no status behind, so it never really began. '.
|
||
'Check the free space and '.self::WORK_DIR.' on the host, then retry.'
|
||
),
|
||
'running' => $this->whileRunning($run, $status),
|
||
'ok' => $this->giveUp(
|
||
$run,
|
||
'The template build reported success, but VMID '.implode(', ', $missing).
|
||
' is still not a template through the Proxmox API — so a clone would fail. '.
|
||
'Check '.self::WORK_DIR.'/build.log on the host.'
|
||
),
|
||
'failed' => $this->giveUp($run, 'The template build failed: '.($status['note'] ?: 'no reason recorded').'.'),
|
||
default => $this->giveUp($run, 'The template build left an unreadable state ("'.$status['state'].'").'),
|
||
};
|
||
}
|
||
|
||
/**
|
||
* What to do about a status this run did not write.
|
||
*
|
||
* Everything except a LIVE build is simply overwritten by starting a new
|
||
* one: a finished or failed verdict from an earlier run answers a question
|
||
* nobody asked here, and its files are replaced by the start command.
|
||
*
|
||
* A live build is the one exception, and it has to be, or the collision
|
||
* arrives from the other side: two drivers on one build.qcow2, both calling
|
||
* `qm destroy --purge` and `qm create` on the same VMIDs. It is building the
|
||
* same thing this run wants, so it is adopted — with a deadline of its own,
|
||
* so an adopted build cannot be polled forever either.
|
||
*
|
||
* @param array<int, int> $missing
|
||
* @param array{state:string, alive:string, phase:string, note:string} $status
|
||
*/
|
||
private function adoptOrStart(ProvisioningRun $run, array $missing, array $status): StepResult
|
||
{
|
||
if ($status['state'] === 'running' && $status['alive'] === 'yes') {
|
||
$run->mergeContext([
|
||
'template_build_deadline' => now()->addMinutes(self::DEADLINE_MINUTES)->toIso8601String(),
|
||
]);
|
||
|
||
return StepResult::poll(30, 'building the VM template: '.($status['phase'] ?: 'running'));
|
||
}
|
||
|
||
return $this->start($run, $missing);
|
||
}
|
||
|
||
/**
|
||
* The required VMIDs that are NOT already a real template on this node.
|
||
*
|
||
* A list rather than a yes/no, because that list is what gets built — and
|
||
* create_proxmox_template purges a VMID before it creates it. Handing it
|
||
* every required id when only one of them is missing destroys the healthy
|
||
* templates on the way past, and a rebuild that then fails leaves the host
|
||
* with fewer templates than it started with.
|
||
*
|
||
* isTemplate rather than vmExists, and that is the whole point of asking:
|
||
* an abandoned build leaves an ordinary VM numbered 9000 behind. Skipping
|
||
* on the strength of its existence would adopt the leftover as finished,
|
||
* and the gap would surface again at the first paid clone.
|
||
*
|
||
* @param array<int, int> $required
|
||
* @return array<int, int>
|
||
*/
|
||
private function templatesStillMissing(Host $host, array $required): array
|
||
{
|
||
$client = $this->pve->forHost($host);
|
||
$node = $client->listNodes()[0]['node'] ?? 'pve';
|
||
|
||
return array_values(array_filter(
|
||
$required,
|
||
fn (int $vmid) => ! $client->isTemplate($node, $vmid),
|
||
));
|
||
}
|
||
|
||
/** @param array<int, int> $missing */
|
||
private function start(ProvisioningRun $run, array $missing): StepResult
|
||
{
|
||
$this->shell->putFile(self::WORK_DIR.'/template.sh', $this->asset('lib/template.sh'));
|
||
$this->shell->putFile(self::WORK_DIR.'/template-run.sh', $this->asset('lib/template-run.sh'));
|
||
$this->shell->putFile(self::WORK_DIR.'/docker-compose.yml', $this->asset('assets/docker-compose.yml'));
|
||
|
||
// Only what is actually missing, and only what the catalogue asks for.
|
||
// Not a number baked into the script: an installation that repoints a
|
||
// plan at another template would otherwise keep building the old one.
|
||
$this->shell->putFile(self::WORK_DIR.'/env', implode("\n", [
|
||
'# Written by CluPilot at the start of every build. Do not edit.',
|
||
'CLUPILOT_WORK_DIR='.self::WORK_DIR,
|
||
'CLUPILOT_TEMPLATE_VMIDS="'.implode(' ', $missing).'"',
|
||
'',
|
||
]));
|
||
|
||
$run->mergeContext([
|
||
'template_build_deadline' => now()->addMinutes(self::DEADLINE_MINUTES)->toIso8601String(),
|
||
]);
|
||
|
||
// `setsid` puts the build in a session of its own, so the pid the script
|
||
// records is also its PROCESS GROUP id — which is what makes giving up
|
||
// possible later. Killing the driver alone would leave the
|
||
// virt-customize it is waiting on chewing through the image.
|
||
//
|
||
// The pid therefore has to come from the script (`$$`), not from `$!`
|
||
// here: with setsid in between, `$!` can be a process that is already
|
||
// gone. Waiting for the file closes the gap that creates — `state` is
|
||
// written by this command, so a poll arriving before the script had a
|
||
// chance to record itself would otherwise read running-but-not-alive
|
||
// and declare a healthy build dead.
|
||
$this->shell->run(implode("\n", [
|
||
': clupilot-template-start',
|
||
'W='.self::WORK_DIR,
|
||
'mkdir -p "$W"',
|
||
"printf 'running' > \"\$W/state\"",
|
||
"printf 'starting' > \"\$W/phase\"",
|
||
': > "$W/note"',
|
||
': > "$W/pid"',
|
||
'nohup setsid sh "$W/template-run.sh" </dev/null >> "$W/build.log" 2>&1 &',
|
||
'i=0; while [ "$i" -lt 15 ] && [ ! -s "$W/pid" ]; do sleep 1; i=$((i + 1)); done',
|
||
]));
|
||
|
||
return StepResult::poll(30, 'building the VM template (this takes 10–20 minutes)');
|
||
}
|
||
|
||
/** @param array{state:string, alive:string, phase:string, note:string} $status */
|
||
private function whileRunning(ProvisioningRun $run, array $status): StepResult
|
||
{
|
||
if ($status['alive'] !== 'yes') {
|
||
// A reboot, an OOM kill, an operator's Ctrl-C. The file says
|
||
// running and always will — there is nobody left to change it.
|
||
return $this->giveUp(
|
||
$run,
|
||
'The template build is no longer running, but never reported a result — it was killed or the host '.
|
||
'restarted. Check '.self::WORK_DIR.'/build.log and retry.'
|
||
);
|
||
}
|
||
|
||
$deadline = $run->context('template_build_deadline');
|
||
if ($deadline !== null && now()->greaterThan(Carbon::parse($deadline))) {
|
||
return $this->giveUp(
|
||
$run,
|
||
'The template build passed its deadline of '.self::DEADLINE_MINUTES.' minutes while still in phase "'.
|
||
($status['phase'] ?: 'unknown').'". Check '.self::WORK_DIR.'/build.log on the host.'
|
||
);
|
||
}
|
||
|
||
return StepResult::poll(30, 'building the VM template: '.($status['phase'] ?: 'running'));
|
||
}
|
||
|
||
/**
|
||
* Fail, and leave nothing behind that would make a retry read this verdict
|
||
* again — neither the status on the host nor the deadline in the run, which
|
||
* is what startOrGiveUp() reads to tell a first visit from a lost one.
|
||
*
|
||
* The verified cloud image is deliberately kept: it is the one expensive
|
||
* thing a retry does not have to redo, and it is only ever under its final
|
||
* name once its checksum has been confirmed.
|
||
*/
|
||
private function giveUp(ProvisioningRun $run, string $reason): StepResult
|
||
{
|
||
$run->forgetContext('template_build_deadline');
|
||
|
||
$tail = trim($this->shell->run(': clupilot-template-log'."\n".'tail -n 15 '.self::WORK_DIR.'/build.log 2>/dev/null')->stdout);
|
||
|
||
// Stop it BEFORE the status files go, never after. Clearing them makes
|
||
// the step retryable, and a retry that meets a build still running gets
|
||
// two of them on one build.qcow2, both purging and recreating the same
|
||
// VMIDs. The deadline path reaches here with the process very much
|
||
// alive, which is exactly the case this covers.
|
||
//
|
||
// The whole process group (`-$P`), because the driver spends its time
|
||
// waiting on a virt-customize that would otherwise carry on alone. The
|
||
// cmdline guard is against a recycled pid: this file can be minutes old,
|
||
// and killing a stranger's process group would be a far worse bug than
|
||
// the one being handled.
|
||
$this->shell->run(implode("\n", [
|
||
': clupilot-template-reset',
|
||
'W='.self::WORK_DIR,
|
||
'P=$(cat "$W/pid" 2>/dev/null)',
|
||
'if [ -n "$P" ] && kill -0 "$P" 2>/dev/null && tr "\\0" " " < /proc/"$P"/cmdline 2>/dev/null | grep -q template-run; then',
|
||
' kill -TERM -"$P" 2>/dev/null || kill -TERM "$P" 2>/dev/null',
|
||
' i=0; while [ "$i" -lt 10 ] && kill -0 "$P" 2>/dev/null; do sleep 1; i=$((i + 1)); done',
|
||
' kill -KILL -"$P" 2>/dev/null || kill -KILL "$P" 2>/dev/null',
|
||
'fi',
|
||
'rm -f "$W/state" "$W/pid" "$W/note"',
|
||
]));
|
||
|
||
return StepResult::fail($tail === '' ? $reason : $reason.' Last lines: '.$tail);
|
||
}
|
||
|
||
/**
|
||
* The four facts about a build in flight, in one round trip.
|
||
*
|
||
* `alive` is computed on the host because that is the only place the answer
|
||
* exists: the pid means nothing here.
|
||
*
|
||
* @return array{state:string, alive:string, phase:string, note:string}
|
||
*/
|
||
private function readStatus(): array
|
||
{
|
||
$out = $this->shell->run(implode("\n", [
|
||
': clupilot-template-status',
|
||
'W='.self::WORK_DIR,
|
||
'S=$(cat "$W/state" 2>/dev/null)',
|
||
'P=$(cat "$W/pid" 2>/dev/null)',
|
||
'A=no',
|
||
'if [ -n "$P" ] && kill -0 "$P" 2>/dev/null; then A=yes; fi',
|
||
"printf 'state=%s\\n' \"\$S\"",
|
||
"printf 'alive=%s\\n' \"\$A\"",
|
||
"printf 'phase=%s\\n' \"\$(head -1 \"\$W/phase\" 2>/dev/null)\"",
|
||
"printf 'note=%s\\n' \"\$(head -1 \"\$W/note\" 2>/dev/null)\"",
|
||
]))->stdout;
|
||
|
||
$status = ['state' => '', 'alive' => 'no', 'phase' => '', 'note' => ''];
|
||
|
||
foreach (preg_split('/\R/', $out) ?: [] as $line) {
|
||
[$key, $value] = array_pad(explode('=', $line, 2), 2, '');
|
||
if (array_key_exists($key, $status)) {
|
||
$status[$key] = trim($value);
|
||
}
|
||
}
|
||
|
||
return $status;
|
||
}
|
||
|
||
/** The shipped file, read from the repo — never rendered from a string in here. */
|
||
private function asset(string $relative): string
|
||
{
|
||
return (string) file_get_contents(base_path('deploy/bootstrap/'.$relative));
|
||
}
|
||
}
|