476 lines
22 KiB
PHP
476 lines
22 KiB
PHP
<?php
|
|
|
|
namespace App\Provisioning\Steps\Host;
|
|
|
|
use App\Models\ProvisioningRun;
|
|
use App\Provisioning\StepResult;
|
|
use App\Services\Ssh\RemoteShell;
|
|
use App\Services\Wireguard\WireguardHub;
|
|
use Illuminate\Support\Carbon;
|
|
use Throwable;
|
|
|
|
/**
|
|
* Builds vmbr0 on a Debian-installed Proxmox host, under a timer that puts the
|
|
* old network configuration back if the bridge takes the machine off the net.
|
|
*
|
|
* Proxmox installed on top of Debian does not create vmbr0 — only the ISO
|
|
* installer writes that bridge into /etc/network/interfaces. BuildVmTemplate
|
|
* creates the golden image with `--net0 virtio,bridge=vmbr0` and every clone
|
|
* inherits it, so a host without the bridge serves no customer at all.
|
|
*
|
|
* ConfigureProxmox used to refuse here and send an operator to the keyboard.
|
|
* That was right while nobody could do this safely from a remote shell. What
|
|
* makes it safe is not cleverness about network layouts — it is the way back.
|
|
*
|
|
* ---------------------------------------------------------------------------
|
|
* The step saws through the branch it is sitting on
|
|
* ---------------------------------------------------------------------------
|
|
*
|
|
* By this point SSH runs over the WireGuard tunnel (keyLogin prefers wg_ip once
|
|
* the handshake is proven, and RebootIntoPveKernel has proven it), and wg0 hangs
|
|
* off the same NIC that is about to become a bridge port. A wrong bridge takes
|
|
* the tunnel AND the public address with it.
|
|
*
|
|
* So the order is: arm the rollback timer, change, reconnect, check, cancel. If
|
|
* the connection does not come back, the timer restores the old configuration
|
|
* and this step lands in a retry rather than on a dead server.
|
|
*
|
|
* ---------------------------------------------------------------------------
|
|
* Why it runs detached
|
|
* ---------------------------------------------------------------------------
|
|
*
|
|
* Not because it is slow — the build takes seconds. Because `ifreload -a` takes
|
|
* the line the command itself is running over. A synchronous call would not
|
|
* return a value; it would return a corpse. So the driver is started with
|
|
* `nohup setsid` and this step comes back every 20 seconds, each poll its own
|
|
* short job — and poll() does not consume the retry budget.
|
|
*
|
|
* The shell library is SHIPPED, never re-implemented here — same as
|
|
* BuildVmTemplate. A PHP copy of the provider-shape rules would be two
|
|
* installations to keep level with every Proxmox release, and the second one
|
|
* only ever reveals itself to whoever runs it.
|
|
*/
|
|
class EnsureNetworkBridge extends HostStep
|
|
{
|
|
/** Where the library, its env and its status files live on the host. */
|
|
public const WORK_DIR = '/var/lib/clupilot/bridge';
|
|
|
|
/** The rollback timer's fuse, in minutes. */
|
|
private const ROLLBACK_MINUTES = 5;
|
|
|
|
/**
|
|
* The step's own deadline — deliberately three times the timer's fuse.
|
|
*
|
|
* By the time this step gives up, the rollback has necessarily already run,
|
|
* so "gave up" always means "the host is back on its old configuration".
|
|
* That is the line that makes "a retry rather than a dead server" true.
|
|
*/
|
|
private const DEADLINE_MINUTES = 15;
|
|
|
|
/** One automatic attempt, plus one for the case an operator fixed something. */
|
|
private const MAX_ATTEMPTS = 2;
|
|
|
|
public function __construct(private RemoteShell $shell, private WireguardHub $hub) {}
|
|
|
|
public function key(): string
|
|
{
|
|
return 'ensure_network_bridge';
|
|
}
|
|
|
|
public function maxDuration(): int
|
|
{
|
|
// Above DEADLINE_MINUTES so the step's own deadline decides the failure
|
|
// and can say what went wrong, rather than the generic step timeout —
|
|
// which would additionally spend a retry. Same shape as
|
|
// RebootIntoPveKernel and BuildVmTemplate.
|
|
return 3600;
|
|
}
|
|
|
|
public function execute(ProvisioningRun $run): StepResult
|
|
{
|
|
$host = $this->host($run);
|
|
|
|
// The connection dropping is the EXPECTED case here, not an error:
|
|
// `ifreload -a` takes the line this step is speaking over. Left
|
|
// uncaught, RunRunner turns the exception into retry() — and retry
|
|
// spends the attempt budget, five of which are gone in minutes, long
|
|
// before the host is back.
|
|
try {
|
|
$this->keyLogin($this->shell, $host);
|
|
} catch (Throwable $e) {
|
|
return $this->whileDisconnected($run, $e);
|
|
}
|
|
|
|
$state = $this->readBridgeState();
|
|
|
|
// Already there means touch nothing. The same shortcut as "the template
|
|
// exists and reports template: 1" — and here it matters more, because
|
|
// rebuilding a working network is the one thing that can end this badly.
|
|
//
|
|
// It is also the success path. Reaching this line means SSH arrived over
|
|
// the tunnel AND vmbr0 carries the default route with an address — that
|
|
// is bridge_proven, asked from the outside, which is the strongest form
|
|
// of the question there is. Only now is the timer thrown away, and only
|
|
// if this run is the one that armed it: without a deadline in the
|
|
// context, those unit files belong to somebody else.
|
|
if ($state['up']) {
|
|
if ($run->context('bridge_deadline') !== null) {
|
|
$this->cancelRollback();
|
|
$run->forgetContext('bridge_deadline');
|
|
}
|
|
|
|
return StepResult::advance();
|
|
}
|
|
|
|
if ($state['iface'] === '') {
|
|
return StepResult::fail(
|
|
'No interface on this host carries the default route, so there is nothing to derive a bridge '.
|
|
'from. Check the machine on the provider console.'
|
|
);
|
|
}
|
|
|
|
// bridge_ports on a bond or an existing bridge is wrong: the bridge
|
|
// would take its own substrate as a port, and what comes out of that
|
|
// cannot be repaired from a remote shell.
|
|
if (! $state['physical']) {
|
|
return StepResult::fail(
|
|
'The interface carrying the default route ('.$state['iface'].') is not a physical NIC — it is a '.
|
|
'bridge, a bond or a VLAN. CluPilot will not put a bridge on top of it. Build vmbr0 by hand in '.
|
|
'/etc/network/interfaces (bridge_ports = the NIC carrying the default route, host address and '.
|
|
'gateway moved onto vmbr0), apply it with `ifreload -a`, confirm you still have SSH, then retry. '.
|
|
'Current default route: '.($state['route'] ?: '(none reported)').'.'
|
|
);
|
|
}
|
|
|
|
// Nothing on the host is cleaned up when a run ends, so a status file
|
|
// can be older than this run. The deadline in the run context is the
|
|
// only thing that tells a first visit from a lost one: it is written
|
|
// when THIS run starts a build, and cleared again by giveUp().
|
|
if ($run->context('bridge_deadline') === null) {
|
|
return $this->start($run);
|
|
}
|
|
|
|
$status = $this->readStatus();
|
|
|
|
return match ($status['state']) {
|
|
'' => $this->giveUp(
|
|
$run,
|
|
'The bridge build was started but left no status behind, so it never really began. Check '.
|
|
self::WORK_DIR.' and the free space on the host, then retry.'
|
|
),
|
|
'running' => $this->whileRunning($run, $status),
|
|
// Reaching here means the driver said ok while vmbr0 does NOT carry
|
|
// the default route — so the timer got there first, or is about to.
|
|
'ok' => $this->awaitRollback($run, $status, 'The bridge build reported success, but vmbr0 does not carry the default route.'),
|
|
'failed' => $this->awaitRollback($run, $status, 'The bridge build failed: '.($status['note'] ?: 'no reason recorded').'.'),
|
|
default => $this->giveUp($run, 'The bridge build left an unreadable state ("'.$status['state'].'").'),
|
|
};
|
|
}
|
|
|
|
/**
|
|
* The host did not answer.
|
|
*
|
|
* With a deadline in the context, this run has changed the network and the
|
|
* silence is the change taking effect — poll, do not retry. Without one,
|
|
* nothing has been touched and an unreachable host is an ordinary
|
|
* connection error that may cost an attempt.
|
|
*/
|
|
private function whileDisconnected(ProvisioningRun $run, Throwable $e): StepResult
|
|
{
|
|
$deadline = $run->context('bridge_deadline');
|
|
|
|
if ($deadline === null) {
|
|
return StepResult::retry(20, 'host not reachable: '.$e->getMessage());
|
|
}
|
|
|
|
if (now()->greaterThan(Carbon::parse($deadline))) {
|
|
$run->forgetContext('bridge_deadline');
|
|
|
|
return StepResult::fail(
|
|
'The host did not come back over the tunnel after the bridge was applied, and the deadline of '.
|
|
self::DEADLINE_MINUTES.' minutes has passed. The rollback timer will have restored the previous '.
|
|
'network configuration '.self::ROLLBACK_MINUTES.' minutes after the change, so the machine should '.
|
|
'be reachable on its old settings — check it, then retry. Last error: '.$e->getMessage()
|
|
);
|
|
}
|
|
|
|
return StepResult::poll(20, 'warte darauf, dass der Host über die neue Brücke wieder antwortet');
|
|
}
|
|
|
|
/** Ship the library, write the env, launch detached. */
|
|
private function start(ProvisioningRun $run): StepResult
|
|
{
|
|
$attempts = (int) $run->context('bridge_attempts', 0);
|
|
|
|
if ($attempts >= self::MAX_ATTEMPTS) {
|
|
return StepResult::fail(
|
|
'CluPilot tried to build vmbr0 automatically '.$attempts.' times, and each time the host did not '.
|
|
'come back both ways; the previous network configuration was restored. Build the bridge by hand '.
|
|
'in /etc/network/interfaces (bridge_ports = the NIC carrying the default route, host address and '.
|
|
'gateway moved onto vmbr0), apply it with `ifreload -a`, confirm you still have SSH, then retry '.
|
|
'this run. The stanza CluPilot wrote is in '.self::WORK_DIR.'/bridge.log.'
|
|
);
|
|
}
|
|
|
|
$this->shell->putFile(self::WORK_DIR.'/bridge.sh', $this->asset('lib/bridge.sh'));
|
|
$this->shell->putFile(self::WORK_DIR.'/bridge-run.sh', $this->asset('lib/bridge-run.sh'));
|
|
|
|
// The hub key so the driver can check the handshake against the RIGHT
|
|
// peer: staff peers live on the same hub, and their handshake says
|
|
// nothing about CluPilot's own way in.
|
|
$this->shell->putFile(self::WORK_DIR.'/env', implode("\n", [
|
|
'# Written by CluPilot at the start of every bridge build. Do not edit.',
|
|
'CLUPILOT_WORK_DIR='.self::WORK_DIR,
|
|
'CLUPILOT_ROLLBACK_MINUTES='.self::ROLLBACK_MINUTES,
|
|
'CLUPILOT_WG_HUB_PUBKEY='.escapeshellarg(trim($this->hub->publicKey())),
|
|
'',
|
|
]));
|
|
|
|
$run->mergeContext([
|
|
'bridge_deadline' => now()->addMinutes(self::DEADLINE_MINUTES)->toIso8601String(),
|
|
'bridge_attempts' => $attempts + 1,
|
|
]);
|
|
|
|
// `setsid` puts the driver in a session of its own, so the pid it
|
|
// records is also its PROCESS GROUP id — which is what makes giving up
|
|
// possible later. The pid therefore comes from the script (`$$`), not
|
|
// from `$!` here: with setsid in between, `$!` can be a process that is
|
|
// already gone. Waiting for the file closes the gap that creates.
|
|
//
|
|
// `rolled-back` is removed first: it is the signal that a PREVIOUS
|
|
// attempt's timer fired, and a leftover would make this attempt read its
|
|
// own change as already rolled back.
|
|
$this->shell->run(implode("\n", [
|
|
': clupilot-bridge-start',
|
|
'W='.self::WORK_DIR,
|
|
'mkdir -p "$W"',
|
|
'rm -f "$W/rolled-back"',
|
|
"printf 'running' > \"\$W/state\"",
|
|
"printf 'starting' > \"\$W/phase\"",
|
|
': > "$W/note"',
|
|
': > "$W/pid"',
|
|
'nohup setsid sh "$W/bridge-run.sh" </dev/null >> "$W/bridge.log" 2>&1 &',
|
|
'i=0; while [ "$i" -lt 15 ] && [ ! -s "$W/pid" ]; do sleep 1; i=$((i + 1)); done',
|
|
]));
|
|
|
|
return StepResult::poll(20, 'Netzbrücke wird gebaut');
|
|
}
|
|
|
|
/** @param array{state:string, alive:string, phase:string, note:string, rolledback:string} $status */
|
|
private function whileRunning(ProvisioningRun $run, array $status): StepResult
|
|
{
|
|
if ($status['alive'] !== 'yes') {
|
|
// A kill, an OOM, a reboot. The file says running and always will —
|
|
// there is nobody left to change it. The timer is still armed, so
|
|
// the machine gets its old configuration back either way.
|
|
return $this->awaitRollback(
|
|
$run,
|
|
$status,
|
|
'The bridge build stopped running without reporting a result — it was killed or the host restarted.'
|
|
);
|
|
}
|
|
|
|
$deadline = $run->context('bridge_deadline');
|
|
if ($deadline !== null && now()->greaterThan(Carbon::parse($deadline))) {
|
|
return $this->giveUp(
|
|
$run,
|
|
'The bridge build passed its deadline of '.self::DEADLINE_MINUTES.' minutes while still in phase "'.
|
|
($status['phase'] ?: 'unknown').'". Check '.self::WORK_DIR.'/bridge.log on the host.'
|
|
);
|
|
}
|
|
|
|
return StepResult::poll(20, 'Netzbrücke wird gebaut: '.($status['phase'] ?: 'läuft'));
|
|
}
|
|
|
|
/**
|
|
* The five facts about a build in flight, in one round trip.
|
|
*
|
|
* `alive` is computed on the host because that is the only place the answer
|
|
* exists: the pid means nothing here.
|
|
*
|
|
* @return array{state:string, alive:string, phase:string, note:string, rolledback:string}
|
|
*/
|
|
private function readStatus(): array
|
|
{
|
|
$out = $this->shell->run(implode("\n", [
|
|
': clupilot-bridge-status',
|
|
'W='.self::WORK_DIR,
|
|
'S=$(cat "$W/state" 2>/dev/null)',
|
|
'P=$(cat "$W/pid" 2>/dev/null)',
|
|
'A=no',
|
|
'if [ -n "$P" ] && kill -0 "$P" 2>/dev/null; then A=yes; fi',
|
|
'R=no',
|
|
'if [ -f "$W/rolled-back" ]; then R=yes; fi',
|
|
"printf 'state=%s\\n' \"\$S\"",
|
|
"printf 'alive=%s\\n' \"\$A\"",
|
|
"printf 'phase=%s\\n' \"\$(head -1 \"\$W/phase\" 2>/dev/null)\"",
|
|
"printf 'note=%s\\n' \"\$(head -1 \"\$W/note\" 2>/dev/null)\"",
|
|
"printf 'rolledback=%s\\n' \"\$R\"",
|
|
]))->stdout;
|
|
|
|
$status = ['state' => '', 'alive' => 'no', 'phase' => '', 'note' => '', 'rolledback' => 'no'];
|
|
|
|
foreach (preg_split('/\R/', $out) ?: [] as $line) {
|
|
[$key, $value] = array_pad(explode('=', $line, 2), 2, '');
|
|
if (array_key_exists($key, $status)) {
|
|
$status[$key] = trim($value);
|
|
}
|
|
}
|
|
|
|
return $status;
|
|
}
|
|
|
|
/** The shipped file, read from the repo — never rendered from a string in here. */
|
|
private function asset(string $relative): string
|
|
{
|
|
return (string) file_get_contents(base_path('deploy/bootstrap/'.$relative));
|
|
}
|
|
|
|
/**
|
|
* Wait for the rollback to complete, and only then fail.
|
|
*
|
|
* Not immediately: while the timer is still pending, the machine is in the
|
|
* middle of a change. A fail() would leave it there, and the next person to
|
|
* look would find half a host with no explanation of why.
|
|
*
|
|
* The rollback script writes `failed` plus a reason BEFORE it restores, so
|
|
* even a partial restore leaves a verdict, and touches `rolled-back` as its
|
|
* LAST action. That file is the signal: it is only there once the old
|
|
* configuration is genuinely back.
|
|
*
|
|
* The wait is capped by the step's deadline, which is three times the
|
|
* timer's fuse — so this cannot poll forever.
|
|
*
|
|
* @param array{state:string, alive:string, phase:string, note:string, rolledback:string} $status
|
|
*/
|
|
private function awaitRollback(ProvisioningRun $run, array $status, string $reason): StepResult
|
|
{
|
|
$deadline = $run->context('bridge_deadline');
|
|
$expired = $deadline !== null && now()->greaterThan(Carbon::parse($deadline));
|
|
|
|
if ($status['rolledback'] !== 'yes' && ! $expired) {
|
|
return StepResult::poll(20, 'die Brücke trägt nicht — warte auf die Rücknahme des Zeitgebers');
|
|
}
|
|
|
|
return $this->giveUp(
|
|
$run,
|
|
$reason.' The rollback timer has put the previous network configuration back'.
|
|
($status['rolledback'] === 'yes'
|
|
? ''
|
|
: ' (or is about to — its fuse is '.self::ROLLBACK_MINUTES.' minutes)').
|
|
', so the machine is reachable on its old settings.'
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Throw the return ticket away — and never before the bridge has been
|
|
* checked. Reaching the caller means SSH arrived over the tunnel AND vmbr0
|
|
* carries the default route with an address, which is bridge_proven asked
|
|
* from the outside, the strongest form of the question there is.
|
|
*/
|
|
private function cancelRollback(): void
|
|
{
|
|
$this->shell->run(implode("\n", [
|
|
': clupilot-bridge-cancel',
|
|
'U=clupilot-network-rollback',
|
|
'systemctl stop "$U.timer" 2>/dev/null || true',
|
|
'rm -f "/etc/systemd/system/$U.timer" "/etc/systemd/system/$U.service" "/usr/local/sbin/$U.sh"',
|
|
'systemctl daemon-reload 2>/dev/null || true',
|
|
]));
|
|
}
|
|
|
|
/**
|
|
* Fail, and leave nothing behind that would make a retry read this verdict
|
|
* again — the status on the host and the deadline in the run, which is what
|
|
* execute() reads to tell a first visit from a lost one.
|
|
*
|
|
* `bridge_attempts` deliberately SURVIVES: it is the cap on how often
|
|
* CluPilot will drive a live host's network into a rollback, and clearing it
|
|
* here would make that cap meaningless.
|
|
*
|
|
* The rollback timer is deliberately NOT cancelled. It is the return ticket;
|
|
* if it is still armed, that is because nobody has proven the host came back
|
|
* — and cancelling it while giving up would leave the machine in exactly the
|
|
* state the timer exists for.
|
|
*/
|
|
private function giveUp(ProvisioningRun $run, string $reason): StepResult
|
|
{
|
|
$run->forgetContext('bridge_deadline');
|
|
|
|
$tail = trim($this->shell->run(
|
|
': clupilot-bridge-log'."\n".'tail -n 25 '.self::WORK_DIR.'/bridge.log 2>/dev/null'
|
|
)->stdout);
|
|
|
|
// Stop the driver BEFORE the status files go, never after. Clearing them
|
|
// makes the step retryable, and a retry meeting a driver still running
|
|
// gets two of them on one /etc/network/interfaces, each with a timer of
|
|
// its own — one cancelling what the other armed.
|
|
//
|
|
// The whole process group (`-$P`), because the driver spends its time
|
|
// waiting on an ifreload. The cmdline guard is against a recycled pid:
|
|
// this file can be minutes old, and killing a stranger's process group
|
|
// would be a far worse bug than the one being handled.
|
|
$this->shell->run(implode("\n", [
|
|
': clupilot-bridge-reset',
|
|
'W='.self::WORK_DIR,
|
|
'P=$(cat "$W/pid" 2>/dev/null)',
|
|
'if [ -n "$P" ] && kill -0 "$P" 2>/dev/null && tr "\\0" " " < /proc/"$P"/cmdline 2>/dev/null | grep -q bridge-run; then',
|
|
' kill -TERM -"$P" 2>/dev/null || kill -TERM "$P" 2>/dev/null',
|
|
' i=0; while [ "$i" -lt 10 ] && kill -0 "$P" 2>/dev/null; do sleep 1; i=$((i + 1)); done',
|
|
' kill -KILL -"$P" 2>/dev/null || kill -KILL "$P" 2>/dev/null',
|
|
'fi',
|
|
'rm -f "$W/state" "$W/pid" "$W/note" "$W/rolled-back"',
|
|
]));
|
|
|
|
return StepResult::fail($tail === '' ? $reason : $reason.' Last lines: '.$tail);
|
|
}
|
|
|
|
/**
|
|
* What the host says about its bridge and its primary NIC, in one round trip.
|
|
*
|
|
* Derived from the RUNNING state — `ip`, `/sys/class/net` — never from the
|
|
* provider's file. What runs is structured the same everywhere; how it was
|
|
* written down is not, and that is the part that carries Hetzner and netcup
|
|
* at once.
|
|
*
|
|
* "up" is deliberately all three facts and not `ip link show`: a vmbr0 with
|
|
* no address and no default route is a bridge in the sense of `ip link` and
|
|
* nothing else. The step this replaces checked exactly that and threw the
|
|
* answer away.
|
|
*
|
|
* @return array{up:bool, iface:string, physical:bool, route:string}
|
|
*/
|
|
private function readBridgeState(): array
|
|
{
|
|
$out = $this->shell->run(implode("\n", [
|
|
': clupilot-bridge-state',
|
|
'B=vmbr0',
|
|
'IF=$(ip -4 route show default 2>/dev/null | awk \'{ for (i = 1; i < NF; i++) if ($i == "dev") { print $(i+1); exit } }\')',
|
|
'UP=no',
|
|
'if ip link show "$B" >/dev/null 2>&1 && [ "$IF" = "$B" ] && [ -n "$(ip -4 -o addr show dev "$B" scope global 2>/dev/null)" ]; then UP=yes; fi',
|
|
'PHY=no',
|
|
'if [ -n "$IF" ] && [ -e "/sys/class/net/$IF/device" ] && [ ! -d "/sys/class/net/$IF/bridge" ] && [ ! -d "/sys/class/net/$IF/bonding" ]; then PHY=yes; fi',
|
|
"printf 'up=%s\\n' \"\$UP\"",
|
|
"printf 'iface=%s\\n' \"\$IF\"",
|
|
"printf 'physical=%s\\n' \"\$PHY\"",
|
|
"printf 'route=%s\\n' \"\$(ip -4 route show default 2>/dev/null | head -1)\"",
|
|
]))->stdout;
|
|
|
|
$parsed = ['up' => 'no', 'iface' => '', 'physical' => 'no', 'route' => ''];
|
|
|
|
foreach (preg_split('/\R/', $out) ?: [] as $line) {
|
|
[$key, $value] = array_pad(explode('=', $line, 2), 2, '');
|
|
if (array_key_exists($key, $parsed)) {
|
|
$parsed[$key] = trim($value);
|
|
}
|
|
}
|
|
|
|
return [
|
|
'up' => $parsed['up'] === 'yes',
|
|
'iface' => $parsed['iface'],
|
|
'physical' => $parsed['physical'] === 'yes',
|
|
'route' => $parsed['route'],
|
|
];
|
|
}
|
|
}
|