host($run); // The connection dropping is the EXPECTED case here, not an error: // `ifreload -a` takes the line this step is speaking over. Left // uncaught, RunRunner turns the exception into retry() — and retry // spends the attempt budget, five of which are gone in minutes, long // before the host is back. try { $this->keyLogin($this->shell, $host); } catch (Throwable $e) { return $this->whileDisconnected($run, $e); } $state = $this->readBridgeState(); // Already there means touch nothing. The same shortcut as "the template // exists and reports template: 1" — and here it matters more, because // rebuilding a working network is the one thing that can end this badly. if ($state['up']) { return StepResult::advance(); } if ($state['iface'] === '') { return StepResult::fail( 'No interface on this host carries the default route, so there is nothing to derive a bridge '. 'from. Check the machine on the provider console.' ); } // bridge_ports on a bond or an existing bridge is wrong: the bridge // would take its own substrate as a port, and what comes out of that // cannot be repaired from a remote shell. if (! $state['physical']) { return StepResult::fail( 'The interface carrying the default route ('.$state['iface'].') is not a physical NIC — it is a '. 'bridge, a bond or a VLAN. CluPilot will not put a bridge on top of it. Build vmbr0 by hand in '. '/etc/network/interfaces (bridge_ports = the NIC carrying the default route, host address and '. 'gateway moved onto vmbr0), apply it with `ifreload -a`, confirm you still have SSH, then retry. '. 'Current default route: '.($state['route'] ?: '(none reported)').'.' ); } // Nothing on the host is cleaned up when a run ends, so a status file // can be older than this run. The deadline in the run context is the // only thing that tells a first visit from a lost one: it is written // when THIS run starts a build, and cleared again by giveUp(). if ($run->context('bridge_deadline') === null) { return $this->start($run); } $status = $this->readStatus(); return match ($status['state']) { '' => $this->giveUp( $run, 'The bridge build was started but left no status behind, so it never really began. Check '. self::WORK_DIR.' and the free space on the host, then retry.' ), 'running' => $this->whileRunning($run, $status), // Reaching here means the driver said ok while vmbr0 does NOT carry // the default route — so the timer got there first, or is about to. 'ok' => $this->awaitRollback($run, $status, 'The bridge build reported success, but vmbr0 does not carry the default route.'), 'failed' => $this->awaitRollback($run, $status, 'The bridge build failed: '.($status['note'] ?: 'no reason recorded').'.'), default => $this->giveUp($run, 'The bridge build left an unreadable state ("'.$status['state'].'").'), }; } /** * The host did not answer. * * With a deadline in the context, this run has changed the network and the * silence is the change taking effect — poll, do not retry. Without one, * nothing has been touched and an unreachable host is an ordinary * connection error that may cost an attempt. */ private function whileDisconnected(ProvisioningRun $run, Throwable $e): StepResult { $deadline = $run->context('bridge_deadline'); if ($deadline === null) { return StepResult::retry(20, 'host not reachable: '.$e->getMessage()); } if (now()->greaterThan(Carbon::parse($deadline))) { $run->forgetContext('bridge_deadline'); return StepResult::fail( 'The host did not come back over the tunnel after the bridge was applied, and the deadline of '. self::DEADLINE_MINUTES.' minutes has passed. The rollback timer will have restored the previous '. 'network configuration '.self::ROLLBACK_MINUTES.' minutes after the change, so the machine should '. 'be reachable on its old settings — check it, then retry. Last error: '.$e->getMessage() ); } return StepResult::poll(20, 'warte darauf, dass der Host über die neue Brücke wieder antwortet'); } /** Ship the library, write the env, launch detached. */ private function start(ProvisioningRun $run): StepResult { $attempts = (int) $run->context('bridge_attempts', 0); if ($attempts >= self::MAX_ATTEMPTS) { return StepResult::fail( 'CluPilot tried to build vmbr0 automatically '.$attempts.' times, and each time the host did not '. 'come back both ways; the previous network configuration was restored. Build the bridge by hand '. 'in /etc/network/interfaces (bridge_ports = the NIC carrying the default route, host address and '. 'gateway moved onto vmbr0), apply it with `ifreload -a`, confirm you still have SSH, then retry '. 'this run. The stanza CluPilot wrote is in '.self::WORK_DIR.'/bridge.log.' ); } $this->shell->putFile(self::WORK_DIR.'/bridge.sh', $this->asset('lib/bridge.sh')); $this->shell->putFile(self::WORK_DIR.'/bridge-run.sh', $this->asset('lib/bridge-run.sh')); // The hub key so the driver can check the handshake against the RIGHT // peer: staff peers live on the same hub, and their handshake says // nothing about CluPilot's own way in. $this->shell->putFile(self::WORK_DIR.'/env', implode("\n", [ '# Written by CluPilot at the start of every bridge build. Do not edit.', 'CLUPILOT_WORK_DIR='.self::WORK_DIR, 'CLUPILOT_ROLLBACK_MINUTES='.self::ROLLBACK_MINUTES, 'CLUPILOT_WG_HUB_PUBKEY='.escapeshellarg(trim($this->hub->publicKey())), '', ])); $run->mergeContext([ 'bridge_deadline' => now()->addMinutes(self::DEADLINE_MINUTES)->toIso8601String(), 'bridge_attempts' => $attempts + 1, ]); // `setsid` puts the driver in a session of its own, so the pid it // records is also its PROCESS GROUP id — which is what makes giving up // possible later. The pid therefore comes from the script (`$$`), not // from `$!` here: with setsid in between, `$!` can be a process that is // already gone. Waiting for the file closes the gap that creates. // // `rolled-back` is removed first: it is the signal that a PREVIOUS // attempt's timer fired, and a leftover would make this attempt read its // own change as already rolled back. $this->shell->run(implode("\n", [ ': clupilot-bridge-start', 'W='.self::WORK_DIR, 'mkdir -p "$W"', 'rm -f "$W/rolled-back"', "printf 'running' > \"\$W/state\"", "printf 'starting' > \"\$W/phase\"", ': > "$W/note"', ': > "$W/pid"', 'nohup setsid sh "$W/bridge-run.sh" > "$W/bridge.log" 2>&1 &', 'i=0; while [ "$i" -lt 15 ] && [ ! -s "$W/pid" ]; do sleep 1; i=$((i + 1)); done', ])); return StepResult::poll(20, 'Netzbrücke wird gebaut'); } /** @param array{state:string, alive:string, phase:string, note:string, rolledback:string} $status */ private function whileRunning(ProvisioningRun $run, array $status): StepResult { if ($status['alive'] !== 'yes') { // A kill, an OOM, a reboot. The file says running and always will — // there is nobody left to change it. The timer is still armed, so // the machine gets its old configuration back either way. return $this->awaitRollback( $run, $status, 'The bridge build stopped running without reporting a result — it was killed or the host restarted.' ); } $deadline = $run->context('bridge_deadline'); if ($deadline !== null && now()->greaterThan(Carbon::parse($deadline))) { return $this->giveUp( $run, 'The bridge build passed its deadline of '.self::DEADLINE_MINUTES.' minutes while still in phase "'. ($status['phase'] ?: 'unknown').'". Check '.self::WORK_DIR.'/bridge.log on the host.' ); } return StepResult::poll(20, 'Netzbrücke wird gebaut: '.($status['phase'] ?: 'läuft')); } /** * The five facts about a build in flight, in one round trip. * * `alive` is computed on the host because that is the only place the answer * exists: the pid means nothing here. * * @return array{state:string, alive:string, phase:string, note:string, rolledback:string} */ private function readStatus(): array { $out = $this->shell->run(implode("\n", [ ': clupilot-bridge-status', 'W='.self::WORK_DIR, 'S=$(cat "$W/state" 2>/dev/null)', 'P=$(cat "$W/pid" 2>/dev/null)', 'A=no', 'if [ -n "$P" ] && kill -0 "$P" 2>/dev/null; then A=yes; fi', 'R=no', 'if [ -f "$W/rolled-back" ]; then R=yes; fi', "printf 'state=%s\\n' \"\$S\"", "printf 'alive=%s\\n' \"\$A\"", "printf 'phase=%s\\n' \"\$(head -1 \"\$W/phase\" 2>/dev/null)\"", "printf 'note=%s\\n' \"\$(head -1 \"\$W/note\" 2>/dev/null)\"", "printf 'rolledback=%s\\n' \"\$R\"", ]))->stdout; $status = ['state' => '', 'alive' => 'no', 'phase' => '', 'note' => '', 'rolledback' => 'no']; foreach (preg_split('/\R/', $out) ?: [] as $line) { [$key, $value] = array_pad(explode('=', $line, 2), 2, ''); if (array_key_exists($key, $status)) { $status[$key] = trim($value); } } return $status; } /** The shipped file, read from the repo — never rendered from a string in here. */ private function asset(string $relative): string { return (string) file_get_contents(base_path('deploy/bootstrap/'.$relative)); } /** @param array{state:string, alive:string, phase:string, note:string, rolledback:string} $status */ private function awaitRollback(ProvisioningRun $run, array $status, string $reason): StepResult { return $this->giveUp($run, $reason); } private function giveUp(ProvisioningRun $run, string $reason): StepResult { $run->forgetContext('bridge_deadline'); return StepResult::fail($reason); } /** * What the host says about its bridge and its primary NIC, in one round trip. * * Derived from the RUNNING state — `ip`, `/sys/class/net` — never from the * provider's file. What runs is structured the same everywhere; how it was * written down is not, and that is the part that carries Hetzner and netcup * at once. * * "up" is deliberately all three facts and not `ip link show`: a vmbr0 with * no address and no default route is a bridge in the sense of `ip link` and * nothing else. The step this replaces checked exactly that and threw the * answer away. * * @return array{up:bool, iface:string, physical:bool, route:string} */ private function readBridgeState(): array { $out = $this->shell->run(implode("\n", [ ': clupilot-bridge-state', 'B=vmbr0', 'IF=$(ip -4 route show default 2>/dev/null | awk \'{ for (i = 1; i < NF; i++) if ($i == "dev") { print $(i+1); exit } }\')', 'UP=no', 'if ip link show "$B" >/dev/null 2>&1 && [ "$IF" = "$B" ] && [ -n "$(ip -4 -o addr show dev "$B" scope global 2>/dev/null)" ]; then UP=yes; fi', 'PHY=no', 'if [ -n "$IF" ] && [ -e "/sys/class/net/$IF/device" ] && [ ! -d "/sys/class/net/$IF/bridge" ] && [ ! -d "/sys/class/net/$IF/bonding" ]; then PHY=yes; fi', "printf 'up=%s\\n' \"\$UP\"", "printf 'iface=%s\\n' \"\$IF\"", "printf 'physical=%s\\n' \"\$PHY\"", "printf 'route=%s\\n' \"\$(ip -4 route show default 2>/dev/null | head -1)\"", ]))->stdout; $parsed = ['up' => 'no', 'iface' => '', 'physical' => 'no', 'route' => '']; foreach (preg_split('/\R/', $out) ?: [] as $line) { [$key, $value] = array_pad(explode('=', $line, 2), 2, ''); if (array_key_exists($key, $parsed)) { $parsed[$key] = trim($value); } } return [ 'up' => $parsed['up'] === 'yes', 'iface' => $parsed['iface'], 'physical' => $parsed['physical'] === 'yes', 'route' => $parsed['route'], ]; } }