diff --git a/app/Provisioning/Steps/Host/EnsureNetworkBridge.php b/app/Provisioning/Steps/Host/EnsureNetworkBridge.php index 49085ce..7a2d022 100644 --- a/app/Provisioning/Steps/Host/EnsureNetworkBridge.php +++ b/app/Provisioning/Steps/Host/EnsureNetworkBridge.php @@ -5,6 +5,9 @@ namespace App\Provisioning\Steps\Host; use App\Models\ProvisioningRun; use App\Provisioning\StepResult; use App\Services\Ssh\RemoteShell; +use App\Services\Wireguard\WireguardHub; +use Illuminate\Support\Carbon; +use Throwable; /** * Builds vmbr0 on a Debian-installed Proxmox host, under a timer that puts the @@ -67,7 +70,7 @@ class EnsureNetworkBridge extends HostStep /** One automatic attempt, plus one for the case an operator fixed something. */ private const MAX_ATTEMPTS = 2; - public function __construct(private RemoteShell $shell) {} + public function __construct(private RemoteShell $shell, private WireguardHub $hub) {} public function key(): string { @@ -86,7 +89,17 @@ class EnsureNetworkBridge extends HostStep public function execute(ProvisioningRun $run): StepResult { $host = $this->host($run); - $this->keyLogin($this->shell, $host); + + // The connection dropping is the EXPECTED case here, not an error: + // `ifreload -a` takes the line this step is speaking over. Left + // uncaught, RunRunner turns the exception into retry() — and retry + // spends the attempt budget, five of which are gone in minutes, long + // before the host is back. + try { + $this->keyLogin($this->shell, $host); + } catch (Throwable $e) { + return $this->whileDisconnected($run, $e); + } $state = $this->readBridgeState(); @@ -117,9 +130,201 @@ class EnsureNetworkBridge extends HostStep ); } - // Start, poll and cancel arrive in the next task. Harmless until then: - // the step is not in the pipeline yet, so no run can reach this line. - return StepResult::fail('not implemented yet'); + // Nothing on the host is cleaned up when a run ends, so a status file + // can be older than this run. The deadline in the run context is the + // only thing that tells a first visit from a lost one: it is written + // when THIS run starts a build, and cleared again by giveUp(). + if ($run->context('bridge_deadline') === null) { + return $this->start($run); + } + + $status = $this->readStatus(); + + return match ($status['state']) { + '' => $this->giveUp( + $run, + 'The bridge build was started but left no status behind, so it never really began. Check '. + self::WORK_DIR.' and the free space on the host, then retry.' + ), + 'running' => $this->whileRunning($run, $status), + // Reaching here means the driver said ok while vmbr0 does NOT carry + // the default route — so the timer got there first, or is about to. + 'ok' => $this->awaitRollback($run, $status, 'The bridge build reported success, but vmbr0 does not carry the default route.'), + 'failed' => $this->awaitRollback($run, $status, 'The bridge build failed: '.($status['note'] ?: 'no reason recorded').'.'), + default => $this->giveUp($run, 'The bridge build left an unreadable state ("'.$status['state'].'").'), + }; + } + + /** + * The host did not answer. + * + * With a deadline in the context, this run has changed the network and the + * silence is the change taking effect — poll, do not retry. Without one, + * nothing has been touched and an unreachable host is an ordinary + * connection error that may cost an attempt. + */ + private function whileDisconnected(ProvisioningRun $run, Throwable $e): StepResult + { + $deadline = $run->context('bridge_deadline'); + + if ($deadline === null) { + return StepResult::retry(20, 'host not reachable: '.$e->getMessage()); + } + + if (now()->greaterThan(Carbon::parse($deadline))) { + $run->forgetContext('bridge_deadline'); + + return StepResult::fail( + 'The host did not come back over the tunnel after the bridge was applied, and the deadline of '. + self::DEADLINE_MINUTES.' minutes has passed. The rollback timer will have restored the previous '. + 'network configuration '.self::ROLLBACK_MINUTES.' minutes after the change, so the machine should '. + 'be reachable on its old settings — check it, then retry. Last error: '.$e->getMessage() + ); + } + + return StepResult::poll(20, 'warte darauf, dass der Host über die neue Brücke wieder antwortet'); + } + + /** Ship the library, write the env, launch detached. */ + private function start(ProvisioningRun $run): StepResult + { + $attempts = (int) $run->context('bridge_attempts', 0); + + if ($attempts >= self::MAX_ATTEMPTS) { + return StepResult::fail( + 'CluPilot tried to build vmbr0 automatically '.$attempts.' times, and each time the host did not '. + 'come back both ways; the previous network configuration was restored. Build the bridge by hand '. + 'in /etc/network/interfaces (bridge_ports = the NIC carrying the default route, host address and '. + 'gateway moved onto vmbr0), apply it with `ifreload -a`, confirm you still have SSH, then retry '. + 'this run. The stanza CluPilot wrote is in '.self::WORK_DIR.'/bridge.log.' + ); + } + + $this->shell->putFile(self::WORK_DIR.'/bridge.sh', $this->asset('lib/bridge.sh')); + $this->shell->putFile(self::WORK_DIR.'/bridge-run.sh', $this->asset('lib/bridge-run.sh')); + + // The hub key so the driver can check the handshake against the RIGHT + // peer: staff peers live on the same hub, and their handshake says + // nothing about CluPilot's own way in. + $this->shell->putFile(self::WORK_DIR.'/env', implode("\n", [ + '# Written by CluPilot at the start of every bridge build. Do not edit.', + 'CLUPILOT_WORK_DIR='.self::WORK_DIR, + 'CLUPILOT_ROLLBACK_MINUTES='.self::ROLLBACK_MINUTES, + 'CLUPILOT_WG_HUB_PUBKEY='.escapeshellarg(trim($this->hub->publicKey())), + '', + ])); + + $run->mergeContext([ + 'bridge_deadline' => now()->addMinutes(self::DEADLINE_MINUTES)->toIso8601String(), + 'bridge_attempts' => $attempts + 1, + ]); + + // `setsid` puts the driver in a session of its own, so the pid it + // records is also its PROCESS GROUP id — which is what makes giving up + // possible later. The pid therefore comes from the script (`$$`), not + // from `$!` here: with setsid in between, `$!` can be a process that is + // already gone. Waiting for the file closes the gap that creates. + // + // `rolled-back` is removed first: it is the signal that a PREVIOUS + // attempt's timer fired, and a leftover would make this attempt read its + // own change as already rolled back. + $this->shell->run(implode("\n", [ + ': clupilot-bridge-start', + 'W='.self::WORK_DIR, + 'mkdir -p "$W"', + 'rm -f "$W/rolled-back"', + "printf 'running' > \"\$W/state\"", + "printf 'starting' > \"\$W/phase\"", + ': > "$W/note"', + ': > "$W/pid"', + 'nohup setsid sh "$W/bridge-run.sh" > "$W/bridge.log" 2>&1 &', + 'i=0; while [ "$i" -lt 15 ] && [ ! -s "$W/pid" ]; do sleep 1; i=$((i + 1)); done', + ])); + + return StepResult::poll(20, 'Netzbrücke wird gebaut'); + } + + /** @param array{state:string, alive:string, phase:string, note:string, rolledback:string} $status */ + private function whileRunning(ProvisioningRun $run, array $status): StepResult + { + if ($status['alive'] !== 'yes') { + // A kill, an OOM, a reboot. The file says running and always will — + // there is nobody left to change it. The timer is still armed, so + // the machine gets its old configuration back either way. + return $this->awaitRollback( + $run, + $status, + 'The bridge build stopped running without reporting a result — it was killed or the host restarted.' + ); + } + + $deadline = $run->context('bridge_deadline'); + if ($deadline !== null && now()->greaterThan(Carbon::parse($deadline))) { + return $this->giveUp( + $run, + 'The bridge build passed its deadline of '.self::DEADLINE_MINUTES.' minutes while still in phase "'. + ($status['phase'] ?: 'unknown').'". Check '.self::WORK_DIR.'/bridge.log on the host.' + ); + } + + return StepResult::poll(20, 'Netzbrücke wird gebaut: '.($status['phase'] ?: 'läuft')); + } + + /** + * The five facts about a build in flight, in one round trip. + * + * `alive` is computed on the host because that is the only place the answer + * exists: the pid means nothing here. + * + * @return array{state:string, alive:string, phase:string, note:string, rolledback:string} + */ + private function readStatus(): array + { + $out = $this->shell->run(implode("\n", [ + ': clupilot-bridge-status', + 'W='.self::WORK_DIR, + 'S=$(cat "$W/state" 2>/dev/null)', + 'P=$(cat "$W/pid" 2>/dev/null)', + 'A=no', + 'if [ -n "$P" ] && kill -0 "$P" 2>/dev/null; then A=yes; fi', + 'R=no', + 'if [ -f "$W/rolled-back" ]; then R=yes; fi', + "printf 'state=%s\\n' \"\$S\"", + "printf 'alive=%s\\n' \"\$A\"", + "printf 'phase=%s\\n' \"\$(head -1 \"\$W/phase\" 2>/dev/null)\"", + "printf 'note=%s\\n' \"\$(head -1 \"\$W/note\" 2>/dev/null)\"", + "printf 'rolledback=%s\\n' \"\$R\"", + ]))->stdout; + + $status = ['state' => '', 'alive' => 'no', 'phase' => '', 'note' => '', 'rolledback' => 'no']; + + foreach (preg_split('/\R/', $out) ?: [] as $line) { + [$key, $value] = array_pad(explode('=', $line, 2), 2, ''); + if (array_key_exists($key, $status)) { + $status[$key] = trim($value); + } + } + + return $status; + } + + /** The shipped file, read from the repo — never rendered from a string in here. */ + private function asset(string $relative): string + { + return (string) file_get_contents(base_path('deploy/bootstrap/'.$relative)); + } + + /** @param array{state:string, alive:string, phase:string, note:string, rolledback:string} $status */ + private function awaitRollback(ProvisioningRun $run, array $status, string $reason): StepResult + { + return $this->giveUp($run, $reason); + } + + private function giveUp(ProvisioningRun $run, string $reason): StepResult + { + $run->forgetContext('bridge_deadline'); + + return StepResult::fail($reason); } /** diff --git a/tests/Feature/Provisioning/HostStepsTest.php b/tests/Feature/Provisioning/HostStepsTest.php index b5e74ef..cee5a8b 100644 --- a/tests/Feature/Provisioning/HostStepsTest.php +++ b/tests/Feature/Provisioning/HostStepsTest.php @@ -1722,3 +1722,94 @@ it('gibt auf, wenn gar keine Karte die Standardroute trägt', function () { expect(app(EnsureNetworkBridge::class)->execute($run)->type)->toBe('fail') ->and($s['shell']->ran('clupilot-bridge-start'))->toBeFalse(); }); + +it('lädt bridge.sh und bridge-run.sh wortgleich hoch und startet abgekoppelt', function () { + // Byte für Byte. In dem Moment, in dem diese Datei aus einer PHP-Vorlage + // gerendert statt gelesen wird, gibt es wieder zwei Fassungen der + // Anbieterformen — das eine, was der Entwurf verbietet. + $s = fakeServices(); + $host = Host::factory()->active()->create(); + $run = hostRun($host); + proveTunnel($run, $host); + scriptBridgeState($s['shell'], up: false); + + $result = app(EnsureNetworkBridge::class)->execute($run); + $files = $s['shell']->files(); + + expect($result->type)->toBe('poll') + ->and($files['/var/lib/clupilot/bridge/bridge.sh'] ?? null) + ->toBe(file_get_contents(base_path('deploy/bootstrap/lib/bridge.sh'))) + ->and($files['/var/lib/clupilot/bridge/bridge-run.sh'] ?? null) + ->toBe(file_get_contents(base_path('deploy/bootstrap/lib/bridge-run.sh'))) + // Der Hub-Schlüssel muss mit, sonst prüft der Treiber den Handshake + // gegen irgendeinen Peer statt gegen den, über den CluPilot kommt. + ->and($files['/var/lib/clupilot/bridge/env'] ?? '')->toContain('CLUPILOT_WG_HUB_PUBKEY=') + ->and($files['/var/lib/clupilot/bridge/env'] ?? '')->toContain('CLUPILOT_ROLLBACK_MINUTES=5') + ->and($s['shell']->ran('nohup'))->toBeTrue() + ->and($s['shell']->ran('setsid'))->toBeTrue() + ->and($run->fresh()->context('bridge_deadline'))->not->toBeNull(); +}); + +it('pollt weiter, wenn der Host während der Umstellung nicht antwortet', function () { + // DIE tragende Zeile. RunRunner verwandelt jede geworfene Ausnahme in ein + // retry(), und retry verbraucht das Versuchskonto — ungefangen brennt der + // Verbindungsabriss die fünf Versuche in wenigen Minuten durch und lässt den + // Lauf scheitern, BEVOR der Host wieder da ist. + $s = fakeServices(); + $host = Host::factory()->active()->create(); + $run = hostRun($host, ['bridge_deadline' => now()->addMinutes(10)->toIso8601String()]); + proveTunnel($run, $host); + $s['shell']->failConnect = true; + + $result = app(EnsureNetworkBridge::class)->execute($run); + + expect($result->type)->toBe('poll') + ->and($result->reason)->toContain('Brücke'); +}); + +it('lässt einen Verbindungsfehler VOR der Umstellung ein gewöhnlicher bleiben', function () { + // Ohne Termin hat dieser Lauf nichts angefasst. Dann ist ein + // Verbindungsfehler kein erwarteter Abriss, sondern ein Fehler — und darf + // das Versuchskonto kosten, statt eine Viertelstunde lang gepollt zu werden. + $s = fakeServices(); + $host = Host::factory()->active()->create(); + $run = hostRun($host); + proveTunnel($run, $host); + $s['shell']->failConnect = true; + + expect(app(EnsureNetworkBridge::class)->execute($run)->type)->toBe('retry'); +}); + +it('scheitert nach der Frist, wenn der Host gar nicht mehr antwortet', function () { + // Der Zeitgeber hat dann laengst zurückgespielt — seine Frist ist ein + // Drittel dieser. Genau deshalb heißt Aufgeben hier „der Host ist wieder da, + // nur ohne Brücke" und nicht „der Host ist weg". + $s = fakeServices(); + $host = Host::factory()->active()->create(); + $run = hostRun($host, ['bridge_deadline' => now()->subMinute()->toIso8601String()]); + proveTunnel($run, $host); + $s['shell']->failConnect = true; + + $result = app(EnsureNetworkBridge::class)->execute($run); + + expect($result->type)->toBe('fail') + ->and($result->reason)->toContain('rollback timer') + ->and($run->fresh()->context('bridge_deadline'))->toBeNull(); +}); + +it('startet keinen zweiten Treiber, solange der erste lebt', function () { + // Zwei Treiber auf einer /etc/network/interfaces, jeder mit eigenem + // Zeitgeber: der eine bestellt ab, was der andere gestellt hat. + $s = fakeServices(); + $host = Host::factory()->active()->create(); + $run = hostRun($host, ['bridge_deadline' => now()->addMinutes(10)->toIso8601String()]); + proveTunnel($run, $host); + scriptBridgeState($s['shell'], up: false); + scriptBridgeStatus($s['shell'], 'running', alive: true, phase: 'umstellen'); + + $result = app(EnsureNetworkBridge::class)->execute($run); + + expect($result->type)->toBe('poll') + ->and($result->reason)->toContain('umstellen') + ->and($s['shell']->ran('clupilot-bridge-start'))->toBeFalse(); +});