EnsureNetworkBridge: Start abgekoppelt, und der Verbindungsabriss wird gepollt statt gezaehlt
Die tragende Zeile ist das try/catch um keyLogin. RunRunner:102 verwandelt jede geworfene Ausnahme in ein retry(), und retry verbraucht das Versuchskonto — ungefangen brennt der erwartete Verbindungsabriss die fuenf Versuche in wenigen Minuten durch und laesst den Lauf scheitern, BEVOR der Host wieder da ist. Genau der Unterschied zwischen 'Wiederholung' und 'toter Server'. Unterschieden wird am Termin im Run-Kontext: ohne ihn hat dieser Lauf nichts angefasst, dann ist ein Verbindungsfehler ein gewoehnlicher und darf einen Versuch kosten. Mit ihm laeuft gerade eine Umstellung, dann wird gepollt. bridge.sh und bridge-run.sh gehen wortgleich hoch (Test vergleicht Byte fuer Byte gegen die Repo-Datei), env traegt den Hub-Schluessel mit, damit der Treiber den Handshake gegen den RICHTIGEN Peer prueft. Start per nohup setsid, PID-Datei abgewartet. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>main
nexxo
parent
786dd54257
commit
712422c161
|
|
@ -5,6 +5,9 @@ namespace App\Provisioning\Steps\Host;
|
|||
use App\Models\ProvisioningRun;
|
||||
use App\Provisioning\StepResult;
|
||||
use App\Services\Ssh\RemoteShell;
|
||||
use App\Services\Wireguard\WireguardHub;
|
||||
use Illuminate\Support\Carbon;
|
||||
use Throwable;
|
||||
|
||||
/**
|
||||
* Builds vmbr0 on a Debian-installed Proxmox host, under a timer that puts the
|
||||
|
|
@ -67,7 +70,7 @@ class EnsureNetworkBridge extends HostStep
|
|||
/** One automatic attempt, plus one for the case an operator fixed something. */
|
||||
private const MAX_ATTEMPTS = 2;
|
||||
|
||||
public function __construct(private RemoteShell $shell) {}
|
||||
public function __construct(private RemoteShell $shell, private WireguardHub $hub) {}
|
||||
|
||||
public function key(): string
|
||||
{
|
||||
|
|
@ -86,7 +89,17 @@ class EnsureNetworkBridge extends HostStep
|
|||
public function execute(ProvisioningRun $run): StepResult
|
||||
{
|
||||
$host = $this->host($run);
|
||||
$this->keyLogin($this->shell, $host);
|
||||
|
||||
// The connection dropping is the EXPECTED case here, not an error:
|
||||
// `ifreload -a` takes the line this step is speaking over. Left
|
||||
// uncaught, RunRunner turns the exception into retry() — and retry
|
||||
// spends the attempt budget, five of which are gone in minutes, long
|
||||
// before the host is back.
|
||||
try {
|
||||
$this->keyLogin($this->shell, $host);
|
||||
} catch (Throwable $e) {
|
||||
return $this->whileDisconnected($run, $e);
|
||||
}
|
||||
|
||||
$state = $this->readBridgeState();
|
||||
|
||||
|
|
@ -117,9 +130,201 @@ class EnsureNetworkBridge extends HostStep
|
|||
);
|
||||
}
|
||||
|
||||
// Start, poll and cancel arrive in the next task. Harmless until then:
|
||||
// the step is not in the pipeline yet, so no run can reach this line.
|
||||
return StepResult::fail('not implemented yet');
|
||||
// Nothing on the host is cleaned up when a run ends, so a status file
|
||||
// can be older than this run. The deadline in the run context is the
|
||||
// only thing that tells a first visit from a lost one: it is written
|
||||
// when THIS run starts a build, and cleared again by giveUp().
|
||||
if ($run->context('bridge_deadline') === null) {
|
||||
return $this->start($run);
|
||||
}
|
||||
|
||||
$status = $this->readStatus();
|
||||
|
||||
return match ($status['state']) {
|
||||
'' => $this->giveUp(
|
||||
$run,
|
||||
'The bridge build was started but left no status behind, so it never really began. Check '.
|
||||
self::WORK_DIR.' and the free space on the host, then retry.'
|
||||
),
|
||||
'running' => $this->whileRunning($run, $status),
|
||||
// Reaching here means the driver said ok while vmbr0 does NOT carry
|
||||
// the default route — so the timer got there first, or is about to.
|
||||
'ok' => $this->awaitRollback($run, $status, 'The bridge build reported success, but vmbr0 does not carry the default route.'),
|
||||
'failed' => $this->awaitRollback($run, $status, 'The bridge build failed: '.($status['note'] ?: 'no reason recorded').'.'),
|
||||
default => $this->giveUp($run, 'The bridge build left an unreadable state ("'.$status['state'].'").'),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The host did not answer.
|
||||
*
|
||||
* With a deadline in the context, this run has changed the network and the
|
||||
* silence is the change taking effect — poll, do not retry. Without one,
|
||||
* nothing has been touched and an unreachable host is an ordinary
|
||||
* connection error that may cost an attempt.
|
||||
*/
|
||||
private function whileDisconnected(ProvisioningRun $run, Throwable $e): StepResult
|
||||
{
|
||||
$deadline = $run->context('bridge_deadline');
|
||||
|
||||
if ($deadline === null) {
|
||||
return StepResult::retry(20, 'host not reachable: '.$e->getMessage());
|
||||
}
|
||||
|
||||
if (now()->greaterThan(Carbon::parse($deadline))) {
|
||||
$run->forgetContext('bridge_deadline');
|
||||
|
||||
return StepResult::fail(
|
||||
'The host did not come back over the tunnel after the bridge was applied, and the deadline of '.
|
||||
self::DEADLINE_MINUTES.' minutes has passed. The rollback timer will have restored the previous '.
|
||||
'network configuration '.self::ROLLBACK_MINUTES.' minutes after the change, so the machine should '.
|
||||
'be reachable on its old settings — check it, then retry. Last error: '.$e->getMessage()
|
||||
);
|
||||
}
|
||||
|
||||
return StepResult::poll(20, 'warte darauf, dass der Host über die neue Brücke wieder antwortet');
|
||||
}
|
||||
|
||||
/** Ship the library, write the env, launch detached. */
|
||||
private function start(ProvisioningRun $run): StepResult
|
||||
{
|
||||
$attempts = (int) $run->context('bridge_attempts', 0);
|
||||
|
||||
if ($attempts >= self::MAX_ATTEMPTS) {
|
||||
return StepResult::fail(
|
||||
'CluPilot tried to build vmbr0 automatically '.$attempts.' times, and each time the host did not '.
|
||||
'come back both ways; the previous network configuration was restored. Build the bridge by hand '.
|
||||
'in /etc/network/interfaces (bridge_ports = the NIC carrying the default route, host address and '.
|
||||
'gateway moved onto vmbr0), apply it with `ifreload -a`, confirm you still have SSH, then retry '.
|
||||
'this run. The stanza CluPilot wrote is in '.self::WORK_DIR.'/bridge.log.'
|
||||
);
|
||||
}
|
||||
|
||||
$this->shell->putFile(self::WORK_DIR.'/bridge.sh', $this->asset('lib/bridge.sh'));
|
||||
$this->shell->putFile(self::WORK_DIR.'/bridge-run.sh', $this->asset('lib/bridge-run.sh'));
|
||||
|
||||
// The hub key so the driver can check the handshake against the RIGHT
|
||||
// peer: staff peers live on the same hub, and their handshake says
|
||||
// nothing about CluPilot's own way in.
|
||||
$this->shell->putFile(self::WORK_DIR.'/env', implode("\n", [
|
||||
'# Written by CluPilot at the start of every bridge build. Do not edit.',
|
||||
'CLUPILOT_WORK_DIR='.self::WORK_DIR,
|
||||
'CLUPILOT_ROLLBACK_MINUTES='.self::ROLLBACK_MINUTES,
|
||||
'CLUPILOT_WG_HUB_PUBKEY='.escapeshellarg(trim($this->hub->publicKey())),
|
||||
'',
|
||||
]));
|
||||
|
||||
$run->mergeContext([
|
||||
'bridge_deadline' => now()->addMinutes(self::DEADLINE_MINUTES)->toIso8601String(),
|
||||
'bridge_attempts' => $attempts + 1,
|
||||
]);
|
||||
|
||||
// `setsid` puts the driver in a session of its own, so the pid it
|
||||
// records is also its PROCESS GROUP id — which is what makes giving up
|
||||
// possible later. The pid therefore comes from the script (`$$`), not
|
||||
// from `$!` here: with setsid in between, `$!` can be a process that is
|
||||
// already gone. Waiting for the file closes the gap that creates.
|
||||
//
|
||||
// `rolled-back` is removed first: it is the signal that a PREVIOUS
|
||||
// attempt's timer fired, and a leftover would make this attempt read its
|
||||
// own change as already rolled back.
|
||||
$this->shell->run(implode("\n", [
|
||||
': clupilot-bridge-start',
|
||||
'W='.self::WORK_DIR,
|
||||
'mkdir -p "$W"',
|
||||
'rm -f "$W/rolled-back"',
|
||||
"printf 'running' > \"\$W/state\"",
|
||||
"printf 'starting' > \"\$W/phase\"",
|
||||
': > "$W/note"',
|
||||
': > "$W/pid"',
|
||||
'nohup setsid sh "$W/bridge-run.sh" </dev/null >> "$W/bridge.log" 2>&1 &',
|
||||
'i=0; while [ "$i" -lt 15 ] && [ ! -s "$W/pid" ]; do sleep 1; i=$((i + 1)); done',
|
||||
]));
|
||||
|
||||
return StepResult::poll(20, 'Netzbrücke wird gebaut');
|
||||
}
|
||||
|
||||
/** @param array{state:string, alive:string, phase:string, note:string, rolledback:string} $status */
|
||||
private function whileRunning(ProvisioningRun $run, array $status): StepResult
|
||||
{
|
||||
if ($status['alive'] !== 'yes') {
|
||||
// A kill, an OOM, a reboot. The file says running and always will —
|
||||
// there is nobody left to change it. The timer is still armed, so
|
||||
// the machine gets its old configuration back either way.
|
||||
return $this->awaitRollback(
|
||||
$run,
|
||||
$status,
|
||||
'The bridge build stopped running without reporting a result — it was killed or the host restarted.'
|
||||
);
|
||||
}
|
||||
|
||||
$deadline = $run->context('bridge_deadline');
|
||||
if ($deadline !== null && now()->greaterThan(Carbon::parse($deadline))) {
|
||||
return $this->giveUp(
|
||||
$run,
|
||||
'The bridge build passed its deadline of '.self::DEADLINE_MINUTES.' minutes while still in phase "'.
|
||||
($status['phase'] ?: 'unknown').'". Check '.self::WORK_DIR.'/bridge.log on the host.'
|
||||
);
|
||||
}
|
||||
|
||||
return StepResult::poll(20, 'Netzbrücke wird gebaut: '.($status['phase'] ?: 'läuft'));
|
||||
}
|
||||
|
||||
/**
|
||||
* The five facts about a build in flight, in one round trip.
|
||||
*
|
||||
* `alive` is computed on the host because that is the only place the answer
|
||||
* exists: the pid means nothing here.
|
||||
*
|
||||
* @return array{state:string, alive:string, phase:string, note:string, rolledback:string}
|
||||
*/
|
||||
private function readStatus(): array
|
||||
{
|
||||
$out = $this->shell->run(implode("\n", [
|
||||
': clupilot-bridge-status',
|
||||
'W='.self::WORK_DIR,
|
||||
'S=$(cat "$W/state" 2>/dev/null)',
|
||||
'P=$(cat "$W/pid" 2>/dev/null)',
|
||||
'A=no',
|
||||
'if [ -n "$P" ] && kill -0 "$P" 2>/dev/null; then A=yes; fi',
|
||||
'R=no',
|
||||
'if [ -f "$W/rolled-back" ]; then R=yes; fi',
|
||||
"printf 'state=%s\\n' \"\$S\"",
|
||||
"printf 'alive=%s\\n' \"\$A\"",
|
||||
"printf 'phase=%s\\n' \"\$(head -1 \"\$W/phase\" 2>/dev/null)\"",
|
||||
"printf 'note=%s\\n' \"\$(head -1 \"\$W/note\" 2>/dev/null)\"",
|
||||
"printf 'rolledback=%s\\n' \"\$R\"",
|
||||
]))->stdout;
|
||||
|
||||
$status = ['state' => '', 'alive' => 'no', 'phase' => '', 'note' => '', 'rolledback' => 'no'];
|
||||
|
||||
foreach (preg_split('/\R/', $out) ?: [] as $line) {
|
||||
[$key, $value] = array_pad(explode('=', $line, 2), 2, '');
|
||||
if (array_key_exists($key, $status)) {
|
||||
$status[$key] = trim($value);
|
||||
}
|
||||
}
|
||||
|
||||
return $status;
|
||||
}
|
||||
|
||||
/** The shipped file, read from the repo — never rendered from a string in here. */
|
||||
private function asset(string $relative): string
|
||||
{
|
||||
return (string) file_get_contents(base_path('deploy/bootstrap/'.$relative));
|
||||
}
|
||||
|
||||
/** @param array{state:string, alive:string, phase:string, note:string, rolledback:string} $status */
|
||||
private function awaitRollback(ProvisioningRun $run, array $status, string $reason): StepResult
|
||||
{
|
||||
return $this->giveUp($run, $reason);
|
||||
}
|
||||
|
||||
private function giveUp(ProvisioningRun $run, string $reason): StepResult
|
||||
{
|
||||
$run->forgetContext('bridge_deadline');
|
||||
|
||||
return StepResult::fail($reason);
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
|
|
@ -1722,3 +1722,94 @@ it('gibt auf, wenn gar keine Karte die Standardroute trägt', function () {
|
|||
expect(app(EnsureNetworkBridge::class)->execute($run)->type)->toBe('fail')
|
||||
->and($s['shell']->ran('clupilot-bridge-start'))->toBeFalse();
|
||||
});
|
||||
|
||||
it('lädt bridge.sh und bridge-run.sh wortgleich hoch und startet abgekoppelt', function () {
|
||||
// Byte für Byte. In dem Moment, in dem diese Datei aus einer PHP-Vorlage
|
||||
// gerendert statt gelesen wird, gibt es wieder zwei Fassungen der
|
||||
// Anbieterformen — das eine, was der Entwurf verbietet.
|
||||
$s = fakeServices();
|
||||
$host = Host::factory()->active()->create();
|
||||
$run = hostRun($host);
|
||||
proveTunnel($run, $host);
|
||||
scriptBridgeState($s['shell'], up: false);
|
||||
|
||||
$result = app(EnsureNetworkBridge::class)->execute($run);
|
||||
$files = $s['shell']->files();
|
||||
|
||||
expect($result->type)->toBe('poll')
|
||||
->and($files['/var/lib/clupilot/bridge/bridge.sh'] ?? null)
|
||||
->toBe(file_get_contents(base_path('deploy/bootstrap/lib/bridge.sh')))
|
||||
->and($files['/var/lib/clupilot/bridge/bridge-run.sh'] ?? null)
|
||||
->toBe(file_get_contents(base_path('deploy/bootstrap/lib/bridge-run.sh')))
|
||||
// Der Hub-Schlüssel muss mit, sonst prüft der Treiber den Handshake
|
||||
// gegen irgendeinen Peer statt gegen den, über den CluPilot kommt.
|
||||
->and($files['/var/lib/clupilot/bridge/env'] ?? '')->toContain('CLUPILOT_WG_HUB_PUBKEY=')
|
||||
->and($files['/var/lib/clupilot/bridge/env'] ?? '')->toContain('CLUPILOT_ROLLBACK_MINUTES=5')
|
||||
->and($s['shell']->ran('nohup'))->toBeTrue()
|
||||
->and($s['shell']->ran('setsid'))->toBeTrue()
|
||||
->and($run->fresh()->context('bridge_deadline'))->not->toBeNull();
|
||||
});
|
||||
|
||||
it('pollt weiter, wenn der Host während der Umstellung nicht antwortet', function () {
|
||||
// DIE tragende Zeile. RunRunner verwandelt jede geworfene Ausnahme in ein
|
||||
// retry(), und retry verbraucht das Versuchskonto — ungefangen brennt der
|
||||
// Verbindungsabriss die fünf Versuche in wenigen Minuten durch und lässt den
|
||||
// Lauf scheitern, BEVOR der Host wieder da ist.
|
||||
$s = fakeServices();
|
||||
$host = Host::factory()->active()->create();
|
||||
$run = hostRun($host, ['bridge_deadline' => now()->addMinutes(10)->toIso8601String()]);
|
||||
proveTunnel($run, $host);
|
||||
$s['shell']->failConnect = true;
|
||||
|
||||
$result = app(EnsureNetworkBridge::class)->execute($run);
|
||||
|
||||
expect($result->type)->toBe('poll')
|
||||
->and($result->reason)->toContain('Brücke');
|
||||
});
|
||||
|
||||
it('lässt einen Verbindungsfehler VOR der Umstellung ein gewöhnlicher bleiben', function () {
|
||||
// Ohne Termin hat dieser Lauf nichts angefasst. Dann ist ein
|
||||
// Verbindungsfehler kein erwarteter Abriss, sondern ein Fehler — und darf
|
||||
// das Versuchskonto kosten, statt eine Viertelstunde lang gepollt zu werden.
|
||||
$s = fakeServices();
|
||||
$host = Host::factory()->active()->create();
|
||||
$run = hostRun($host);
|
||||
proveTunnel($run, $host);
|
||||
$s['shell']->failConnect = true;
|
||||
|
||||
expect(app(EnsureNetworkBridge::class)->execute($run)->type)->toBe('retry');
|
||||
});
|
||||
|
||||
it('scheitert nach der Frist, wenn der Host gar nicht mehr antwortet', function () {
|
||||
// Der Zeitgeber hat dann laengst zurückgespielt — seine Frist ist ein
|
||||
// Drittel dieser. Genau deshalb heißt Aufgeben hier „der Host ist wieder da,
|
||||
// nur ohne Brücke" und nicht „der Host ist weg".
|
||||
$s = fakeServices();
|
||||
$host = Host::factory()->active()->create();
|
||||
$run = hostRun($host, ['bridge_deadline' => now()->subMinute()->toIso8601String()]);
|
||||
proveTunnel($run, $host);
|
||||
$s['shell']->failConnect = true;
|
||||
|
||||
$result = app(EnsureNetworkBridge::class)->execute($run);
|
||||
|
||||
expect($result->type)->toBe('fail')
|
||||
->and($result->reason)->toContain('rollback timer')
|
||||
->and($run->fresh()->context('bridge_deadline'))->toBeNull();
|
||||
});
|
||||
|
||||
it('startet keinen zweiten Treiber, solange der erste lebt', function () {
|
||||
// Zwei Treiber auf einer /etc/network/interfaces, jeder mit eigenem
|
||||
// Zeitgeber: der eine bestellt ab, was der andere gestellt hat.
|
||||
$s = fakeServices();
|
||||
$host = Host::factory()->active()->create();
|
||||
$run = hostRun($host, ['bridge_deadline' => now()->addMinutes(10)->toIso8601String()]);
|
||||
proveTunnel($run, $host);
|
||||
scriptBridgeState($s['shell'], up: false);
|
||||
scriptBridgeStatus($s['shell'], 'running', alive: true, phase: 'umstellen');
|
||||
|
||||
$result = app(EnsureNetworkBridge::class)->execute($run);
|
||||
|
||||
expect($result->type)->toBe('poll')
|
||||
->and($result->reason)->toContain('umstellen')
|
||||
->and($s['shell']->ran('clupilot-bridge-start'))->toBeFalse();
|
||||
});
|
||||
|
|
|
|||
Loading…
Reference in New Issue