Close SSH and the Proxmox UI to the world once the tunnel is proven up

app/Provisioning/Steps/Host/HostStep.php

@@ -34,10 +34,17 @@ abstract class HostStep implements ProvisioningStep
         return $host;
     }
 
+    /**
+     * Prefer the WireGuard management address once the tunnel exists, so every
+     * step after ConfigureWireguard — and every later maintenance run — reaches
+     * the host over the tunnel rather than its public IP. Falls back to
+     * public_ip for the handful of steps that run before wg_ip is set (the
+     * tunnel cannot carry SSH before it exists).
+     */
     protected function keyLogin(RemoteShell $shell, Host $host): void
     {
         $shell->connectWithKey(
-            $host->public_ip,
+            filled($host->wg_ip) ? $host->wg_ip : $host->public_ip,
             'root',
             (string) config('provisioning.ssh.private_key'),
             $host->ssh_host_key, // pinned during EstablishSshTrust

app/Provisioning/Steps/Host/SecureHostFirewall.php

@@ -0,0 +1,167 @@
+<?php
+
+namespace App\Provisioning\Steps\Host;
+
+use App\Models\ProvisioningRun;
+use App\Provisioning\StepResult;
+use App\Services\Ssh\RemoteShell;
+
+/**
+ * Locks the host down to only what it needs to expose: 80/443 open to the
+ * world (Traefik on this host serves customer traffic there), 22 and 8006
+ * (Proxmox UI/API) reachable only from the WireGuard management subnet,
+ * everything else inbound dropped.
+ *
+ * Runs LAST in the host pipeline — after RegisterCapacity, before
+ * CompleteHostOnboarding — so the tunnel has already carried every prior step
+ * through a reboot before SSH-to-the-world is ever closed. Closing 22 earlier
+ * would risk stranding a host mid-onboarding if the tunnel turned out not to
+ * be solid.
+ *
+ * Refuses to apply anything unless the tunnel is confirmed up from the HOST'S
+ * OWN side first (a ping to the WireGuard hub, run over the already-open SSH
+ * session — the same check ConfigureWireguard uses to confirm its handshake).
+ * Applying firewall rules on a host that cannot prove its tunnel works is
+ * exactly how a host becomes permanently unreachable, so a failed check
+ * retries and touches nothing.
+ *
+ * Persisted via nftables: the step installs the package, writes
+ * /etc/nftables.conf, and enables the `nftables` systemd unit, which reloads
+ * that file on every boot — including a later maintenance reboot (this step
+ * runs after RebootIntoPveKernel, so from here on a reboot is a maintenance
+ * one). Chosen over Proxmox's own firewall (pve-firewall needs the /etc/pve
+ * cluster filesystem plus a separate datacenter+node enable flag — two more
+ * places this could silently stay off) and over iptables-persistent (nftables
+ * is bookworm's default packet-filtering framework and already ships the
+ * boot-time unit the package needs). Only the INPUT chain is touched — FORWARD
+ * and OUTPUT are left at the kernel default so customer VM traffic through
+ * the bridges is never affected by a host-management rule.
+ *
+ * Idempotent: once applied, the resource breadcrumb short-circuits a re-run
+ * exactly like the other steps' external-resource guards.
+ *
+ * Deliberately has NO automatic recovery. If a host becomes unreachable, an
+ * operator drops the rules by running
+ * /usr/local/sbin/clupilot-emergency-open-firewall.sh from the provider's
+ * out-of-band console — no timer, no "reopen after N minutes without a
+ * handshake". A firewall that reopens itself under failure is not a firewall.
+ */
+class SecureHostFirewall extends HostStep
+{
+    private const EMERGENCY_SCRIPT_PATH = '/usr/local/sbin/clupilot-emergency-open-firewall.sh';
+
+    public function __construct(private RemoteShell $shell) {}
+
+    public function key(): string
+    {
+        return 'secure_host_firewall';
+    }
+
+    public function execute(ProvisioningRun $run): StepResult
+    {
+        $host = $this->host($run);
+        $this->keyLogin($this->shell, $host);
+
+        // Idempotent replay: already applied and recorded.
+        if ($this->hasResource($run, 'host_firewall')) {
+            return StepResult::advance();
+        }
+
+        // Verify FIRST, from the host's own side, and touch nothing if it fails.
+        if (! $this->tunnelConfirmedUp()) {
+            return StepResult::retry(15, 'WireGuard tunnel not confirmed up; refusing to apply the host firewall');
+        }
+
+        $wgSubnet = (string) config('provisioning.wireguard.subnet');
+
+        if (! $this->shell->run('export DEBIAN_FRONTEND=noninteractive; apt-get install -y nftables')->ok()) {
+            return StepResult::retry(30, 'installing nftables failed');
+        }
+
+        $this->shell->putFile('/etc/nftables.conf', $this->renderNftablesConfig($wgSubnet));
+        $this->shell->putFile(self::EMERGENCY_SCRIPT_PATH, $this->renderEmergencyScript());
+        $this->shell->run('chmod 700 '.escapeshellarg(self::EMERGENCY_SCRIPT_PATH));
+
+        if (! $this->shell->run('nft -c -f /etc/nftables.conf')->ok()) {
+            return StepResult::retry(30, 'nftables ruleset failed to validate');
+        }
+
+        if (! $this->shell->run('systemctl enable --now nftables')->ok()) {
+            return StepResult::retry(30, 'enabling nftables failed');
+        }
+        $this->shell->run('systemctl reload-or-restart nftables || nft -f /etc/nftables.conf');
+
+        $this->recordResource($run, $host, 'host_firewall', 'nftables');
+
+        return StepResult::advance();
+    }
+
+    /** Same mechanism ConfigureWireguard::verifyHandshake() uses to confirm the tunnel. */
+    private function tunnelConfirmedUp(): bool
+    {
+        $hubIp = (string) config('provisioning.wireguard.hub_ip');
+
+        return $this->shell->run('ping -c1 -W2 '.escapeshellarg($hubIp))->ok();
+    }
+
+    private function renderNftablesConfig(string $wgSubnet): string
+    {
+        return <<<NFT
+#!/usr/sbin/nft -f
+# Written by CluPilot's SecureHostFirewall provisioning step.
+# Do not edit by hand — the next run of that step overwrites this file.
+# To fully re-open the host in an emergency, run
+# {$this->emergencyScriptPath()} from the provider's out-of-band console.
+
+flush ruleset
+
+table inet clupilot_filter {
+    chain input {
+        type filter hook input priority 0; policy drop;
+
+        iif "lo" accept
+        ct state invalid drop
+        ct state established,related accept
+
+        # Public web — Traefik on this host serves customer traffic here.
+        tcp dport { 80, 443 } accept
+
+        # Management surfaces: only reachable over the WireGuard tunnel.
+        ip saddr {$wgSubnet} tcp dport { 22, 8006 } accept
+    }
+}
+NFT;
+    }
+
+    private function renderEmergencyScript(): string
+    {
+        return <<<'SH'
+#!/bin/sh
+# CluPilot emergency firewall release.
+#
+# Run this ONLY from the provider's out-of-band console (KVM/serial) if this
+# host has become unreachable over the network. It does exactly one thing:
+# disable nftables and flush every rule, so the host reverts to the kernel's
+# default-accept state on every port.
+#
+# Deliberately manual. Nothing here reopens the firewall automatically, on a
+# timer, or because a handshake looked stale for a while — a firewall that
+# reopens itself under failure is not a firewall. Put the rules back by
+# re-running the SecureHostFirewall provisioning step from CluPilot, or by
+# hand:
+#   nft -f /etc/nftables.conf && systemctl enable --now nftables
+
+set -e
+
+systemctl disable --now nftables 2>/dev/null || true
+nft flush ruleset 2>/dev/null || true
+
+echo "Firewall rules dropped: this host now accepts inbound traffic on every port."
+SH;
+    }
+
+    private function emergencyScriptPath(): string
+    {
+        return self::EMERGENCY_SCRIPT_PATH;
+    }
+}

config/provisioning.php

@@ -41,6 +41,7 @@ return [
             Host\VerifyProxmoxApi::class,
             Host\RegisterHostDns::class,
             Host\RegisterCapacity::class,
+            Host\SecureHostFirewall::class,
             Host\CompleteHostOnboarding::class,
         ],
         'customer' => [

lang/de/hosts.php

@@ -121,6 +121,7 @@ return [
         'create_automation_token' => 'Automation-Token erstellen',
         'verify_proxmox_api' => 'Proxmox-API prüfen',
         'register_capacity' => 'Kapazität registrieren',
+        'secure_host_firewall' => 'Host-Firewall absichern',
         'complete_host_onboarding' => 'Onboarding abschließen',
     ],
 ];

lang/en/hosts.php

@@ -121,6 +121,7 @@ return [
         'create_automation_token' => 'Create automation token',
         'verify_proxmox_api' => 'Verify Proxmox API',
         'register_capacity' => 'Register capacity',
+        'secure_host_firewall' => 'Secure host firewall',
         'complete_host_onboarding' => 'Complete onboarding',
     ],
 ];

tests/Feature/Provisioning/HostOnboardingEndToEndTest.php

@@ -4,6 +4,7 @@ use App\Actions\StartHostOnboarding;
 use App\Models\ProvisioningRun;
 use App\Models\RunResource;
 use App\Provisioning\RunRunner;
+use App\Provisioning\Steps\Host\ConfigureWireguard;
 use App\Services\Ssh\CommandResult;
 use Illuminate\Support\Facades\Queue;
 
@@ -50,8 +51,14 @@ it('drives a fresh host all the way to active (mocked)', function () {
         // external resources created exactly once
         ->and(RunResource::where('run_id', $run->id)->where('kind', 'wg_peer')->count())->toBe(1)
         ->and(RunResource::where('run_id', $run->id)->where('kind', 'pve_token')->count())->toBe(1)
+        ->and(RunResource::where('run_id', $run->id)->where('kind', 'host_firewall')->count())->toBe(1)
         // one-time root password scrubbed from state
         ->and($run->context('root_password'))->toBeNull();
+
+    // The firewall step ran last, over the tunnel, and left its rules + the
+    // manual emergency-release script on the host.
+    expect($s['shell']->files())->toHaveKey('/etc/nftables.conf')
+        ->and($s['shell']->files())->toHaveKey('/usr/local/sbin/clupilot-emergency-open-firewall.sh');
 });
 
 it('does not duplicate external resources when a step re-runs after a crash', function () {
@@ -83,7 +90,7 @@ it('does not duplicate external resources when a step re-runs after a crash', fu
     }
     $run->refresh();
     $wgIndex = array_search(
-        \App\Provisioning\Steps\Host\ConfigureWireguard::class,
+        ConfigureWireguard::class,
         config('provisioning.pipelines.host'),
         true,
     );

tests/Feature/Provisioning/HostStepsTest.php

@@ -6,6 +6,7 @@ use App\Models\ProvisioningRun;
 use App\Models\RunResource;
 use App\Provisioning\Jobs\PurgeHost;
 use App\Provisioning\Steps\Host\CompleteHostOnboarding;
+use App\Provisioning\Steps\Host\ConfigureProxmox;
 use App\Provisioning\Steps\Host\ConfigureWireguard;
 use App\Provisioning\Steps\Host\CreateAutomationToken;
 use App\Provisioning\Steps\Host\EstablishSshTrust;
@@ -14,6 +15,7 @@ use App\Provisioning\Steps\Host\PrepareBaseSystem;
 use App\Provisioning\Steps\Host\RebootIntoPveKernel;
 use App\Provisioning\Steps\Host\RegisterCapacity;
 use App\Provisioning\Steps\Host\RegisterHostDns;
+use App\Provisioning\Steps\Host\SecureHostFirewall;
 use App\Provisioning\Steps\Host\ValidateHostInput;
 use App\Provisioning\Steps\Host\VerifyProxmoxApi;
 use App\Services\Ssh\CommandResult;
@@ -71,6 +73,42 @@ it('throws when the host is unreachable (runner turns it into a retry)', functio
     expect(fn () => app(EstablishSshTrust::class)->execute($run))->toThrow(RuntimeException::class);
 });
 
+// --- HostStep::keyLogin address preference ---
+//
+// Mutation boundary: every step after ConfigureWireguard (and every later
+// maintenance run) MUST reach the host over the tunnel once wg_ip is set —
+// closing SSH to the world without this would strand the host permanently.
+// ConfigureProxmox is used as a plain probe: keyLogin then two commands, no
+// other branching that could obscure which address was actually dialled.
+
+it('uses the WireGuard address for ssh once the tunnel exists', function () {
+    $s = fakeServices();
+    $host = Host::factory()->create(['public_ip' => '203.0.113.20', 'wg_ip' => '10.66.0.20']);
+    $run = hostRun($host);
+
+    app(ConfigureProxmox::class)->execute($run);
+
+    $connections = $s['shell']->connectionsWith('key');
+    expect($connections)->not->toBeEmpty();
+    foreach ($connections as $connection) {
+        expect($connection[1])->toBe('10.66.0.20');
+    }
+});
+
+it('falls back to the public IP before the tunnel exists', function () {
+    $s = fakeServices();
+    $host = Host::factory()->create(['public_ip' => '203.0.113.21', 'wg_ip' => null]);
+    $run = hostRun($host);
+
+    app(ConfigureProxmox::class)->execute($run);
+
+    $connections = $s['shell']->connectionsWith('key');
+    expect($connections)->not->toBeEmpty();
+    foreach ($connections as $connection) {
+        expect($connection[1])->toBe('203.0.113.21');
+    }
+});
+
 // --- PrepareBaseSystem ---
 
 it('prepares the base system', function () {
@@ -234,6 +272,90 @@ it('registers node capacity on the host', function () {
         ->and($host->last_seen_at)->not->toBeNull();
 });
 
+// --- SecureHostFirewall ---
+
+it('places SecureHostFirewall after every other host step and before completion', function () {
+    $pipeline = config('provisioning.pipelines.host');
+
+    $secureIndex = array_search(SecureHostFirewall::class, $pipeline, true);
+    $capacityIndex = array_search(RegisterCapacity::class, $pipeline, true);
+    $completeIndex = array_search(CompleteHostOnboarding::class, $pipeline, true);
+
+    expect($secureIndex)->not->toBeFalse()
+        ->and($secureIndex)->toBe($capacityIndex + 1)
+        ->and($secureIndex)->toBe($completeIndex - 1);
+});
+
+it('refuses to apply the firewall when the tunnel is not confirmed up from the host', function () {
+    // Mutation boundary: a host whose tunnel does not verify must come away
+    // completely untouched — retrying is not enough on its own, nothing may
+    // have been written either.
+    $s = fakeServices();
+    $s['shell']->script('ping', CommandResult::failure(1));
+    $host = Host::factory()->active()->create();
+    $run = hostRun($host);
+
+    expect(app(SecureHostFirewall::class)->execute($run)->type)->toBe('retry');
+
+    expect($s['shell']->files())->not->toHaveKey('/etc/nftables.conf')
+        ->and($s['shell']->ran('apt-get install -y nftables'))->toBeFalse()
+        ->and(RunResource::where('run_id', $run->id)->where('kind', 'host_firewall')->exists())->toBeFalse();
+});
+
+it('applies the firewall once the tunnel is confirmed up, over the tunnel itself', function () {
+    $s = fakeServices();
+    $host = Host::factory()->active()->create(['public_ip' => '203.0.113.30']);
+    $run = hostRun($host);
+
+    expect(app(SecureHostFirewall::class)->execute($run)->type)->toBe('advance');
+
+    // Connected over the tunnel, not the public address.
+    foreach ($s['shell']->connectionsWith('key') as $connection) {
+        expect($connection[1])->toBe($host->wg_ip);
+    }
+
+    $wgSubnet = (string) config('provisioning.wireguard.subnet');
+    $config = $s['shell']->files()['/etc/nftables.conf'];
+    expect($config)->toContain('policy drop')
+        ->and($config)->toContain('tcp dport { 80, 443 } accept')
+        ->and($config)->toContain('ip saddr '.$wgSubnet.' tcp dport { 22, 8006 } accept')
+        ->and($config)->toContain('ct state established,related accept');
+
+    expect($s['shell']->files())->toHaveKey('/usr/local/sbin/clupilot-emergency-open-firewall.sh')
+        ->and($s['shell']->ran('chmod 700'))->toBeTrue()
+        ->and($s['shell']->ran('/usr/local/sbin/clupilot-emergency-open-firewall.sh'))->toBeTrue()
+        ->and($s['shell']->ran('systemctl enable --now nftables'))->toBeTrue()
+        ->and(RunResource::where('run_id', $run->id)->where('kind', 'host_firewall')->count())->toBe(1);
+});
+
+it("keeps the WireGuard interface's established connections working (never a bare drop-all)", function () {
+    $s = fakeServices();
+    $run = hostRun(Host::factory()->active()->create());
+
+    app(SecureHostFirewall::class)->execute($run);
+
+    $config = $s['shell']->files()['/etc/nftables.conf'];
+    expect($config)->toContain('ct state established,related accept')
+        ->and($config)->toContain('iif "lo" accept');
+});
+
+it('never reapplies the firewall on a re-run', function () {
+    $s = fakeServices();
+    $host = Host::factory()->active()->create();
+    $run = hostRun($host);
+
+    app(SecureHostFirewall::class)->execute($run);
+    app(SecureHostFirewall::class)->execute($run->fresh());
+
+    $installCalls = fn () => count(array_filter(
+        $s['shell']->recorded(),
+        fn ($c) => str_contains($c, 'apt-get install -y nftables'),
+    ));
+
+    expect($installCalls())->toBe(1)
+        ->and(RunResource::where('run_id', $run->id)->where('kind', 'host_firewall')->count())->toBe(1);
+});
+
 // --- CompleteHostOnboarding ---
 
 it('marks the host active on completion', function () {