host($run); $this->keyLogin($this->shell, $host); if (($refusal = $this->refuseWithoutBridge()) !== null) { return $refusal; } if (($failure = $this->enableDatacenterFirewall()) !== null) { return $failure; } // AFTER the install, not before: the proxmox-ve package ships the // enterprise repository itself, so InstallProxmoxVe's removal (which // happens before apt runs) cannot be the one that sticks. Both spellings, // because PVE 9 ships the deb822 file and PVE 8 the one-line one. $this->shell->run('rm -f /etc/apt/sources.list.d/pve-enterprise.list /etc/apt/sources.list.d/pve-enterprise.sources'); return StepResult::advance(); } /** * vmbr0 must exist — and this step will NOT create it. * * Creating a bridge over the primary NIC from a remote shell means editing * /etc/network/interfaces and reloading the network on the same link the * shell is running over. Getting it even slightly wrong (a provider that * needs a pointopoint route, a MAC the upstream switch will not accept on a * bridge, an IPv6 gateway on the wrong device) leaves the machine off the * network with no way back except the provider's console. The failure below * costs an operator ten minutes; the alternative costs a dead server. * * The message carries the default route, because that is the one fact the * operator needs and the one they would otherwise go and look up. */ private function refuseWithoutBridge(): ?StepResult { if ($this->shell->run('ip link show vmbr0')->ok()) { return null; } $route = trim($this->shell->run('ip -4 route show default')->stdout); return StepResult::fail( 'No vmbr0 bridge on this host. Proxmox installed on top of Debian does not create one — only the '. 'ISO installer does — and CluPilot deliberately will not build it over the primary NIC from a remote '. 'shell, because a mistake there takes the machine off the network for good. Create it by hand in '. '/etc/network/interfaces (bridge_ports = the NIC carrying the default route, the host address and '. 'gateway moved onto vmbr0), apply it with `ifreload -a`, confirm you still have SSH, then retry this '. 'run. Current default route: '.($route === '' ? '(none reported)' : $route).'.' ); } /** * Enable the datacenter firewall WITHOUT locking the host out. * * The plain `enable 1` is a trap. Proxmox's datacenter input policy defaults * to DROP, the node's own host firewall defaults to ENABLED, and the only * sources it lets reach 8006 and 22 are the members of the auto-generated * `management` ipset — which is seeded from `local_network`, the subnet of * the node's own management address, i.e. the PUBLIC subnet. The WireGuard * management address every later step and every maintenance run comes from is * not in it. Enabling the firewall blindly therefore drops CluPilot's own SSH * and API access the moment pve-firewall next compiles, on a host whose * public port 22 SecureHostFirewall is about to close as well. * * So three settings, in this order, each doing one job: * * - `policy_in`/`policy_out` ACCEPT at datacenter level. This is the HOST * input policy only: a guest reads its own vm.fw `policy_in`, which * applyFirewall() sets to DROP explicitly for every customer VM, so ACCEPT * here cannot loosen a single customer rule. What it does do is make sure * that if anyone ever re-enables the node firewall from the GUI, the host * does not become unreachable over the tunnel in the same instant. * - The node's host firewall off. The host policy lives in nftables, written * by SecureHostFirewall — see that step's docblock for why that was chosen * over pve-firewall. Two filters both claiming to own the input chain is * how a host ends up unreachable for a reason nobody can find: with both * active a packet must be accepted by BOTH, so the WireGuard rules in * nftables would be silently overruled by an ipset compiled from the public * subnet. One owner, named in one place. * - `enable 1` last, once neither of the above can bite. * * `policy_forward` is deliberately left alone: it only applies on a node * running the new nftables-based pve-firewall backend, which is opt-in and * off here, and setting a knob whose scope depends on which backend is * active would be guessing. * * The node is addressed by `$(hostname)` rather than by hosts.name because * that is exactly how Proxmox names the node; if the two ever diverge, the * machine is right and the database row is not. */ private function enableDatacenterFirewall(): ?StepResult { $commands = [ 'pvesh set /cluster/firewall/options --policy_in ACCEPT --policy_out ACCEPT', 'pvesh set /nodes/"$(hostname)"/firewall/options --enable 0', 'pvesh set /cluster/firewall/options --enable 1', ]; foreach ($commands as $command) { if (! $this->shell->run($command)->ok()) { return StepResult::retry(20, 'could not configure the Proxmox datacenter firewall: '.$command); } } // Read it back. `pvesh set` exiting 0 is not the same as the setting // being in cluster.fw: /etc/pve is a replicated filesystem and a write // that lost quorum is precisely the case where every customer VM's rules // would go on being inert while onboarding reported success. $options = $this->shell->run('pvesh get /cluster/firewall/options --output-format json')->stdout; if (! str_contains(preg_replace('/\s+/', '', $options) ?? '', '"enable":1')) { return StepResult::retry( 20, 'the Proxmox datacenter firewall does not read back as enabled; per-VM rules would have no effect' ); } return null; } }