feat(engine): SSH / WireGuard / Proxmox service layer

RemoteShell (phpseclib + fake), WireguardHub (local wg + fake), ProxmoxClient
(REST + fake). Interfaces bound in AppServiceProvider; tests swap fakes via
fakeServices() helper.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

app/Services/Proxmox/FakeProxmoxClient.php

@@ -0,0 +1,70 @@
+<?php
+
+namespace App\Services\Proxmox;
+
+use App\Models\Host;
+
+class FakeProxmoxClient implements ProxmoxClient
+{
+    public ?Host $host = null;
+
+    /** @var array<int, array<string, mixed>> */
+    public array $nodes = [['node' => 'pve']];
+
+    /** @var array<string, mixed> */
+    public array $status = [
+        'cpuinfo' => ['cpus' => 16],
+        'memory' => ['total' => 68719476736], // 64 GiB
+        'pveversion' => 'pve-manager/8.2.2',
+    ];
+
+    /** @var array<int, array<string, mixed>> */
+    public array $storage = [
+        ['storage' => 'local-lvm', 'type' => 'lvmthin', 'total' => 1099511627776], // 1 TiB
+    ];
+
+    /** @var array<int, string> */
+    public array $createdRoles = [];
+
+    /** @var array{token_id: string, secret: string}|null */
+    public ?array $createdToken = null;
+
+    public int $tokenCalls = 0;
+
+    public function forHost(Host $host): static
+    {
+        $this->host = $host;
+
+        return $this;
+    }
+
+    public function listNodes(): array
+    {
+        return $this->nodes;
+    }
+
+    public function nodeStatus(string $node): array
+    {
+        return $this->status;
+    }
+
+    public function nodeStorage(string $node): array
+    {
+        return $this->storage;
+    }
+
+    public function createRole(string $roleId, string $privs): void
+    {
+        $this->createdRoles[] = $roleId;
+    }
+
+    public function createUserAndToken(string $user, string $roleId): array
+    {
+        $this->tokenCalls++;
+
+        return $this->createdToken = [
+            'token_id' => $user.'!clupilot',
+            'secret' => 'fake-secret-'.substr(md5($user.$roleId), 0, 12),
+        ];
+    }
+}

app/Services/Proxmox/HttpProxmoxClient.php

@@ -0,0 +1,91 @@
+<?php
+
+namespace App\Services\Proxmox;
+
+use App\Models\Host;
+use Illuminate\Http\Client\PendingRequest;
+use Illuminate\Support\Facades\Http;
+use Illuminate\Support\Str;
+
+/**
+ * Real Proxmox VE REST client. Token auth over the WireGuard tunnel; the PVE
+ * certificate is self-signed so TLS verification is disabled (traffic already
+ * runs inside the encrypted tunnel). Not unit-tested (live I/O).
+ */
+class HttpProxmoxClient implements ProxmoxClient
+{
+    private string $baseUrl = '';
+
+    private string $token = '';
+
+    public function forHost(Host $host): static
+    {
+        $this->baseUrl = 'https://'.$host->wg_ip.':8006/api2/json';
+        $this->token = (string) $host->api_token_ref;
+
+        return $this;
+    }
+
+    private function http(): PendingRequest
+    {
+        return Http::withoutVerifying()
+            ->withHeaders(['Authorization' => 'PVEAPIToken='.$this->token])
+            ->baseUrl($this->baseUrl)
+            ->timeout(15);
+    }
+
+    public function listNodes(): array
+    {
+        return $this->http()->get('/nodes')->throw()->json('data', []);
+    }
+
+    public function nodeStatus(string $node): array
+    {
+        return $this->http()->get("/nodes/{$node}/status")->throw()->json('data', []);
+    }
+
+    public function nodeStorage(string $node): array
+    {
+        return $this->http()->get("/nodes/{$node}/storage")->throw()->json('data', []);
+    }
+
+    public function createRole(string $roleId, string $privs): void
+    {
+        $response = $this->http()->asForm()->post('/access/roles', [
+            'roleid' => $roleId,
+            'privs' => $privs,
+        ]);
+
+        // Idempotent: an existing role is not an error.
+        if ($response->failed() && ! str_contains($response->body(), 'already exists')) {
+            $response->throw();
+        }
+    }
+
+    public function createUserAndToken(string $user, string $roleId): array
+    {
+        // Ensure the user exists (ignore "already exists"), grant the role, mint a token.
+        $this->http()->asForm()->post('/access/users', [
+            'userid' => $user,
+            'password' => Str::random(40),
+        ]);
+
+        $this->http()->asForm()->put('/access/acl', [
+            'path' => '/',
+            'users' => $user,
+            'roles' => $roleId,
+        ])->throw();
+
+        $tokenName = config('provisioning.proxmox.token_name', 'clupilot');
+
+        $data = $this->http()->asForm()
+            ->post("/access/users/{$user}/token/{$tokenName}", ['privsep' => 0])
+            ->throw()
+            ->json('data', []);
+
+        return [
+            'token_id' => $data['full-tokenid'] ?? "{$user}!{$tokenName}",
+            'secret' => $data['value'] ?? '',
+        ];
+    }
+}

app/Services/Proxmox/ProxmoxClient.php

@@ -0,0 +1,30 @@
+<?php
+
+namespace App\Services\Proxmox;
+
+use App\Models\Host;
+
+/**
+ * Proxmox VE REST client, addressed per host over its wg_ip with a token.
+ * A v1.0 reads capacity and provisions the automation token; Subsystem B
+ * extends it with VM lifecycle methods (clone, cloud-init, guestExec …).
+ */
+interface ProxmoxClient
+{
+    /** Configure this client for a host (base URL from wg_ip, token from api_token_ref). */
+    public function forHost(Host $host): static;
+
+    /** @return array<int, array<string, mixed>> */
+    public function listNodes(): array;
+
+    /** @return array<string, mixed> */
+    public function nodeStatus(string $node): array;
+
+    /** @return array<int, array<string, mixed>> */
+    public function nodeStorage(string $node): array;
+
+    public function createRole(string $roleId, string $privs): void;
+
+    /** @return array{token_id: string, secret: string} */
+    public function createUserAndToken(string $user, string $roleId): array;
+}

app/Services/Ssh/CommandResult.php

@@ -0,0 +1,27 @@
+<?php
+
+namespace App\Services\Ssh;
+
+final class CommandResult
+{
+    public function __construct(
+        public readonly int $exitCode,
+        public readonly string $stdout = '',
+        public readonly string $stderr = '',
+    ) {}
+
+    public function ok(): bool
+    {
+        return $this->exitCode === 0;
+    }
+
+    public static function success(string $stdout = ''): self
+    {
+        return new self(0, $stdout);
+    }
+
+    public static function failure(int $exitCode = 1, string $stderr = ''): self
+    {
+        return new self($exitCode, '', $stderr);
+    }
+}

app/Services/Ssh/FakeRemoteShell.php

@@ -0,0 +1,108 @@
+<?php
+
+namespace App\Services\Ssh;
+
+/**
+ * Test double: script command output by substring, record everything.
+ */
+class FakeRemoteShell implements RemoteShell
+{
+    /** @var array<string, CommandResult|callable> */
+    private array $scripts = [];
+
+    /** @var array<int, string> */
+    private array $recorded = [];
+
+    /** @var array<string, string> */
+    private array $files = [];
+
+    /** @var array<int, array{0:string,1:string,2:string}> */
+    public array $connections = [];
+
+    public string $fingerprint = 'SHA256:fakehostkeyfingerprint';
+
+    private CommandResult $default;
+
+    public function __construct()
+    {
+        $this->default = CommandResult::success();
+    }
+
+    public function script(string $substring, CommandResult|callable $result): static
+    {
+        $this->scripts[$substring] = $result;
+
+        return $this;
+    }
+
+    public function defaultResult(CommandResult $result): static
+    {
+        $this->default = $result;
+
+        return $this;
+    }
+
+    public function connectWithPassword(string $host, string $user, string $password): void
+    {
+        $this->connections[] = ['password', $host, $user];
+    }
+
+    public function connectWithKey(string $host, string $user, string $privateKey): void
+    {
+        $this->connections[] = ['key', $host, $user];
+    }
+
+    public function run(string $command): CommandResult
+    {
+        $this->recorded[] = $command;
+
+        foreach ($this->scripts as $substring => $result) {
+            if (str_contains($command, $substring)) {
+                return is_callable($result) ? $result($command) : $result;
+            }
+        }
+
+        return $this->default;
+    }
+
+    public function putFile(string $remotePath, string $contents): void
+    {
+        $this->files[$remotePath] = $contents;
+    }
+
+    public function hostKeyFingerprint(): string
+    {
+        return $this->fingerprint;
+    }
+
+    // --- introspection helpers for tests ---
+
+    /** @return array<int, string> */
+    public function recorded(): array
+    {
+        return $this->recorded;
+    }
+
+    public function ran(string $substring): bool
+    {
+        foreach ($this->recorded as $command) {
+            if (str_contains($command, $substring)) {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
+    /** @return array<string, string> */
+    public function files(): array
+    {
+        return $this->files;
+    }
+
+    /** @return array<int, mixed> */
+    public function connectionsWith(string $type): array
+    {
+        return array_values(array_filter($this->connections, fn ($c) => $c[0] === $type));
+    }
+}

app/Services/Ssh/PhpseclibRemoteShell.php

@@ -0,0 +1,77 @@
+<?php
+
+namespace App\Services\Ssh;
+
+use phpseclib3\Crypt\PublicKeyLoader;
+use phpseclib3\Net\SSH2;
+use RuntimeException;
+
+/**
+ * Real SSH session via phpseclib3. Not unit-tested (live I/O); verified against
+ * a real host during end-to-end runs. Files are written via a base64 heredoc so
+ * a single SSH channel suffices (no separate SFTP subsystem needed).
+ */
+class PhpseclibRemoteShell implements RemoteShell
+{
+    private ?SSH2 $ssh = null;
+
+    public function connectWithPassword(string $host, string $user, string $password): void
+    {
+        $ssh = new SSH2($host);
+
+        if (! $ssh->login($user, $password)) {
+            throw new RuntimeException("SSH password login failed for {$user}@{$host}");
+        }
+
+        $this->ssh = $ssh;
+    }
+
+    public function connectWithKey(string $host, string $user, string $privateKey): void
+    {
+        $key = PublicKeyLoader::load($privateKey);
+        $ssh = new SSH2($host);
+
+        if (! $ssh->login($user, $key)) {
+            throw new RuntimeException("SSH key login failed for {$user}@{$host}");
+        }
+
+        $this->ssh = $ssh;
+    }
+
+    public function run(string $command): CommandResult
+    {
+        $stdout = (string) $this->ssh()->exec($command);
+        $exit = $this->ssh()->getExitStatus();
+
+        return new CommandResult(is_int($exit) ? $exit : 0, $stdout, '');
+    }
+
+    public function putFile(string $remotePath, string $contents): void
+    {
+        $b64 = base64_encode($contents);
+        $dir = escapeshellarg(dirname($remotePath));
+        $path = escapeshellarg($remotePath);
+
+        $result = $this->run("mkdir -p {$dir} && printf %s ".escapeshellarg($b64)." | base64 -d > {$path}");
+
+        if (! $result->ok()) {
+            throw new RuntimeException("Failed to write remote file {$remotePath}");
+        }
+    }
+
+    public function hostKeyFingerprint(): string
+    {
+        $key = $this->ssh()->getServerPublicHostKey();
+
+        return $key === false ? '' : 'SHA256:'.base64_encode(hash('sha256', (string) $key, true));
+    }
+
+    private function ssh(): SSH2
+    {
+        if ($this->ssh === null) {
+            throw new RuntimeException('RemoteShell is not connected.');
+        }
+
+        return $this->ssh;
+    }
+}

app/Services/Ssh/RemoteShell.php

@@ -0,0 +1,21 @@
+<?php
+
+namespace App\Services\Ssh;
+
+/**
+ * A connected SSH session to a single host. Implementations are stateful:
+ * connect once, then run/putFile against that connection.
+ */
+interface RemoteShell
+{
+    public function connectWithPassword(string $host, string $user, string $password): void;
+
+    public function connectWithKey(string $host, string $user, string $privateKey): void;
+
+    public function run(string $command): CommandResult;
+
+    public function putFile(string $remotePath, string $contents): void;
+
+    /** SSH host key fingerprint of the connected server (for pinning). */
+    public function hostKeyFingerprint(): string;
+}

app/Services/Wireguard/FakeWireguardHub.php

@@ -0,0 +1,46 @@
+<?php
+
+namespace App\Services\Wireguard;
+
+class FakeWireguardHub implements WireguardHub
+{
+    /** @var array<string, string> pubkey => ip */
+    private array $peers = [];
+
+    private int $next = 2;
+
+    public string $endpointValue = 'vpn.clupilot.test:51820';
+
+    public string $publicKeyValue = 'HUBPUBLICKEY0000000000000000000000000000000=';
+
+    public function allocateIp(): string
+    {
+        return '10.66.0.'.$this->next++;
+    }
+
+    public function addPeer(string $publicKey, string $ip): void
+    {
+        $this->peers[$publicKey] = $ip;
+    }
+
+    public function removePeer(string $publicKey): void
+    {
+        unset($this->peers[$publicKey]);
+    }
+
+    public function endpoint(): string
+    {
+        return $this->endpointValue;
+    }
+
+    public function publicKey(): string
+    {
+        return $this->publicKeyValue;
+    }
+
+    /** @return array<string, string> */
+    public function peers(): array
+    {
+        return $this->peers;
+    }
+}

app/Services/Wireguard/LocalWireguardHub.php

@@ -0,0 +1,54 @@
+<?php
+
+namespace App\Services\Wireguard;
+
+use App\Models\Host;
+use Illuminate\Support\Facades\Process;
+use Illuminate\Support\Str;
+use RuntimeException;
+
+/**
+ * Real hub on the CluPilot VM. Allocates the next free address in the configured
+ * subnet and manages peers with `wg`. Watch-item: the app runs in a container, so
+ * the wg interface must be reachable from here (host network / mounted config);
+ * verified on the real VM, not in the mocked test-suite.
+ */
+class LocalWireguardHub implements WireguardHub
+{
+    public function allocateIp(): string
+    {
+        $used = Host::query()->whereNotNull('wg_ip')->pluck('wg_ip')->all();
+        $prefix = Str::beforeLast(config('provisioning.wireguard.subnet', '10.66.0.0/24'), '.');
+
+        for ($octet = 2; $octet <= 254; $octet++) {
+            $ip = "{$prefix}.{$octet}";
+            if (! in_array($ip, $used, true)) {
+                return $ip;
+            }
+        }
+
+        throw new RuntimeException('WireGuard management subnet exhausted.');
+    }
+
+    public function addPeer(string $publicKey, string $ip): void
+    {
+        Process::run(['wg', 'set', 'wg0', 'peer', $publicKey, 'allowed-ips', $ip.'/32'])->throw();
+        Process::run('wg-quick save wg0');
+    }
+
+    public function removePeer(string $publicKey): void
+    {
+        Process::run(['wg', 'set', 'wg0', 'peer', $publicKey, 'remove'])->throw();
+        Process::run('wg-quick save wg0');
+    }
+
+    public function endpoint(): string
+    {
+        return (string) config('provisioning.wireguard.endpoint');
+    }
+
+    public function publicKey(): string
+    {
+        return (string) config('provisioning.wireguard.hub_public_key');
+    }
+}

app/Services/Wireguard/WireguardHub.php

@@ -0,0 +1,23 @@
+<?php
+
+namespace App\Services\Wireguard;
+
+/**
+ * The CluPilot VM's WireGuard hub. Onboarding allocates a management IP and
+ * registers the new host as a peer so its Proxmox API is reachable over the tunnel.
+ */
+interface WireguardHub
+{
+    /** Allocate a free management IP from the hub subnet. */
+    public function allocateIp(): string;
+
+    public function addPeer(string $publicKey, string $ip): void;
+
+    public function removePeer(string $publicKey): void;
+
+    /** Public endpoint (host:port) peers dial to reach the hub. */
+    public function endpoint(): string;
+
+    /** The hub's WireGuard public key. */
+    public function publicKey(): string;
+}

tests/Feature/Provisioning/ServicesTest.php

@@ -0,0 +1,48 @@
+<?php
+
+use App\Models\Host;
+use App\Services\Proxmox\FakeProxmoxClient;
+use App\Services\Ssh\CommandResult;
+use App\Services\Ssh\FakeRemoteShell;
+use App\Services\Wireguard\FakeWireguardHub;
+
+it('scripts remote command output and records calls (FakeRemoteShell)', function () {
+    $shell = new FakeRemoteShell;
+    $shell->script('dpkg -l proxmox-ve', CommandResult::success('ii  proxmox-ve  8.2'));
+    $shell->connectWithPassword('1.2.3.4', 'root', 'pw');
+
+    expect($shell->run('dpkg -l proxmox-ve')->stdout)->toContain('proxmox-ve')
+        ->and($shell->run('whoami')->ok())->toBeTrue()          // default success
+        ->and($shell->ran('dpkg -l proxmox-ve'))->toBeTrue();
+
+    $shell->putFile('/etc/wireguard/wg0.conf', '[Interface]');
+    expect($shell->files()['/etc/wireguard/wg0.conf'])->toBe('[Interface]')
+        ->and($shell->connectionsWith('password'))->toHaveCount(1);
+});
+
+it('allocates ips and tracks peers (FakeWireguardHub)', function () {
+    $hub = new FakeWireguardHub;
+
+    $a = $hub->allocateIp();
+    $b = $hub->allocateIp();
+    expect($a)->not->toBe($b);
+
+    $hub->addPeer('PUBKEY', '10.66.0.9');
+    expect($hub->peers())->toBe(['PUBKEY' => '10.66.0.9']);
+
+    $hub->removePeer('PUBKEY');
+    expect($hub->peers())->toBe([]);
+});
+
+it('reports capacity and mints a token for a host (FakeProxmoxClient)', function () {
+    $host = Host::factory()->create(['wg_ip' => '10.66.0.5']);
+    $client = (new FakeProxmoxClient)->forHost($host);
+
+    expect($client->listNodes())->not->toBeEmpty()
+        ->and($client->nodeStatus('pve')['memory']['total'])->toBeGreaterThan(0)
+        ->and($client->nodeStorage('pve')[0]['total'])->toBeGreaterThan(0);
+
+    $token = $client->createUserAndToken('automation@pve', 'CluPilotAutomation');
+    expect($token)->toHaveKeys(['token_id', 'secret'])
+        ->and($client->tokenCalls)->toBe(1);
+});

tests/Pest.php

@@ -48,3 +48,22 @@ function something()
 {
     // ..
 }
+
+/**
+ * Bind fake provisioning services into the container and return them so a test
+ * can script/inspect SSH, WireGuard and Proxmox interactions.
+ *
+ * @return array{shell: \App\Services\Ssh\FakeRemoteShell, hub: \App\Services\Wireguard\FakeWireguardHub, pve: \App\Services\Proxmox\FakeProxmoxClient}
+ */
+function fakeServices(): array
+{
+    $shell = new \App\Services\Ssh\FakeRemoteShell;
+    $hub = new \App\Services\Wireguard\FakeWireguardHub;
+    $pve = new \App\Services\Proxmox\FakeProxmoxClient;
+
+    app()->instance(\App\Services\Ssh\RemoteShell::class, $shell);
+    app()->instance(\App\Services\Wireguard\WireguardHub::class, $hub);
+    app()->instance(\App\Services\Proxmox\ProxmoxClient::class, $pve);
+
+    return ['shell' => $shell, 'hub' => $hub, 'pve' => $pve];
+}