From 446da58061969d5b7a638a841fea64def534e816 Mon Sep 17 00:00:00 2001
From: nexxo <nexxo@clupilot.local>
Date: Sat, 25 Jul 2026 11:50:26 +0200
Subject: [PATCH] feat(engine-b): 15-step customer pipeline + DNS/Traefik
 services

validate->reserve(placement)->clone->cloudinit->start->guestagent->network->
deploy->nextcloud->admin->dns/tls->backup->monitoring->acceptance->complete.
HetznerDnsClient + TraefikWriter (interface+fake+real), CloudReady notification,
hosts.node (set by RegisterCapacity). Secrets transient/encrypted, never
plaintext. 21 tests incl. mocked end-to-end + crash idempotency.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
---
 app/Models/Host.php                           |   2 +-
 app/Notifications/CloudReady.php              |  42 +++
 app/Providers/AppServiceProvider.php          |   6 +
 .../Steps/Customer/CloneVirtualMachine.php    |  65 ++++
 .../Steps/Customer/CompleteProvisioning.php   |  46 +++
 .../Steps/Customer/ConfigureCloudInit.php     |  40 +++
 .../Steps/Customer/ConfigureDnsAndTls.php     |  62 ++++
 .../Steps/Customer/ConfigureNetwork.php       |  40 +++
 .../Steps/Customer/ConfigureNextcloud.php     |  41 +++
 .../Steps/Customer/CreateCustomerAdmin.php    |  52 +++
 .../Steps/Customer/DeployApplicationStack.php |  55 ++++
 .../Steps/Customer/RegisterBackup.php         |  38 +++
 .../Steps/Customer/RegisterMonitoring.php     |  39 +++
 .../Steps/Customer/ReserveResources.php       |  82 +++++
 .../Steps/Customer/RunAcceptanceChecks.php    |  59 ++++
 .../Steps/Customer/StartVirtualMachine.php    |  51 +++
 .../Steps/Customer/ValidateOrder.php          |  38 +++
 .../Steps/Customer/WaitForGuestAgent.php      |  40 +++
 .../Steps/Host/RegisterCapacity.php           |   1 +
 app/Services/Dns/FakeHetznerDnsClient.php     |  29 ++
 app/Services/Dns/HetznerDnsClient.php         |  11 +
 app/Services/Dns/HttpHetznerDnsClient.php     |  55 ++++
 app/Services/Traefik/FakeTraefikWriter.php    |  26 ++
 app/Services/Traefik/SshTraefikWriter.php     |  57 ++++
 app/Services/Traefik/TraefikWriter.php        |  14 +
 ...6_07_25_080008_add_node_to_hosts_table.php |  24 ++
 .../CustomerProvisioningEndToEndTest.php      |  90 ++++++
 .../Provisioning/CustomerStepsTest.php        | 302 ++++++++++++++++++
 tests/Pest.php                                |  10 +-
 29 files changed, 1413 insertions(+), 4 deletions(-)
 create mode 100644 app/Notifications/CloudReady.php
 create mode 100644 app/Provisioning/Steps/Customer/CloneVirtualMachine.php
 create mode 100644 app/Provisioning/Steps/Customer/CompleteProvisioning.php
 create mode 100644 app/Provisioning/Steps/Customer/ConfigureCloudInit.php
 create mode 100644 app/Provisioning/Steps/Customer/ConfigureDnsAndTls.php
 create mode 100644 app/Provisioning/Steps/Customer/ConfigureNetwork.php
 create mode 100644 app/Provisioning/Steps/Customer/ConfigureNextcloud.php
 create mode 100644 app/Provisioning/Steps/Customer/CreateCustomerAdmin.php
 create mode 100644 app/Provisioning/Steps/Customer/DeployApplicationStack.php
 create mode 100644 app/Provisioning/Steps/Customer/RegisterBackup.php
 create mode 100644 app/Provisioning/Steps/Customer/RegisterMonitoring.php
 create mode 100644 app/Provisioning/Steps/Customer/ReserveResources.php
 create mode 100644 app/Provisioning/Steps/Customer/RunAcceptanceChecks.php
 create mode 100644 app/Provisioning/Steps/Customer/StartVirtualMachine.php
 create mode 100644 app/Provisioning/Steps/Customer/ValidateOrder.php
 create mode 100644 app/Provisioning/Steps/Customer/WaitForGuestAgent.php
 create mode 100644 app/Services/Dns/FakeHetznerDnsClient.php
 create mode 100644 app/Services/Dns/HetznerDnsClient.php
 create mode 100644 app/Services/Dns/HttpHetznerDnsClient.php
 create mode 100644 app/Services/Traefik/FakeTraefikWriter.php
 create mode 100644 app/Services/Traefik/SshTraefikWriter.php
 create mode 100644 app/Services/Traefik/TraefikWriter.php
 create mode 100644 database/migrations/2026_07_25_080008_add_node_to_hosts_table.php
 create mode 100644 tests/Feature/Provisioning/CustomerProvisioningEndToEndTest.php
 create mode 100644 tests/Feature/Provisioning/CustomerStepsTest.php

diff --git a/app/Models/Host.php b/app/Models/Host.php
index 69719bc..185b082 100644
--- a/app/Models/Host.php
+++ b/app/Models/Host.php
@@ -17,7 +17,7 @@ class Host extends Model implements ProvisioningSubject
     protected $fillable = [
         'name', 'datacenter', 'cluster', 'public_ip', 'wg_ip', 'wg_pubkey',
         'ssh_host_key', 'api_token_ref', 'total_gb', 'total_ram_mb', 'cpu_cores',
-        'cpu_weight', 'reserve_pct', 'pve_version', 'status', 'last_seen_at',
+        'cpu_weight', 'reserve_pct', 'pve_version', 'node', 'status', 'last_seen_at',
     ];
 
     protected $hidden = ['api_token_ref'];
diff --git a/app/Notifications/CloudReady.php b/app/Notifications/CloudReady.php
new file mode 100644
index 0000000..193f677
--- /dev/null
+++ b/app/Notifications/CloudReady.php
@@ -0,0 +1,42 @@
+<?php
+
+namespace App\Notifications;
+
+use App\Models\Instance;
+use Illuminate\Bus\Queueable;
+use Illuminate\Notifications\Messages\MailMessage;
+use Illuminate\Notifications\Notification;
+
+/**
+ * Delivers the initial admin credentials once provisioning completes. The
+ * password is passed transiently and never persisted in plaintext.
+ */
+class CloudReady extends Notification
+{
+    use Queueable;
+
+    public function __construct(
+        public Instance $instance,
+        public string $adminUser,
+        public string $adminPassword,
+    ) {}
+
+    /** @return array<int, string> */
+    public function via(object $notifiable): array
+    {
+        return ['mail'];
+    }
+
+    public function toMail(object $notifiable): MailMessage
+    {
+        $url = 'https://'.$this->instance->subdomain.'.'.config('provisioning.dns.zone');
+
+        return (new MailMessage)
+            ->subject(__('provisioning.mail.ready_subject'))
+            ->greeting(__('provisioning.mail.ready_greeting'))
+            ->line(__('provisioning.mail.ready_line'))
+            ->line(__('provisioning.mail.ready_user', ['user' => $this->adminUser]))
+            ->line(__('provisioning.mail.ready_password', ['password' => $this->adminPassword]))
+            ->action(__('provisioning.mail.ready_action'), $url);
+    }
+}
diff --git a/app/Providers/AppServiceProvider.php b/app/Providers/AppServiceProvider.php
index 2d6bdf7..fad8432 100644
--- a/app/Providers/AppServiceProvider.php
+++ b/app/Providers/AppServiceProvider.php
@@ -3,10 +3,14 @@
 namespace App\Providers;
 
 use App\Provisioning\PipelineRegistry;
+use App\Services\Dns\HetznerDnsClient;
+use App\Services\Dns\HttpHetznerDnsClient;
 use App\Services\Proxmox\HttpProxmoxClient;
 use App\Services\Proxmox\ProxmoxClient;
 use App\Services\Ssh\PhpseclibRemoteShell;
 use App\Services\Ssh\RemoteShell;
+use App\Services\Traefik\SshTraefikWriter;
+use App\Services\Traefik\TraefikWriter;
 use App\Services\Wireguard\LocalWireguardHub;
 use App\Services\Wireguard\WireguardHub;
 use Illuminate\Support\ServiceProvider;
@@ -26,6 +30,8 @@ class AppServiceProvider extends ServiceProvider
         $this->app->bind(RemoteShell::class, PhpseclibRemoteShell::class);
         $this->app->bind(WireguardHub::class, LocalWireguardHub::class);
         $this->app->bind(ProxmoxClient::class, HttpProxmoxClient::class);
+        $this->app->bind(HetznerDnsClient::class, HttpHetznerDnsClient::class);
+        $this->app->bind(TraefikWriter::class, SshTraefikWriter::class);
     }
 
     /**
diff --git a/app/Provisioning/Steps/Customer/CloneVirtualMachine.php b/app/Provisioning/Steps/Customer/CloneVirtualMachine.php
new file mode 100644
index 0000000..9c7d21e
--- /dev/null
+++ b/app/Provisioning/Steps/Customer/CloneVirtualMachine.php
@@ -0,0 +1,65 @@
+<?php
+
+namespace App\Provisioning\Steps\Customer;
+
+use App\Models\ProvisioningRun;
+use App\Provisioning\StepResult;
+use App\Services\Proxmox\ProxmoxClient;
+
+class CloneVirtualMachine extends CustomerStep
+{
+    public function __construct(private ProxmoxClient $pve) {}
+
+    public function key(): string
+    {
+        return 'clone_vm';
+    }
+
+    public function maxDuration(): int
+    {
+        return 900;
+    }
+
+    public function execute(ProvisioningRun $run): StepResult
+    {
+        $instance = $this->instance($run);
+        $node = (string) $run->context('node');
+        $plan = $this->plan($run);
+        $templateVmid = (int) ($plan['template_vmid'] ?? 0);
+
+        if ($templateVmid === 0) {
+            return StepResult::fail('template_missing');
+        }
+
+        $pve = $this->pve->forHost($instance->host);
+
+        // Idempotent: vmid already minted → resume polling the clone task.
+        if ($this->hasResource($run, 'vmid')) {
+            return $this->pollClone($pve, $node, (string) $run->context('clone_upid'));
+        }
+
+        $vmid = $pve->nextVmid();
+        $upid = $pve->cloneVm($node, $templateVmid, $vmid, 'nc-'.$instance->subdomain);
+
+        // [E] persist vmid + task ref BEFORE polling (crash-safe).
+        $instance->update(['vmid' => $vmid, 'status' => 'provisioning']);
+        $run->mergeContext(['vmid' => $vmid, 'clone_upid' => $upid]);
+        $this->recordResource($run, $instance->host, 'vmid', (string) $vmid);
+
+        return $this->pollClone($pve, $node, $upid);
+    }
+
+    private function pollClone(ProxmoxClient $pve, string $node, string $upid): StepResult
+    {
+        $task = $pve->taskStatus($node, $upid);
+
+        if (($task['status'] ?? '') === 'running') {
+            return StepResult::poll(10, 'cloning template');
+        }
+        if (($task['exitstatus'] ?? 'OK') !== 'OK') {
+            return StepResult::fail('clone_failed: '.($task['exitstatus'] ?? 'unknown'));
+        }
+
+        return StepResult::advance();
+    }
+}
diff --git a/app/Provisioning/Steps/Customer/CompleteProvisioning.php b/app/Provisioning/Steps/Customer/CompleteProvisioning.php
new file mode 100644
index 0000000..2641dbb
--- /dev/null
+++ b/app/Provisioning/Steps/Customer/CompleteProvisioning.php
@@ -0,0 +1,46 @@
+<?php
+
+namespace App\Provisioning\Steps\Customer;
+
+use App\Models\ProvisioningRun;
+use App\Notifications\CloudReady;
+use App\Provisioning\StepResult;
+use Illuminate\Support\Facades\Crypt;
+use Illuminate\Support\Facades\Notification;
+
+class CompleteProvisioning extends CustomerStep
+{
+    public function key(): string
+    {
+        return 'complete_provisioning';
+    }
+
+    public function maxDuration(): int
+    {
+        return 60;
+    }
+
+    public function execute(ProvisioningRun $run): StepResult
+    {
+        $order = $this->order($run);
+        $instance = $this->instance($run);
+
+        $instance->update(['status' => 'active']);
+        $order->update(['status' => 'active']);
+
+        foreach (['invite_users', 'upload_logo', 'connect_desktop', 'explore_apps'] as $key) {
+            $instance->onboardingTasks()->firstOrCreate(['key' => $key], ['done' => false]);
+        }
+
+        // Deliver the initial credentials, then scrub the transient password.
+        $encrypted = $run->context('admin_password');
+        if ($encrypted !== null) {
+            Notification::route('mail', $order->customer->email)->notify(
+                new CloudReady($instance, (string) $instance->nc_admin_ref, Crypt::decryptString($encrypted)),
+            );
+            $run->forgetContext('admin_password');
+        }
+
+        return StepResult::advance();
+    }
+}
diff --git a/app/Provisioning/Steps/Customer/ConfigureCloudInit.php b/app/Provisioning/Steps/Customer/ConfigureCloudInit.php
new file mode 100644
index 0000000..2672f22
--- /dev/null
+++ b/app/Provisioning/Steps/Customer/ConfigureCloudInit.php
@@ -0,0 +1,40 @@
+<?php
+
+namespace App\Provisioning\Steps\Customer;
+
+use App\Models\ProvisioningRun;
+use App\Provisioning\StepResult;
+use App\Services\Proxmox\ProxmoxClient;
+
+class ConfigureCloudInit extends CustomerStep
+{
+    public function __construct(private ProxmoxClient $pve) {}
+
+    public function key(): string
+    {
+        return 'configure_cloud_init';
+    }
+
+    public function maxDuration(): int
+    {
+        return 120;
+    }
+
+    public function execute(ProvisioningRun $run): StepResult
+    {
+        $instance = $this->instance($run);
+        $node = (string) $run->context('node');
+        $vmid = (int) $run->context('vmid');
+        $pve = $this->pve->forHost($instance->host);
+
+        // Idempotent: setting the same values / resizing to a size not smaller is a no-op.
+        $pve->setCloudInit($node, $vmid, [
+            'ipconfig0' => 'ip=dhcp',
+            'ciuser' => 'clupilot',
+            'nameserver' => '1.1.1.1',
+        ]);
+        $pve->resizeDisk($node, $vmid, 'scsi0', '+'.$instance->disk_gb.'G');
+
+        return StepResult::advance();
+    }
+}
diff --git a/app/Provisioning/Steps/Customer/ConfigureDnsAndTls.php b/app/Provisioning/Steps/Customer/ConfigureDnsAndTls.php
new file mode 100644
index 0000000..9cc14af
--- /dev/null
+++ b/app/Provisioning/Steps/Customer/ConfigureDnsAndTls.php
@@ -0,0 +1,62 @@
+<?php
+
+namespace App\Provisioning\Steps\Customer;
+
+use App\Models\ProvisioningRun;
+use App\Provisioning\StepResult;
+use App\Services\Dns\HetznerDnsClient;
+use App\Services\Traefik\TraefikWriter;
+
+class ConfigureDnsAndTls extends CustomerStep
+{
+    public function __construct(
+        private HetznerDnsClient $dns,
+        private TraefikWriter $traefik,
+    ) {}
+
+    public function key(): string
+    {
+        return 'configure_dns_and_tls';
+    }
+
+    public function maxDuration(): int
+    {
+        return 900;
+    }
+
+    public function execute(ProvisioningRun $run): StepResult
+    {
+        $instance = $this->instance($run);
+        $host = $instance->host;
+        $fqdn = $instance->subdomain.'.'.config('provisioning.dns.zone');
+
+        // DNS — persist the record id [E] before anything downstream.
+        if (! $this->hasResource($run, 'dns_record_id')) {
+            $recordId = $this->dns->upsertRecord($fqdn, 'A', $host->public_ip);
+            $instance->dnsRecords()->create([
+                'provider' => 'hetzner', 'record_id' => $recordId,
+                'fqdn' => $fqdn, 'type' => 'A', 'value' => $host->public_ip,
+            ]);
+            $this->recordResource($run, $host, 'dns_record_id', $recordId);
+        }
+
+        // Traefik file-provider route (subdomain → guest over the tunnel).
+        if (! $instance->route_written) {
+            $this->traefik->write($instance->subdomain, $host->wg_ip ?? $host->public_ip);
+            $instance->update(['route_written' => true]);
+        }
+
+        // TLS via HTTP-01 — poll until the certificate is served.
+        if (! $instance->cert_ok) {
+            if ($this->traefik->certReachable($fqdn)) {
+                $instance->update(['cert_ok' => true]);
+            } elseif ($run->started_at !== null && $run->started_at->copy()->addSeconds(840)->isPast()) {
+                return StepResult::fail('cert_timeout');
+            } else {
+                return StepResult::poll(20, 'waiting for certificate');
+            }
+        }
+
+        return StepResult::advance();
+    }
+}
diff --git a/app/Provisioning/Steps/Customer/ConfigureNetwork.php b/app/Provisioning/Steps/Customer/ConfigureNetwork.php
new file mode 100644
index 0000000..27e5da9
--- /dev/null
+++ b/app/Provisioning/Steps/Customer/ConfigureNetwork.php
@@ -0,0 +1,40 @@
+<?php
+
+namespace App\Provisioning\Steps\Customer;
+
+use App\Models\ProvisioningRun;
+use App\Provisioning\StepResult;
+use App\Services\Proxmox\ProxmoxClient;
+
+class ConfigureNetwork extends CustomerStep
+{
+    public function __construct(private ProxmoxClient $pve) {}
+
+    public function key(): string
+    {
+        return 'configure_network';
+    }
+
+    public function maxDuration(): int
+    {
+        return 120;
+    }
+
+    public function execute(ProvisioningRun $run): StepResult
+    {
+        $instance = $this->instance($run);
+        $node = (string) $run->context('node');
+        $vmid = (int) $run->context('vmid');
+        $pve = $this->pve->forHost($instance->host);
+
+        $this->guest($pve, $run, 'hostnamectl set-hostname '.escapeshellarg($instance->subdomain));
+
+        // Allow HTTP/HTTPS, deny everything else (management stays on the tunnel).
+        $pve->applyFirewall($node, $vmid, [
+            ['type' => 'in', 'action' => 'ACCEPT', 'proto' => 'tcp', 'dport' => '80'],
+            ['type' => 'in', 'action' => 'ACCEPT', 'proto' => 'tcp', 'dport' => '443'],
+        ]);
+
+        return StepResult::advance();
+    }
+}
diff --git a/app/Provisioning/Steps/Customer/ConfigureNextcloud.php b/app/Provisioning/Steps/Customer/ConfigureNextcloud.php
new file mode 100644
index 0000000..cb13c90
--- /dev/null
+++ b/app/Provisioning/Steps/Customer/ConfigureNextcloud.php
@@ -0,0 +1,41 @@
+<?php
+
+namespace App\Provisioning\Steps\Customer;
+
+use App\Models\ProvisioningRun;
+use App\Provisioning\StepResult;
+use App\Services\Proxmox\ProxmoxClient;
+
+class ConfigureNextcloud extends CustomerStep
+{
+    public function __construct(private ProxmoxClient $pve) {}
+
+    public function key(): string
+    {
+        return 'configure_nextcloud';
+    }
+
+    public function maxDuration(): int
+    {
+        return 300;
+    }
+
+    public function execute(ProvisioningRun $run): StepResult
+    {
+        $instance = $this->instance($run);
+        $pve = $this->pve->forHost($instance->host);
+
+        $fqdn = $instance->subdomain.'.'.config('provisioning.dns.zone');
+        $occ = 'cd /opt/nextcloud && docker compose exec -T app php occ ';
+
+        // Idempotent occ calls (setting the same values is a no-op).
+        $this->guest($pve, $run, $occ.'config:system:set trusted_domains 1 --value='.escapeshellarg($fqdn));
+        if (filled($instance->custom_domain)) {
+            $this->guest($pve, $run, $occ.'config:system:set trusted_domains 2 --value='.escapeshellarg($instance->custom_domain));
+        }
+        $this->guest($pve, $run, $occ.'background:cron');
+        $this->guest($pve, $run, $occ.'config:system:set default_phone_region --value=DE');
+
+        return StepResult::advance();
+    }
+}
diff --git a/app/Provisioning/Steps/Customer/CreateCustomerAdmin.php b/app/Provisioning/Steps/Customer/CreateCustomerAdmin.php
new file mode 100644
index 0000000..8bdd90f
--- /dev/null
+++ b/app/Provisioning/Steps/Customer/CreateCustomerAdmin.php
@@ -0,0 +1,52 @@
+<?php
+
+namespace App\Provisioning\Steps\Customer;
+
+use App\Models\ProvisioningRun;
+use App\Provisioning\StepResult;
+use App\Services\Proxmox\ProxmoxClient;
+use Illuminate\Support\Facades\Crypt;
+use Illuminate\Support\Str;
+
+class CreateCustomerAdmin extends CustomerStep
+{
+    public function __construct(private ProxmoxClient $pve) {}
+
+    public function key(): string
+    {
+        return 'create_customer_admin';
+    }
+
+    public function maxDuration(): int
+    {
+        return 120;
+    }
+
+    public function execute(ProvisioningRun $run): StepResult
+    {
+        // Idempotent: admin already created.
+        if ($this->hasResource($run, 'nc_admin')) {
+            return StepResult::advance();
+        }
+
+        $instance = $this->instance($run);
+        $pve = $this->pve->forHost($instance->host);
+
+        $username = 'admin';
+        $password = Str::random(20);
+
+        // Pass the password via env (OC_PASS), never on the argv/command line.
+        $this->guest($pve, $run,
+            'cd /opt/nextcloud && OC_PASS='.escapeshellarg($password).
+            ' docker compose exec -T -e OC_PASS app php occ user:add --password-from-env --group=admin '.
+            escapeshellarg($username));
+
+        // Persist only the username (encrypted ref); hand the password to step 15
+        // encrypted-in-context for delivery, then it is scrubbed. Never plaintext.
+        $instance->update(['nc_admin_ref' => $username]);
+        $run->mergeContext(['admin_password' => Crypt::encryptString($password)]);
+        $this->recordResource($run, $instance->host, 'nc_admin', $username);
+
+        return StepResult::advance();
+    }
+}
diff --git a/app/Provisioning/Steps/Customer/DeployApplicationStack.php b/app/Provisioning/Steps/Customer/DeployApplicationStack.php
new file mode 100644
index 0000000..b42d217
--- /dev/null
+++ b/app/Provisioning/Steps/Customer/DeployApplicationStack.php
@@ -0,0 +1,55 @@
+<?php
+
+namespace App\Provisioning\Steps\Customer;
+
+use App\Models\ProvisioningRun;
+use App\Provisioning\StepResult;
+use App\Services\Proxmox\ProxmoxClient;
+use Illuminate\Support\Str;
+
+class DeployApplicationStack extends CustomerStep
+{
+    public function __construct(private ProxmoxClient $pve) {}
+
+    public function key(): string
+    {
+        return 'deploy_application_stack';
+    }
+
+    public function maxDuration(): int
+    {
+        return 1200;
+    }
+
+    public function execute(ProvisioningRun $run): StepResult
+    {
+        $instance = $this->instance($run);
+        $node = (string) $run->context('node');
+        $vmid = (int) $run->context('vmid');
+        $pve = $this->pve->forHost($instance->host);
+
+        // Deploy once (guarded); the generated DB password is transient.
+        if (! $run->context('stack_deployed')) {
+            $dbPassword = Str::random(28);
+            $run->mergeContext(['db_password' => $dbPassword]);
+
+            $this->guest($pve, $run,
+                'install -d /opt/nextcloud && printf %s '.escapeshellarg('MYSQL_PASSWORD='.$dbPassword."\n").
+                ' > /opt/nextcloud/.env');
+            $this->guest($pve, $run, 'cd /opt/nextcloud && docker compose up -d');
+
+            $run->mergeContext(['stack_deployed' => true]);
+        }
+
+        // Poll until Nextcloud answers (poll doesn't consume the retry budget).
+        $health = $pve->guestExec($node, $vmid,
+            'curl -sf http://localhost/status.php >/dev/null 2>&1 && echo ok || echo wait');
+        if (trim($health['out-data'] ?? '') !== 'ok') {
+            return StepResult::poll(15, 'waiting for application stack');
+        }
+
+        $run->forgetContext('db_password'); // scrub the secret once the stack is up
+
+        return StepResult::advance();
+    }
+}
diff --git a/app/Provisioning/Steps/Customer/RegisterBackup.php b/app/Provisioning/Steps/Customer/RegisterBackup.php
new file mode 100644
index 0000000..fe8836a
--- /dev/null
+++ b/app/Provisioning/Steps/Customer/RegisterBackup.php
@@ -0,0 +1,38 @@
+<?php
+
+namespace App\Provisioning\Steps\Customer;
+
+use App\Models\ProvisioningRun;
+use App\Provisioning\StepResult;
+
+class RegisterBackup extends CustomerStep
+{
+    public function key(): string
+    {
+        return 'register_backup';
+    }
+
+    public function maxDuration(): int
+    {
+        return 120;
+    }
+
+    public function execute(ProvisioningRun $run): StepResult
+    {
+        if ($this->hasResource($run, 'backup_job_id')) {
+            return StepResult::advance();
+        }
+
+        $instance = $this->instance($run);
+        $jobId = 'vzdump-'.$instance->vmid.'-daily';
+
+        $instance->backups()->create([
+            'external_job_id' => $jobId,
+            'schedule' => 'daily 02:00',
+            'status' => 'scheduled',
+        ]);
+        $this->recordResource($run, $instance->host, 'backup_job_id', $jobId);
+
+        return StepResult::advance();
+    }
+}
diff --git a/app/Provisioning/Steps/Customer/RegisterMonitoring.php b/app/Provisioning/Steps/Customer/RegisterMonitoring.php
new file mode 100644
index 0000000..ea267d2
--- /dev/null
+++ b/app/Provisioning/Steps/Customer/RegisterMonitoring.php
@@ -0,0 +1,39 @@
+<?php
+
+namespace App\Provisioning\Steps\Customer;
+
+use App\Models\ProvisioningRun;
+use App\Provisioning\StepResult;
+
+class RegisterMonitoring extends CustomerStep
+{
+    public function key(): string
+    {
+        return 'register_monitoring';
+    }
+
+    public function maxDuration(): int
+    {
+        return 120;
+    }
+
+    public function execute(ProvisioningRun $run): StepResult
+    {
+        if ($this->hasResource($run, 'monitoring_target_id')) {
+            return StepResult::advance();
+        }
+
+        $instance = $this->instance($run);
+        $targetId = 'mon-'.$instance->vmid;
+        $url = 'https://'.$instance->subdomain.'.'.config('provisioning.dns.zone').'/status.php';
+
+        $instance->monitoringTargets()->create([
+            'external_id' => $targetId,
+            'url' => $url,
+            'status' => 'up',
+        ]);
+        $this->recordResource($run, $instance->host, 'monitoring_target_id', $targetId);
+
+        return StepResult::advance();
+    }
+}
diff --git a/app/Provisioning/Steps/Customer/ReserveResources.php b/app/Provisioning/Steps/Customer/ReserveResources.php
new file mode 100644
index 0000000..4415cd5
--- /dev/null
+++ b/app/Provisioning/Steps/Customer/ReserveResources.php
@@ -0,0 +1,82 @@
+<?php
+
+namespace App\Provisioning\Steps\Customer;
+
+use App\Models\Host;
+use App\Models\Instance;
+use App\Models\Order;
+use App\Models\ProvisioningRun;
+use App\Provisioning\StepResult;
+use Illuminate\Support\Str;
+
+class ReserveResources extends CustomerStep
+{
+    public function key(): string
+    {
+        return 'reserve_resources';
+    }
+
+    public function maxDuration(): int
+    {
+        return 60;
+    }
+
+    public function execute(ProvisioningRun $run): StepResult
+    {
+        $order = $this->order($run);
+
+        // Idempotent: an instance already exists for this order.
+        $existing = Instance::query()->where('order_id', $order->id)->first();
+        if ($existing !== null) {
+            $this->putContext($run, $existing);
+
+            return StepResult::advance();
+        }
+
+        $plan = $this->plan($run);
+        $host = Host::placeableIn($order->datacenter, (int) $plan['quota_gb']);
+        if ($host === null) {
+            return StepResult::fail('no_capacity');
+        }
+
+        $instance = Instance::create([
+            'customer_id' => $order->customer_id,
+            'order_id' => $order->id,
+            'host_id' => $host->id,
+            'plan' => $order->plan,
+            'quota_gb' => $plan['quota_gb'],
+            'disk_gb' => $plan['disk_gb'],
+            'ram_mb' => $plan['ram_mb'],
+            'cores' => $plan['cores'],
+            'subdomain' => $this->uniqueSubdomain($order),
+            'status' => 'reserving',
+        ]);
+
+        $this->putContext($run, $instance);
+        $this->recordResource($run, $host, 'instance_id', (string) $instance->id);
+
+        return StepResult::advance();
+    }
+
+    private function putContext(ProvisioningRun $run, Instance $instance): void
+    {
+        $run->mergeContext([
+            'instance_id' => $instance->id,
+            'host_id' => $instance->host_id,
+            'node' => $instance->host?->node ?? 'pve',
+            'subdomain' => $instance->subdomain,
+            'plan' => $instance->plan,
+        ]);
+    }
+
+    private function uniqueSubdomain(Order $order): string
+    {
+        $base = Str::slug($order->customer->name) ?: 'cloud';
+
+        do {
+            $subdomain = $base.'-'.Str::lower(Str::random(5));
+        } while (Instance::query()->where('subdomain', $subdomain)->exists());
+
+        return $subdomain;
+    }
+}
diff --git a/app/Provisioning/Steps/Customer/RunAcceptanceChecks.php b/app/Provisioning/Steps/Customer/RunAcceptanceChecks.php
new file mode 100644
index 0000000..bfda3c7
--- /dev/null
+++ b/app/Provisioning/Steps/Customer/RunAcceptanceChecks.php
@@ -0,0 +1,59 @@
+<?php
+
+namespace App\Provisioning\Steps\Customer;
+
+use App\Models\ProvisioningRun;
+use App\Provisioning\StepResult;
+use App\Services\Proxmox\ProxmoxClient;
+use App\Services\Traefik\TraefikWriter;
+
+/**
+ * Gate: nothing grants the customer access until every check here passes.
+ */
+class RunAcceptanceChecks extends CustomerStep
+{
+    public function __construct(
+        private ProxmoxClient $pve,
+        private TraefikWriter $traefik,
+    ) {}
+
+    public function key(): string
+    {
+        return 'run_acceptance_checks';
+    }
+
+    public function maxDuration(): int
+    {
+        return 180;
+    }
+
+    public function execute(ProvisioningRun $run): StepResult
+    {
+        $instance = $this->instance($run);
+        $node = (string) $run->context('node');
+        $vmid = (int) $run->context('vmid');
+        $fqdn = $instance->subdomain.'.'.config('provisioning.dns.zone');
+
+        if (! $instance->cert_ok || ! $this->traefik->certReachable($fqdn)) {
+            return StepResult::fail('acceptance_failed:cert');
+        }
+
+        $status = $this->pve->forHost($instance->host)
+            ->guestExec($node, $vmid, 'curl -sf http://localhost/status.php');
+        if ((int) ($status['exitcode'] ?? 1) !== 0) {
+            return StepResult::fail('acceptance_failed:nextcloud');
+        }
+
+        if (! $this->hasResource($run, 'nc_admin')) {
+            return StepResult::fail('acceptance_failed:admin');
+        }
+        if (! $this->hasResource($run, 'backup_job_id')) {
+            return StepResult::fail('acceptance_failed:backup');
+        }
+        if (! $this->hasResource($run, 'monitoring_target_id')) {
+            return StepResult::fail('acceptance_failed:monitoring');
+        }
+
+        return StepResult::advance();
+    }
+}
diff --git a/app/Provisioning/Steps/Customer/StartVirtualMachine.php b/app/Provisioning/Steps/Customer/StartVirtualMachine.php
new file mode 100644
index 0000000..750b469
--- /dev/null
+++ b/app/Provisioning/Steps/Customer/StartVirtualMachine.php
@@ -0,0 +1,51 @@
+<?php
+
+namespace App\Provisioning\Steps\Customer;
+
+use App\Models\ProvisioningRun;
+use App\Provisioning\StepResult;
+use App\Services\Proxmox\ProxmoxClient;
+
+class StartVirtualMachine extends CustomerStep
+{
+    public function __construct(private ProxmoxClient $pve) {}
+
+    public function key(): string
+    {
+        return 'start_vm';
+    }
+
+    public function maxDuration(): int
+    {
+        return 180;
+    }
+
+    public function execute(ProvisioningRun $run): StepResult
+    {
+        $instance = $this->instance($run);
+        $node = (string) $run->context('node');
+        $vmid = (int) $run->context('vmid');
+        $pve = $this->pve->forHost($instance->host);
+
+        // Idempotent: already running.
+        if (($pve->vmStatus($node, $vmid)['status'] ?? '') === 'running') {
+            return StepResult::advance();
+        }
+
+        $upid = $run->context('start_upid');
+        if ($upid === null) {
+            $upid = $pve->startVm($node, $vmid);
+            $run->mergeContext(['start_upid' => $upid]);
+        }
+
+        $task = $pve->taskStatus($node, $upid);
+        if (($task['status'] ?? '') === 'running') {
+            return StepResult::poll(5, 'starting VM');
+        }
+        if (($task['exitstatus'] ?? 'OK') !== 'OK') {
+            return StepResult::fail('start_failed: '.($task['exitstatus'] ?? 'unknown'));
+        }
+
+        return StepResult::advance();
+    }
+}
diff --git a/app/Provisioning/Steps/Customer/ValidateOrder.php b/app/Provisioning/Steps/Customer/ValidateOrder.php
new file mode 100644
index 0000000..4fed5db
--- /dev/null
+++ b/app/Provisioning/Steps/Customer/ValidateOrder.php
@@ -0,0 +1,38 @@
+<?php
+
+namespace App\Provisioning\Steps\Customer;
+
+use App\Models\ProvisioningRun;
+use App\Provisioning\StepResult;
+
+class ValidateOrder extends CustomerStep
+{
+    public function key(): string
+    {
+        return 'validate_order';
+    }
+
+    public function maxDuration(): int
+    {
+        return 60;
+    }
+
+    public function execute(ProvisioningRun $run): StepResult
+    {
+        $order = $this->order($run);
+
+        if (! in_array($order->status, ['paid', 'provisioning'], true)) {
+            return StepResult::fail('order_not_paid');
+        }
+        if ($this->plan($run) === []) {
+            return StepResult::fail('unknown_plan');
+        }
+        if (blank($order->datacenter)) {
+            return StepResult::fail('invalid_datacenter');
+        }
+
+        $order->update(['status' => 'provisioning']);
+
+        return StepResult::advance();
+    }
+}
diff --git a/app/Provisioning/Steps/Customer/WaitForGuestAgent.php b/app/Provisioning/Steps/Customer/WaitForGuestAgent.php
new file mode 100644
index 0000000..0cf1278
--- /dev/null
+++ b/app/Provisioning/Steps/Customer/WaitForGuestAgent.php
@@ -0,0 +1,40 @@
+<?php
+
+namespace App\Provisioning\Steps\Customer;
+
+use App\Models\ProvisioningRun;
+use App\Provisioning\StepResult;
+use App\Services\Proxmox\ProxmoxClient;
+
+class WaitForGuestAgent extends CustomerStep
+{
+    public function __construct(private ProxmoxClient $pve) {}
+
+    public function key(): string
+    {
+        return 'wait_for_guest_agent';
+    }
+
+    public function maxDuration(): int
+    {
+        return 300;
+    }
+
+    public function execute(ProvisioningRun $run): StepResult
+    {
+        $instance = $this->instance($run);
+        $node = (string) $run->context('node');
+        $vmid = (int) $run->context('vmid');
+
+        if ($this->pve->forHost($instance->host)->guestAgentPing($node, $vmid)) {
+            return StepResult::advance();
+        }
+
+        // Own deadline (poll doesn't consume the retry budget).
+        if ($run->started_at !== null && $run->started_at->copy()->addSeconds(270)->isPast()) {
+            return StepResult::fail('guest_agent_timeout');
+        }
+
+        return StepResult::poll(10, 'waiting for guest agent');
+    }
+}
diff --git a/app/Provisioning/Steps/Host/RegisterCapacity.php b/app/Provisioning/Steps/Host/RegisterCapacity.php
index b658e93..08e0bfb 100644
--- a/app/Provisioning/Steps/Host/RegisterCapacity.php
+++ b/app/Provisioning/Steps/Host/RegisterCapacity.php
@@ -51,6 +51,7 @@ class RegisterCapacity extends HostStep
             'total_ram_mb' => $ramMb,
             'total_gb' => $totalGb,
             'pve_version' => $status['pveversion'] ?? null,
+            'node' => $node,
             'last_seen_at' => now(),
         ]);
 
diff --git a/app/Services/Dns/FakeHetznerDnsClient.php b/app/Services/Dns/FakeHetznerDnsClient.php
new file mode 100644
index 0000000..f224f60
--- /dev/null
+++ b/app/Services/Dns/FakeHetznerDnsClient.php
@@ -0,0 +1,29 @@
+<?php
+
+namespace App\Services\Dns;
+
+use RuntimeException;
+
+class FakeHetznerDnsClient implements HetznerDnsClient
+{
+    /** @var array<string, string> fqdn => record_id */
+    public array $records = [];
+
+    public bool $failUpsert = false;
+
+    private int $counter = 1;
+
+    public function upsertRecord(string $fqdn, string $type, string $value): string
+    {
+        if ($this->failUpsert) {
+            throw new RuntimeException('hetzner dns 5xx');
+        }
+
+        return $this->records[$fqdn] ??= 'rec-'.$this->counter++;
+    }
+
+    public function deleteRecord(string $recordId): void
+    {
+        $this->records = array_filter($this->records, fn ($id) => $id !== $recordId);
+    }
+}
diff --git a/app/Services/Dns/HetznerDnsClient.php b/app/Services/Dns/HetznerDnsClient.php
new file mode 100644
index 0000000..95a5772
--- /dev/null
+++ b/app/Services/Dns/HetznerDnsClient.php
@@ -0,0 +1,11 @@
+<?php
+
+namespace App\Services\Dns;
+
+interface HetznerDnsClient
+{
+    /** Create or update a record; returns the provider record id. */
+    public function upsertRecord(string $fqdn, string $type, string $value): string;
+
+    public function deleteRecord(string $recordId): void;
+}
diff --git a/app/Services/Dns/HttpHetznerDnsClient.php b/app/Services/Dns/HttpHetznerDnsClient.php
new file mode 100644
index 0000000..21506f9
--- /dev/null
+++ b/app/Services/Dns/HttpHetznerDnsClient.php
@@ -0,0 +1,55 @@
+<?php
+
+namespace App\Services\Dns;
+
+use Illuminate\Http\Client\PendingRequest;
+use Illuminate\Support\Facades\Http;
+use RuntimeException;
+
+/**
+ * Hetzner DNS API client. Zone + token from config/provisioning.php.
+ * Not unit-tested (live I/O).
+ */
+class HttpHetznerDnsClient implements HetznerDnsClient
+{
+    private function http(): PendingRequest
+    {
+        return Http::withHeaders(['Auth-API-Token' => (string) config('provisioning.dns.token')])
+            ->baseUrl('https://dns.hetzner.com/api/v1')
+            ->timeout(15);
+    }
+
+    private function zoneId(): string
+    {
+        $zone = (string) config('provisioning.dns.zone');
+        $zones = $this->http()->get('/zones', ['name' => $zone])->throw()->json('zones', []);
+
+        return $zones[0]['id'] ?? throw new RuntimeException("DNS zone {$zone} not found");
+    }
+
+    public function upsertRecord(string $fqdn, string $type, string $value): string
+    {
+        $zoneId = $this->zoneId();
+        $name = rtrim(str_replace('.'.config('provisioning.dns.zone'), '', $fqdn), '.');
+
+        $existing = collect($this->http()->get('/records', ['zone_id' => $zoneId])->throw()->json('records', []))
+            ->first(fn ($r) => $r['name'] === $name && $r['type'] === $type);
+
+        if ($existing) {
+            $this->http()->put("/records/{$existing['id']}", [
+                'zone_id' => $zoneId, 'type' => $type, 'name' => $name, 'value' => $value, 'ttl' => 300,
+            ])->throw();
+
+            return (string) $existing['id'];
+        }
+
+        return (string) $this->http()->post('/records', [
+            'zone_id' => $zoneId, 'type' => $type, 'name' => $name, 'value' => $value, 'ttl' => 300,
+        ])->throw()->json('record.id');
+    }
+
+    public function deleteRecord(string $recordId): void
+    {
+        $this->http()->delete("/records/{$recordId}")->throw();
+    }
+}
diff --git a/app/Services/Traefik/FakeTraefikWriter.php b/app/Services/Traefik/FakeTraefikWriter.php
new file mode 100644
index 0000000..5417872
--- /dev/null
+++ b/app/Services/Traefik/FakeTraefikWriter.php
@@ -0,0 +1,26 @@
+<?php
+
+namespace App\Services\Traefik;
+
+class FakeTraefikWriter implements TraefikWriter
+{
+    /** @var array<string, string> subdomain => target */
+    public array $routes = [];
+
+    public bool $certReady = true;
+
+    public function write(string $subdomain, string $targetHost): void
+    {
+        $this->routes[$subdomain] = $targetHost;
+    }
+
+    public function remove(string $subdomain): void
+    {
+        unset($this->routes[$subdomain]);
+    }
+
+    public function certReachable(string $fqdn): bool
+    {
+        return $this->certReady;
+    }
+}
diff --git a/app/Services/Traefik/SshTraefikWriter.php b/app/Services/Traefik/SshTraefikWriter.php
new file mode 100644
index 0000000..6facbd5
--- /dev/null
+++ b/app/Services/Traefik/SshTraefikWriter.php
@@ -0,0 +1,57 @@
+<?php
+
+namespace App\Services\Traefik;
+
+use Illuminate\Support\Facades\Http;
+
+/**
+ * Writes Traefik file-provider YAML into the configured dynamic directory (where
+ * Traefik runs) and checks HTTP-01 cert issuance by probing the fqdn over HTTPS.
+ * The concrete write path (host SSH / mounted volume) is a deployment watch-item.
+ * Not unit-tested (live I/O).
+ */
+class SshTraefikWriter implements TraefikWriter
+{
+    public function write(string $subdomain, string $targetHost): void
+    {
+        $zone = (string) config('provisioning.dns.zone');
+        $fqdn = "{$subdomain}.{$zone}";
+        $yaml = $this->render($subdomain, $fqdn, $targetHost);
+
+        $path = rtrim((string) config('provisioning.traefik.dynamic_path'), '/')."/{$subdomain}.yml";
+        file_put_contents($path, $yaml);
+    }
+
+    public function remove(string $subdomain): void
+    {
+        $path = rtrim((string) config('provisioning.traefik.dynamic_path'), '/')."/{$subdomain}.yml";
+        if (is_file($path)) {
+            unlink($path);
+        }
+    }
+
+    public function certReachable(string $fqdn): bool
+    {
+        return Http::timeout(5)->connectTimeout(5)->get("https://{$fqdn}/status.php")->successful();
+    }
+
+    private function render(string $subdomain, string $fqdn, string $targetHost): string
+    {
+        return implode("\n", [
+            'http:',
+            '  routers:',
+            "    {$subdomain}:",
+            "      rule: \"Host(`{$fqdn}`)\"",
+            "      service: \"{$subdomain}\"",
+            '      entryPoints: ["websecure"]',
+            '      tls:',
+            '        certResolver: "letsencrypt"',
+            '  services:',
+            "    {$subdomain}:",
+            '      loadBalancer:',
+            '        servers:',
+            "          - url: \"http://{$targetHost}:80\"",
+            '',
+        ]);
+    }
+}
diff --git a/app/Services/Traefik/TraefikWriter.php b/app/Services/Traefik/TraefikWriter.php
new file mode 100644
index 0000000..cda6622
--- /dev/null
+++ b/app/Services/Traefik/TraefikWriter.php
@@ -0,0 +1,14 @@
+<?php
+
+namespace App\Services\Traefik;
+
+interface TraefikWriter
+{
+    /** Write a file-provider router for subdomain → target host. */
+    public function write(string $subdomain, string $targetHost): void;
+
+    public function remove(string $subdomain): void;
+
+    /** True once the ACME (HTTP-01) certificate for the fqdn is served. */
+    public function certReachable(string $fqdn): bool;
+}
diff --git a/database/migrations/2026_07_25_080008_add_node_to_hosts_table.php b/database/migrations/2026_07_25_080008_add_node_to_hosts_table.php
new file mode 100644
index 0000000..91ca3ef
--- /dev/null
+++ b/database/migrations/2026_07_25_080008_add_node_to_hosts_table.php
@@ -0,0 +1,24 @@
+<?php
+
+use Illuminate\Database\Migrations\Migration;
+use Illuminate\Database\Schema\Blueprint;
+use Illuminate\Support\Facades\Schema;
+
+return new class extends Migration
+{
+    public function up(): void
+    {
+        // Proxmox node name (from RegisterCapacity), used by Subsystem B to
+        // address qemu/{node}/... endpoints. Nullable until a host is active.
+        Schema::table('hosts', function (Blueprint $table) {
+            $table->string('node')->nullable()->after('pve_version');
+        });
+    }
+
+    public function down(): void
+    {
+        Schema::table('hosts', function (Blueprint $table) {
+            $table->dropColumn('node');
+        });
+    }
+};
diff --git a/tests/Feature/Provisioning/CustomerProvisioningEndToEndTest.php b/tests/Feature/Provisioning/CustomerProvisioningEndToEndTest.php
new file mode 100644
index 0000000..e58137b
--- /dev/null
+++ b/tests/Feature/Provisioning/CustomerProvisioningEndToEndTest.php
@@ -0,0 +1,90 @@
+<?php
+
+use App\Models\Host;
+use App\Models\Instance;
+use App\Models\Order;
+use App\Models\ProvisioningRun;
+use App\Models\RunResource;
+use App\Notifications\CloudReady;
+use App\Provisioning\RunRunner;
+use Illuminate\Support\Facades\Notification;
+use Illuminate\Support\Facades\Queue;
+
+it('provisions a paid order all the way to active (mocked)', function () {
+    Notification::fake();
+    Queue::fake(); // drive the runner manually, one step per advance
+    $s = fakeServices();
+    $s['pve']->guestScript('status.php', 0, 'ok'); // deploy health + acceptance probe
+    // Fake defaults: tasks stopped/OK, guest agent up, cert reachable.
+
+    Host::factory()->active()->create(['datacenter' => 'fsn', 'node' => 'pve', 'total_gb' => 1000]);
+    $order = Order::factory()->create(['status' => 'paid', 'plan' => 'start', 'datacenter' => 'fsn']);
+    $run = ProvisioningRun::factory()->create([
+        'subject_type' => Order::class, 'subject_id' => $order->id, 'pipeline' => 'customer', 'context' => [],
+    ]);
+
+    $runner = app(RunRunner::class);
+    for ($i = 0; $i < 60; $i++) {
+        $run->refresh();
+        if (in_array($run->status, ['completed', 'failed'], true)) {
+            break;
+        }
+        if ($run->next_attempt_at?->isFuture()) {
+            $run->update(['next_attempt_at' => now()]);
+        }
+        $runner->advance($run);
+    }
+
+    $run->refresh();
+    $instance = Instance::query()->where('order_id', $order->id)->firstOrFail();
+
+    expect($run->status)->toBe('completed')
+        ->and($run->error)->toBeNull()
+        ->and($order->fresh()->status)->toBe('active')
+        ->and($instance->status)->toBe('active')
+        ->and($instance->vmid)->not->toBeNull()
+        ->and($instance->cert_ok)->toBeTrue()
+        ->and($instance->onboardingTasks()->count())->toBeGreaterThan(0);
+
+    // Every external resource created exactly once.
+    foreach (['instance_id', 'vmid', 'dns_record_id', 'nc_admin', 'backup_job_id', 'monitoring_target_id'] as $kind) {
+        expect(RunResource::where('run_id', $run->id)->where('kind', $kind)->count())->toBe(1);
+    }
+
+    // Secrets scrubbed from state; credentials delivered.
+    expect($run->context('admin_password'))->toBeNull()
+        ->and($run->context('db_password'))->toBeNull();
+    Notification::assertSentOnDemand(CloudReady::class);
+});
+
+it('does not duplicate external resources when a step re-runs after a crash', function () {
+    Notification::fake();
+    Queue::fake();
+    $s = fakeServices();
+    $s['pve']->guestScript('status.php', 0, 'ok');
+
+    Host::factory()->active()->create(['datacenter' => 'fsn', 'node' => 'pve', 'total_gb' => 1000]);
+    $order = Order::factory()->create(['status' => 'paid', 'plan' => 'start', 'datacenter' => 'fsn']);
+    $run = ProvisioningRun::factory()->create([
+        'subject_type' => Order::class, 'subject_id' => $order->id, 'pipeline' => 'customer', 'context' => [],
+    ]);
+    $runner = app(RunRunner::class);
+
+    // Drive a few steps, then rewind to the clone step and replay it.
+    for ($i = 0; $i < 5; $i++) {
+        $run->refresh();
+        if ($run->next_attempt_at?->isFuture()) {
+            $run->update(['next_attempt_at' => now()]);
+        }
+        $runner->advance($run);
+    }
+    $cloneIndex = array_search(
+        \App\Provisioning\Steps\Customer\CloneVirtualMachine::class,
+        config('provisioning.pipelines.customer'),
+        true,
+    );
+    $run->update(['current_step' => $cloneIndex, 'status' => 'running']);
+    $runner->advance($run->fresh());
+
+    expect(RunResource::where('run_id', $run->id)->where('kind', 'vmid')->count())->toBe(1);
+});
diff --git a/tests/Feature/Provisioning/CustomerStepsTest.php b/tests/Feature/Provisioning/CustomerStepsTest.php
new file mode 100644
index 0000000..cec3cbf
--- /dev/null
+++ b/tests/Feature/Provisioning/CustomerStepsTest.php
@@ -0,0 +1,302 @@
+<?php
+
+use App\Models\Host;
+use App\Models\Instance;
+use App\Models\Order;
+use App\Models\ProvisioningRun;
+use App\Models\RunResource;
+use App\Notifications\CloudReady;
+use App\Provisioning\Steps\Customer\CloneVirtualMachine;
+use App\Provisioning\Steps\Customer\CompleteProvisioning;
+use App\Provisioning\Steps\Customer\ConfigureCloudInit;
+use App\Provisioning\Steps\Customer\ConfigureDnsAndTls;
+use App\Provisioning\Steps\Customer\ConfigureNetwork;
+use App\Provisioning\Steps\Customer\ConfigureNextcloud;
+use App\Provisioning\Steps\Customer\CreateCustomerAdmin;
+use App\Provisioning\Steps\Customer\DeployApplicationStack;
+use App\Provisioning\Steps\Customer\RegisterBackup;
+use App\Provisioning\Steps\Customer\RegisterMonitoring;
+use App\Provisioning\Steps\Customer\ReserveResources;
+use App\Provisioning\Steps\Customer\RunAcceptanceChecks;
+use App\Provisioning\Steps\Customer\StartVirtualMachine;
+use App\Provisioning\Steps\Customer\ValidateOrder;
+use App\Provisioning\Steps\Customer\WaitForGuestAgent;
+use Illuminate\Support\Facades\Crypt;
+use Illuminate\Support\Facades\Notification;
+
+/** A run whose order is paid but nothing is reserved yet. */
+function orderRun(array $attrs = [], array $context = []): ProvisioningRun
+{
+    $order = Order::factory()->create($attrs);
+
+    return ProvisioningRun::factory()->create([
+        'subject_type' => Order::class,
+        'subject_id' => $order->id,
+        'pipeline' => 'customer',
+        'context' => $context,
+    ]);
+}
+
+/** A run with a host + reserved instance + full VM context. */
+function reservedRun(array $extraContext = [], array $instanceAttrs = []): array
+{
+    $host = Host::factory()->active()->create(['datacenter' => 'fsn', 'node' => 'pve']);
+    $order = Order::factory()->create(['datacenter' => 'fsn', 'plan' => 'start']);
+    $instance = Instance::factory()->create(array_merge([
+        'order_id' => $order->id, 'customer_id' => $order->customer_id,
+        'host_id' => $host->id, 'vmid' => 101, 'status' => 'provisioning',
+    ], $instanceAttrs));
+    $run = ProvisioningRun::factory()->create([
+        'subject_type' => Order::class, 'subject_id' => $order->id, 'pipeline' => 'customer',
+        'context' => array_merge([
+            'instance_id' => $instance->id, 'host_id' => $host->id, 'node' => 'pve',
+            'vmid' => 101, 'subdomain' => $instance->subdomain, 'plan' => 'start',
+        ], $extraContext),
+    ]);
+
+    return compact('host', 'order', 'instance', 'run');
+}
+
+// 1. ValidateOrder
+it('validates a paid order and moves it to provisioning', function () {
+    $run = orderRun(['status' => 'paid', 'plan' => 'start']);
+    expect(app(ValidateOrder::class)->execute($run)->type)->toBe('advance')
+        ->and($run->subject->fresh()->status)->toBe('provisioning');
+});
+
+it('fails validation for an unpaid order or unknown plan', function () {
+    expect(app(ValidateOrder::class)->execute(orderRun(['status' => 'pending']))->type)->toBe('fail');
+    expect(app(ValidateOrder::class)->execute(orderRun(['status' => 'paid', 'plan' => 'ghost']))->type)->toBe('fail');
+});
+
+// 2. ReserveResources
+it('reserves an instance on a datacenter host and records it once', function () {
+    fakeServices();
+    Host::factory()->active()->create(['datacenter' => 'fsn', 'node' => 'pve', 'total_gb' => 1000]);
+    $run = orderRun(['status' => 'provisioning', 'plan' => 'start', 'datacenter' => 'fsn']);
+
+    expect(app(ReserveResources::class)->execute($run)->type)->toBe('advance');
+    $run->refresh();
+    $instance = Instance::query()->where('order_id', $run->subject_id)->first();
+    expect($instance)->not->toBeNull()
+        ->and($run->context('instance_id'))->toBe($instance->id)
+        ->and($run->context('node'))->toBe('pve');
+
+    // Re-run: idempotent (no second instance, one breadcrumb).
+    app(ReserveResources::class)->execute($run->fresh());
+    expect(Instance::query()->where('order_id', $run->subject_id)->count())->toBe(1)
+        ->and(RunResource::where('run_id', $run->id)->where('kind', 'instance_id')->count())->toBe(1);
+});
+
+it('fails reservation when no host has capacity', function () {
+    $run = orderRun(['status' => 'provisioning', 'plan' => 'start', 'datacenter' => 'nowhere']);
+    expect(app(ReserveResources::class)->execute($run)->type)->toBe('fail');
+});
+
+// 3. CloneVirtualMachine
+it('clones the template, persisting vmid before polling, once', function () {
+    $s = fakeServices();
+    ['run' => $run, 'instance' => $instance] = reservedRun([], ['vmid' => null]);
+
+    expect(app(CloneVirtualMachine::class)->execute($run)->type)->toBe('advance');
+    $run->refresh();
+    expect($run->context('vmid'))->not->toBeNull()
+        ->and($instance->fresh()->vmid)->toBe($run->context('vmid'))
+        ->and(RunResource::where('run_id', $run->id)->where('kind', 'vmid')->count())->toBe(1);
+
+    // Re-run: idempotent, no second clone.
+    app(CloneVirtualMachine::class)->execute($run->fresh());
+    expect(count($s['pve']->clonedVmids))->toBe(1);
+});
+
+it('polls while cloning and fails on a clone error', function () {
+    $s = fakeServices();
+    $s['pve']->forceTaskStatus = 'running';
+    ['run' => $run] = reservedRun([], ['vmid' => null]);
+    expect(app(CloneVirtualMachine::class)->execute($run)->type)->toBe('poll');
+
+    $s2 = fakeServices();
+    $s2['pve']->taskExitStatus = 'clone failed';
+    ['run' => $run2] = reservedRun([], ['vmid' => null]);
+    expect(app(CloneVirtualMachine::class)->execute($run2)->type)->toBe('fail');
+});
+
+it('fails clone when the plan has no template', function () {
+    fakeServices();
+    $host = Host::factory()->active()->create(['node' => 'pve']);
+    $order = Order::factory()->create(['plan' => 'start']);
+    $instance = Instance::factory()->create(['order_id' => $order->id, 'customer_id' => $order->customer_id, 'host_id' => $host->id]);
+    $run = ProvisioningRun::factory()->create([
+        'subject_type' => Order::class, 'subject_id' => $order->id, 'pipeline' => 'customer',
+        'context' => ['instance_id' => $instance->id, 'node' => 'pve'],
+    ]);
+    config()->set('provisioning.plans.start.template_vmid', 0);
+
+    expect(app(CloneVirtualMachine::class)->execute($run)->type)->toBe('fail');
+});
+
+// 4. ConfigureCloudInit
+it('configures cloud-init and resizes the disk', function () {
+    $s = fakeServices();
+    ['run' => $run] = reservedRun();
+    expect(app(ConfigureCloudInit::class)->execute($run)->type)->toBe('advance')
+        ->and($s['pve']->cloudInitCalls)->toContain(101)
+        ->and($s['pve']->resizeCalls)->not->toBeEmpty();
+});
+
+// 5. StartVirtualMachine
+it('starts the VM and is idempotent when already running', function () {
+    $s = fakeServices();
+    ['run' => $run] = reservedRun();
+    expect(app(StartVirtualMachine::class)->execute($run)->type)->toBe('advance')
+        ->and($s['pve']->runningVmids)->toContain(101);
+
+    // already running → advance without another start
+    expect(app(StartVirtualMachine::class)->execute($run->fresh())->type)->toBe('advance')
+        ->and(count($s['pve']->runningVmids))->toBe(1);
+});
+
+// 6. WaitForGuestAgent
+it('waits for the guest agent then advances', function () {
+    $s = fakeServices();
+    $s['pve']->guestAgentUp = false;
+    ['run' => $run] = reservedRun();
+    expect(app(WaitForGuestAgent::class)->execute($run)->type)->toBe('poll');
+
+    $s['pve']->guestAgentUp = true;
+    expect(app(WaitForGuestAgent::class)->execute($run->fresh())->type)->toBe('advance');
+});
+
+it('fails when the guest agent never answers before the deadline', function () {
+    $s = fakeServices();
+    $s['pve']->guestAgentUp = false;
+    ['run' => $run] = reservedRun();
+    $run->update(['started_at' => now()->subMinutes(10)]);
+    expect(app(WaitForGuestAgent::class)->execute($run->fresh())->type)->toBe('fail');
+});
+
+// 7. ConfigureNetwork
+it('configures the guest network and firewall', function () {
+    $s = fakeServices();
+    ['run' => $run] = reservedRun();
+    expect(app(ConfigureNetwork::class)->execute($run)->type)->toBe('advance')
+        ->and($s['pve']->guestRan('set-hostname'))->toBeTrue()
+        ->and($s['pve']->firewallCalls)->toContain('101');
+});
+
+// 8. DeployApplicationStack
+it('deploys the stack, polling until healthy, and scrubs the db password', function () {
+    $s = fakeServices();
+    ['run' => $run] = reservedRun();
+
+    // not healthy yet → poll, secret still present
+    expect(app(DeployApplicationStack::class)->execute($run)->type)->toBe('poll')
+        ->and($run->fresh()->context('db_password'))->not->toBeNull();
+
+    // healthy → advance, secret scrubbed
+    $s['pve']->guestScript('status.php', 0, 'ok');
+    expect(app(DeployApplicationStack::class)->execute($run->fresh())->type)->toBe('advance')
+        ->and($run->fresh()->context('db_password'))->toBeNull();
+});
+
+// 9. ConfigureNextcloud
+it('configures nextcloud via occ', function () {
+    $s = fakeServices();
+    ['run' => $run] = reservedRun();
+    expect(app(ConfigureNextcloud::class)->execute($run)->type)->toBe('advance')
+        ->and($s['pve']->guestRan('occ config:system:set trusted_domains'))->toBeTrue();
+});
+
+// 10. CreateCustomerAdmin
+it('creates the admin once, never persisting the password in plaintext', function () {
+    $s = fakeServices();
+    ['run' => $run, 'instance' => $instance] = reservedRun();
+
+    expect(app(CreateCustomerAdmin::class)->execute($run)->type)->toBe('advance');
+    $run->refresh();
+    expect($instance->fresh()->nc_admin_ref)->toBe('admin')
+        ->and($run->context('admin_password'))->not->toBeNull()          // present but encrypted
+        ->and(Crypt::decryptString($run->context('admin_password')))->toBeString()
+        ->and(RunResource::where('run_id', $run->id)->where('kind', 'nc_admin')->count())->toBe(1);
+
+    // raw context in DB must not contain the decrypted password
+    $rawContext = \DB::table('provisioning_runs')->where('id', $run->id)->value('context');
+    expect($rawContext)->not->toContain(Crypt::decryptString($run->context('admin_password')));
+
+    // idempotent
+    $guestCountBefore = count($s['pve']->guestCommands);
+    app(CreateCustomerAdmin::class)->execute($run->fresh());
+    expect(count($s['pve']->guestCommands))->toBe($guestCountBefore);
+});
+
+// 11. ConfigureDnsAndTls
+it('creates the dns record, writes the route, and polls for the cert', function () {
+    $s = fakeServices();
+    $s['traefik']->certReady = false;
+    ['run' => $run, 'instance' => $instance] = reservedRun();
+
+    expect(app(ConfigureDnsAndTls::class)->execute($run)->type)->toBe('poll');
+    $run->refresh();
+    $instance->refresh();
+    expect(RunResource::where('run_id', $run->id)->where('kind', 'dns_record_id')->count())->toBe(1)
+        ->and($instance->dnsRecords()->count())->toBe(1)
+        ->and($instance->route_written)->toBeTrue()
+        ->and($s['traefik']->routes)->toHaveKey($instance->subdomain);
+
+    // cert now reachable → advance, no duplicate dns record
+    $s['traefik']->certReady = true;
+    expect(app(ConfigureDnsAndTls::class)->execute($run->fresh())->type)->toBe('advance')
+        ->and($instance->fresh()->cert_ok)->toBeTrue()
+        ->and(RunResource::where('run_id', $run->id)->where('kind', 'dns_record_id')->count())->toBe(1);
+});
+
+// 12/13. Backup + Monitoring
+it('registers a backup job and a monitoring target once each', function () {
+    fakeServices();
+    ['run' => $run, 'instance' => $instance] = reservedRun();
+
+    expect(app(RegisterBackup::class)->execute($run)->type)->toBe('advance');
+    app(RegisterBackup::class)->execute($run->fresh());
+    expect($instance->fresh()->backups()->count())->toBe(1)
+        ->and(RunResource::where('run_id', $run->id)->where('kind', 'backup_job_id')->count())->toBe(1);
+
+    expect(app(RegisterMonitoring::class)->execute($run->fresh())->type)->toBe('advance');
+    app(RegisterMonitoring::class)->execute($run->fresh());
+    expect($instance->fresh()->monitoringTargets()->count())->toBe(1)
+        ->and(RunResource::where('run_id', $run->id)->where('kind', 'monitoring_target_id')->count())->toBe(1);
+});
+
+// 14. RunAcceptanceChecks
+it('passes acceptance only when everything is green', function () {
+    fakeServices();
+    ['run' => $run, 'instance' => $instance, 'host' => $host] = reservedRun([], ['cert_ok' => true]);
+    foreach (['nc_admin', 'backup_job_id', 'monitoring_target_id'] as $kind) {
+        RunResource::create(['run_id' => $run->id, 'host_id' => $host->id, 'kind' => $kind, 'external_id' => 'x']);
+    }
+
+    expect(app(RunAcceptanceChecks::class)->execute($run)->type)->toBe('advance');
+
+    // missing monitoring → fail
+    RunResource::where('run_id', $run->id)->where('kind', 'monitoring_target_id')->delete();
+    expect(app(RunAcceptanceChecks::class)->execute($run->fresh())->type)->toBe('fail');
+
+    // failing cert → fail
+    ['run' => $run2, 'host' => $host2] = reservedRun([], ['cert_ok' => false]);
+    expect(app(RunAcceptanceChecks::class)->execute($run2)->type)->toBe('fail');
+});
+
+// 15. CompleteProvisioning
+it('completes: activates instance+order, seeds onboarding, delivers credentials', function () {
+    Notification::fake();
+    fakeServices();
+    ['run' => $run, 'instance' => $instance, 'order' => $order] = reservedRun([
+        'admin_password' => Crypt::encryptString('s3cret-pw'),
+    ]);
+
+    expect(app(CompleteProvisioning::class)->execute($run)->type)->toBe('advance');
+    expect($instance->fresh()->status)->toBe('active')
+        ->and($order->fresh()->status)->toBe('active')
+        ->and($instance->fresh()->onboardingTasks()->count())->toBeGreaterThan(0)
+        ->and($run->fresh()->context('admin_password'))->toBeNull();
+    Notification::assertSentOnDemand(CloudReady::class);
+});
diff --git a/tests/Pest.php b/tests/Pest.php
index c56f3a2..7d8c088 100644
--- a/tests/Pest.php
+++ b/tests/Pest.php
@@ -51,19 +51,23 @@ function something()
 
 /**
  * Bind fake provisioning services into the container and return them so a test
- * can script/inspect SSH, WireGuard and Proxmox interactions.
+ * can script/inspect SSH, WireGuard, Proxmox, DNS and Traefik interactions.
  *
- * @return array{shell: \App\Services\Ssh\FakeRemoteShell, hub: \App\Services\Wireguard\FakeWireguardHub, pve: \App\Services\Proxmox\FakeProxmoxClient}
+ * @return array{shell: \App\Services\Ssh\FakeRemoteShell, hub: \App\Services\Wireguard\FakeWireguardHub, pve: \App\Services\Proxmox\FakeProxmoxClient, dns: \App\Services\Dns\FakeHetznerDnsClient, traefik: \App\Services\Traefik\FakeTraefikWriter}
  */
 function fakeServices(): array
 {
     $shell = new \App\Services\Ssh\FakeRemoteShell;
     $hub = new \App\Services\Wireguard\FakeWireguardHub;
     $pve = new \App\Services\Proxmox\FakeProxmoxClient;
+    $dns = new \App\Services\Dns\FakeHetznerDnsClient;
+    $traefik = new \App\Services\Traefik\FakeTraefikWriter;
 
     app()->instance(\App\Services\Ssh\RemoteShell::class, $shell);
     app()->instance(\App\Services\Wireguard\WireguardHub::class, $hub);
     app()->instance(\App\Services\Proxmox\ProxmoxClient::class, $pve);
+    app()->instance(\App\Services\Dns\HetznerDnsClient::class, $dns);
+    app()->instance(\App\Services\Traefik\TraefikWriter::class, $traefik);
 
-    return ['shell' => $shell, 'hub' => $hub, 'pve' => $pve];
+    return ['shell' => $shell, 'hub' => $hub, 'pve' => $pve, 'dns' => $dns, 'traefik' => $traefik];
 }