From 9162446b1fc41bc2ac84eca27f024c9ae0099fac Mon Sep 17 00:00:00 2001 From: nexxo Date: Sat, 25 Jul 2026 13:00:49 +0200 Subject: [PATCH] fix(engine-b): acceptance gate verifies real health, not just breadcrumbs - Nextcloud: parse occ status JSON (installed + not in maintenance). - Admin: query occ user:info for the account. - Monitoring: MonitoringClient::isHealthy checks the provider, not the local row. Co-Authored-By: Claude Opus 4.8 --- .../Steps/Customer/RunAcceptanceChecks.php | 29 +++++++++++++++---- .../Monitoring/FakeMonitoringClient.php | 7 +++++ .../Monitoring/HttpMonitoringClient.php | 17 +++++++++++ app/Services/Monitoring/MonitoringClient.php | 3 ++ .../CustomerProvisioningEndToEndTest.php | 4 ++- .../Provisioning/CustomerStepsTest.php | 26 ++++++++++------- 6 files changed, 69 insertions(+), 17 deletions(-) diff --git a/app/Provisioning/Steps/Customer/RunAcceptanceChecks.php b/app/Provisioning/Steps/Customer/RunAcceptanceChecks.php index bfda3c7..58a3e17 100644 --- a/app/Provisioning/Steps/Customer/RunAcceptanceChecks.php +++ b/app/Provisioning/Steps/Customer/RunAcceptanceChecks.php @@ -4,17 +4,21 @@ namespace App\Provisioning\Steps\Customer; use App\Models\ProvisioningRun; use App\Provisioning\StepResult; +use App\Services\Monitoring\MonitoringClient; use App\Services\Proxmox\ProxmoxClient; use App\Services\Traefik\TraefikWriter; /** - * Gate: nothing grants the customer access until every check here passes. + * Gate: nothing grants the customer access until every check here actually + * passes — the cert is served, Nextcloud reports healthy, the admin account is + * usable, a backup exists, and monitoring is green. */ class RunAcceptanceChecks extends CustomerStep { public function __construct( private ProxmoxClient $pve, private TraefikWriter $traefik, + private MonitoringClient $monitoring, ) {} public function key(): string @@ -33,24 +37,37 @@ class RunAcceptanceChecks extends CustomerStep $node = (string) $run->context('node'); $vmid = (int) $run->context('vmid'); $fqdn = $instance->subdomain.'.'.config('provisioning.dns.zone'); + $pve = $this->pve->forHost($instance->host); + $occ = 'cd /opt/nextcloud && docker compose exec -T app php occ '; + // TLS + routing actually serving. if (! $instance->cert_ok || ! $this->traefik->certReachable($fqdn)) { return StepResult::fail('acceptance_failed:cert'); } - $status = $this->pve->forHost($instance->host) - ->guestExec($node, $vmid, 'curl -sf http://localhost/status.php'); - if ((int) ($status['exitcode'] ?? 1) !== 0) { + // Nextcloud installed, not in maintenance, DB reachable. + $status = $pve->guestExec($node, $vmid, $occ.'status --output=json'); + $health = json_decode($status['out-data'] ?? '', true) ?: []; + if ((int) ($status['exitcode'] ?? 1) !== 0 + || ($health['installed'] ?? false) !== true + || ($health['maintenance'] ?? false) !== false) { return StepResult::fail('acceptance_failed:nextcloud'); } - if (! $this->hasResource($run, 'nc_admin')) { + // The admin account exists and is queryable. + $admin = $pve->guestExec($node, $vmid, $occ.'user:info '.escapeshellarg((string) $instance->nc_admin_ref)); + if ((int) ($admin['exitcode'] ?? 1) !== 0) { return StepResult::fail('acceptance_failed:admin'); } + + // A backup job is registered. if (! $this->hasResource($run, 'backup_job_id')) { return StepResult::fail('acceptance_failed:backup'); } - if (! $this->hasResource($run, 'monitoring_target_id')) { + + // Monitoring target exists and reports healthy. + $target = $instance->monitoringTargets()->first(); + if ($target === null || ! $this->monitoring->isHealthy($target->external_id)) { return StepResult::fail('acceptance_failed:monitoring'); } diff --git a/app/Services/Monitoring/FakeMonitoringClient.php b/app/Services/Monitoring/FakeMonitoringClient.php index f5ac92c..f86651e 100644 --- a/app/Services/Monitoring/FakeMonitoringClient.php +++ b/app/Services/Monitoring/FakeMonitoringClient.php @@ -21,8 +21,15 @@ class FakeMonitoringClient implements MonitoringClient return $id; } + public bool $healthy = true; + public function deregisterTarget(string $externalId): void { unset($this->targets[$externalId]); } + + public function isHealthy(string $externalId): bool + { + return $this->healthy; + } } diff --git a/app/Services/Monitoring/HttpMonitoringClient.php b/app/Services/Monitoring/HttpMonitoringClient.php index 503cef1..37ce53a 100644 --- a/app/Services/Monitoring/HttpMonitoringClient.php +++ b/app/Services/Monitoring/HttpMonitoringClient.php @@ -34,6 +34,23 @@ class HttpMonitoringClient implements MonitoringClient ->throw()->json('monitor.id'); } + public function isHealthy(string $externalId): bool + { + $endpoint = (string) config('services.monitoring.url'); + if (blank($endpoint)) { + return true; // no monitoring service configured + } + + try { + $monitor = Http::withToken((string) config('services.monitoring.token')) + ->get(rtrim($endpoint, '/').'/monitors/'.$externalId)->throw()->json('monitor', []); + + return ($monitor['status'] ?? '') === 'up'; + } catch (\Throwable) { + return false; + } + } + public function deregisterTarget(string $externalId): void { $endpoint = (string) config('services.monitoring.url'); diff --git a/app/Services/Monitoring/MonitoringClient.php b/app/Services/Monitoring/MonitoringClient.php index b50ca9c..b48520c 100644 --- a/app/Services/Monitoring/MonitoringClient.php +++ b/app/Services/Monitoring/MonitoringClient.php @@ -8,4 +8,7 @@ interface MonitoringClient public function registerTarget(string $name, string $url): string; public function deregisterTarget(string $externalId): void; + + /** Current health of a target as reported by the monitoring provider. */ + public function isHealthy(string $externalId): bool; } diff --git a/tests/Feature/Provisioning/CustomerProvisioningEndToEndTest.php b/tests/Feature/Provisioning/CustomerProvisioningEndToEndTest.php index 7ed2776..7ab87b0 100644 --- a/tests/Feature/Provisioning/CustomerProvisioningEndToEndTest.php +++ b/tests/Feature/Provisioning/CustomerProvisioningEndToEndTest.php @@ -15,7 +15,8 @@ it('provisions a paid order all the way to active (mocked)', function () { Queue::fake(); // drive the runner manually, one step per advance $s = fakeServices(); $s['pve']->guestScript('status.php', 0, 'ok'); - $s['pve']->guestScript('hostname -I', 0, '10.20.0.9'); // deploy health + acceptance probe + $s['pve']->guestScript('hostname -I', 0, '10.20.0.9'); + $s['pve']->guestScript('occ status', 0, '{"installed":true,"maintenance":false}'); // deploy health + acceptance probe // Fake defaults: tasks stopped/OK, guest agent up, cert reachable. Host::factory()->active()->create(['datacenter' => 'fsn', 'node' => 'pve', 'total_gb' => 1000]); @@ -64,6 +65,7 @@ it('does not duplicate external resources when a step re-runs after a crash', fu $s = fakeServices(); $s['pve']->guestScript('status.php', 0, 'ok'); $s['pve']->guestScript('hostname -I', 0, '10.20.0.9'); + $s['pve']->guestScript('occ status', 0, '{"installed":true,"maintenance":false}'); Host::factory()->active()->create(['datacenter' => 'fsn', 'node' => 'pve', 'total_gb' => 1000]); $order = Order::factory()->create(['status' => 'paid', 'plan' => 'start', 'datacenter' => 'fsn']); diff --git a/tests/Feature/Provisioning/CustomerStepsTest.php b/tests/Feature/Provisioning/CustomerStepsTest.php index 393e79f..5c8a11c 100644 --- a/tests/Feature/Provisioning/CustomerStepsTest.php +++ b/tests/Feature/Provisioning/CustomerStepsTest.php @@ -313,22 +313,28 @@ it('registers a backup job and a monitoring target once each', function () { }); // 14. RunAcceptanceChecks -it('passes acceptance only when everything is green', function () { - fakeServices(); +it('passes acceptance only when everything is genuinely green', function () { + $s = fakeServices(); + $s['pve']->guestScript('occ status', 0, '{"installed":true,"maintenance":false}'); // healthy Nextcloud + // occ user:info → default exit 0 (admin usable) ['run' => $run, 'instance' => $instance, 'host' => $host] = reservedRun([], ['cert_ok' => true]); - foreach (['nc_admin', 'backup_job_id', 'monitoring_target_id'] as $kind) { - RunResource::create(['run_id' => $run->id, 'host_id' => $host->id, 'kind' => $kind, 'external_id' => 'x']); - } + RunResource::create(['run_id' => $run->id, 'host_id' => $host->id, 'kind' => 'backup_job_id', 'external_id' => 'b']); + $instance->monitoringTargets()->create(['external_id' => 'mon-1', 'url' => 'https://x/status.php', 'status' => 'up']); expect(app(RunAcceptanceChecks::class)->execute($run)->type)->toBe('advance'); - // missing monitoring → fail - RunResource::where('run_id', $run->id)->where('kind', 'monitoring_target_id')->delete(); + // monitoring unhealthy → fail + $s['monitoring']->healthy = false; + expect(app(RunAcceptanceChecks::class)->execute($run->fresh())->type)->toBe('fail'); + $s['monitoring']->healthy = true; + + // Nextcloud not installed → fail + $s['pve']->guestScript('occ status', 0, '{"installed":false}'); expect(app(RunAcceptanceChecks::class)->execute($run->fresh())->type)->toBe('fail'); - // failing cert → fail - ['run' => $run2, 'host' => $host2] = reservedRun([], ['cert_ok' => false]); - expect(app(RunAcceptanceChecks::class)->execute($run2)->type)->toBe('fail'); + // cert not reachable → fail + $s['traefik']->certReady = false; + expect(app(RunAcceptanceChecks::class)->execute($run->fresh())->type)->toBe('fail'); }); // 15. CompleteProvisioning