CluPilotCloud/app/Provisioning/Steps/Customer/RunAcceptanceChecks.php

<?php

namespace App\Provisioning\Steps\Customer;

use App\Models\ProvisioningRun;
use App\Provisioning\StepResult;
use App\Services\Monitoring\MonitoringClient;
use App\Services\Proxmox\ProxmoxClient;
use App\Services\Traefik\TraefikWriter;
use App\Support\NextcloudOcc;
use App\Support\ProvisioningSettings;

/**
 * Gate: nothing grants the customer access until every check here actually
 * passes — the cert is served, Nextcloud reports healthy, the admin account is
 * usable, a backup exists, and monitoring is green.
 *
 * A check that says no is asked again before it condemns anything. Every one of
 * them is a PROBE — an HTTPS request from this VM, a command inside the guest —
 * and each has its own ways of answering no without anything being wrong:
 * TraefikWriter::certReachable() turns a connect timeout, a TLS handshake that
 * came in late and a DNS blip into exactly the same `false` as a missing
 * certificate, and a guest agent call fails outright while the guest is busy.
 * This step used to return fail() on the first no, which bypasses the retry
 * budget entirely — so one bad second on our side ended a fully built, certified,
 * working Nextcloud as a `failed` order with the instance released, which is the
 * worst outcome the pipeline can produce and the one it produced most easily.
 *
 * So the probes retry, like the rest of the pipeline, and the run still fails for
 * good once the budget is gone: five noes in a row is not a blip. What stays
 * terminal is the two facts a retry cannot change — state we wrote down
 * ourselves, earlier in this same run.
 */
class RunAcceptanceChecks extends CustomerStep
{
    /**
     * How long before a probe that answered no is asked again.
     *
     * Long enough for a warming Traefik router or a busy guest to settle, short
     * enough that five attempts still finish inside the delivery the customer is
     * watching.
     */
    private const PROBE_RETRY_SECONDS = 30;

    public function __construct(
        private ProxmoxClient $pve,
        private TraefikWriter $traefik,
        private MonitoringClient $monitoring,
    ) {}

    public function key(): string
    {
        return 'run_acceptance_checks';
    }

    public function maxDuration(): int
    {
        return 180;
    }

    public function execute(ProvisioningRun $run): StepResult
    {
        $instance = $this->instance($run);
        $node = (string) $run->context('node');
        $vmid = (int) $run->context('vmid');
        $fqdn = $instance->subdomain.'.'.ProvisioningSettings::dnsZone();
        $pve = $this->pve->forHost($instance->host);

        // Terminal, and the only cert case that is: `cert_ok` is written by
        // ConfigureDnsAndTls, which fails the run itself if the platform
        // certificate never appears. Standing here without it means the pipeline
        // reached acceptance in a state it cannot reach, and no number of
        // attempts fixes that.
        if (! $instance->cert_ok) {
            return StepResult::fail('acceptance_failed:cert');
        }

        // TLS + routing actually serving, asked of the outside world.
        if (! $this->traefik->certReachable($fqdn)) {
            return $this->askAgain('acceptance_failed:cert');
        }

        // Nextcloud installed, not in maintenance, DB reachable.
        $status = $pve->guestExec($node, $vmid, NextcloudOcc::command('status --output=json'));
        $health = json_decode($status['out-data'] ?? '', true) ?: [];
        if ((int) ($status['exitcode'] ?? 1) !== 0
            || ($health['installed'] ?? false) !== true
            || ($health['maintenance'] ?? false) !== false) {
            return $this->askAgain('acceptance_failed:nextcloud');
        }

        // The admin account exists and is queryable.
        $admin = $pve->guestExec($node, $vmid, NextcloudOcc::command('user:info '.escapeshellarg((string) $instance->nc_admin_ref)));
        if ((int) ($admin['exitcode'] ?? 1) !== 0) {
            return $this->askAgain('acceptance_failed:admin');
        }

        // Terminal for the same reason as cert_ok above: this is a breadcrumb THIS
        // run wrote at RegisterBackup, read out of our own database. If it is not
        // there, no backup job was registered, and asking again five times only
        // delays saying so.
        if (! $this->hasResource($run, 'backup_job_id')) {
            return StepResult::fail('acceptance_failed:backup');
        }

        // Monitoring is a SECONDARY signal: it depends on an external poller, and
        // a freshly created monitor legitimately has no heartbeat yet. The four
        // checks above already prove the service itself works, so only gate on
        // monitoring when the operator declared it required — otherwise record it
        // and deliver (consistent with RegisterMonitoring's degrade-on-outage).
        $target = $instance->monitoringTargets()->first();
        $monitoringGreen = $target !== null && $this->monitoring->isHealthy($target->external_id);

        if (! $monitoringGreen) {
            if (config('provisioning.monitoring.required', false)) {
                return StepResult::fail('acceptance_failed:monitoring');
            }

            $run->events()->create([
                'step' => $this->key(),
                'attempt' => $run->attempt,
                'outcome' => 'info',
                'message' => $target === null
                    ? 'Abnahme ohne Monitoring: kein Monitor registriert.'
                    : 'Abnahme ohne Monitoring-Bestätigung: Monitor noch nicht grün.',
            ]);
        }

        return StepResult::advance();
    }

    /**
     * Ask the probes again, and keep the reason for the day they run out.
     *
     * retry(), not poll(): a poll does not consume the budget, so a genuinely
     * broken instance would sit here re-probing until the step's own maxDuration
     * turned it into a nameless timeout. A retry is bounded by max_attempts and
     * RunRunner hands this exact reason to failRun() when the last one is spent,
     * so an operator still reads `acceptance_failed:nextcloud` rather than
     * "step timed out".
     */
    private function askAgain(string $reason): StepResult
    {
        return StepResult::retry(self::PROBE_RETRY_SECONDS, $reason);
    }
}