CluPilotCloud/app/Services/Status/ServiceHealth.php

<?php

namespace App\Services\Status;

use App\Models\Instance;
use App\Models\MonitoringTarget;
use App\Models\ProvisioningRun;
use Illuminate\Support\Carbon;

/**
 * What the four public components are doing right now.
 *
 * Lifted out of StatusController so that the page and the sampler that writes
 * the ninety-day history ask the same question in the same words. Two
 * implementations of "is provisioning healthy" would disagree the first time
 * one of them was changed, and the disagreement would show up as a green bar
 * under a red banner.
 *
 * Every line is derived from a record. Where there is no signal a component
 * says "not monitored" rather than "operational" — a status page that reports
 * green because it has nothing to look at is worse than no status page,
 * because someone will believe it.
 *
 * Aggregate only: counts, never customer names, instance addresses or host
 * names. This is world-readable and the estate is not public information.
 */
class ServiceHealth
{
    /** The components, in the order they appear. */
    public const COMPONENTS = ['portal', 'instances', 'provisioning', 'backups'];

    /** A backup older than this stops counting as current. */
    private const BACKUP_STALE_AFTER_HOURS = 48;

    /**
     * How old a monitoring verdict may be before it stops counting.
     *
     * The sync job runs every five minutes; four missed runs is a monitoring
     * pipeline that has stopped, and a verdict from then says nothing about now.
     */
    private const MONITORING_STALE_AFTER_MINUTES = 20;

    /** How far back a provisioning failure still says something about now. */
    private const PROVISIONING_WINDOW_HOURS = 24;

    /**
     * @return array<int, array{key: string, state: string, detail: array<string, int>|null}>
     */
    public function components(): array
    {
        return [
            $this->portal(),
            $this->instances(),
            $this->provisioning(),
            $this->backups(),
        ];
    }

    /**
     * The worst individual state is the state of the whole thing. Averaging it
     * would let one outage disappear behind three healthy components.
     *
     * @param  array<int, array{state: string}>  $components
     */
    public function overall(array $components): string
    {
        $states = array_column($components, 'state');

        return match (true) {
            in_array('down', $states, true) => 'down',
            in_array('degraded', $states, true) => 'degraded',
            in_array('unknown', $states, true) => 'unknown',
            default => 'operational',
        };
    }

    /**
     * The portal and the website.
     *
     * Answering this request is the measurement. There is no honest way for a
     * page to report that the server serving it is down.
     *
     * @return array<string, mixed>
     */
    private function portal(): array
    {
        return ['key' => 'portal', 'state' => 'operational', 'detail' => null];
    }

    /**
     * Customer instances, from what monitoring last saw.
     *
     * @return array<string, mixed>
     */
    private function instances(): array
    {
        $total = Instance::query()->where('status', 'active')->count();

        if ($total === 0) {
            return ['key' => 'instances', 'state' => 'operational', 'detail' => null];
        }

        // Only targets belonging to instances that are actually in service. A
        // healthy check on a decommissioned instance says nothing about a live
        // one, and counting it would let coverage look complete when it is not.
        $onActive = fn ($query) => $query->whereHas(
            'instance',
            fn ($instance) => $instance->where('status', 'active'),
        );

        // Only verdicts the sync job has actually refreshed. A row that has
        // never been checked, or was last checked long enough ago that the
        // answer means nothing, is not evidence of health — and this column was
        // for a long time exactly that: written 'up' at provisioning and never
        // touched again.
        $fresh = Carbon::now()->subMinutes(self::MONITORING_STALE_AFTER_MINUTES);
        $checked = fn ($query) => $query->tap($onActive)->where('checked_at', '>=', $fresh);

        $watched = MonitoringTarget::query()->tap($checked)->distinct()->count('instance_id');
        $down = MonitoringTarget::query()->tap($checked)->where('status', '!=', 'up')->distinct()->count('instance_id');

        // Down first, then coverage. An instance nobody is watching is not a
        // healthy instance — it is an unanswered question, and reporting the
        // absence of a check as the absence of a problem is the one thing this
        // page must never do.
        return [
            'key' => 'instances',
            'state' => match (true) {
                $down > 0 && $down >= $watched => 'down',
                $down > 0 => 'degraded',
                $watched < $total => 'unknown',
                default => 'operational',
            },
            'detail' => match (true) {
                $down > 0 => ['down' => $down, 'total' => $watched],
                $watched < $total => ['down' => $total - $watched, 'total' => $total],
                default => null,
            },
        ];
    }

    /**
     * Whether new instances are being delivered.
     *
     * @return array<string, mixed>
     */
    private function provisioning(): array
    {
        $since = Carbon::now()->subHours(self::PROVISIONING_WINDOW_HOURS);

        $failed = ProvisioningRun::query()
            ->where('status', ProvisioningRun::STATUS_FAILED)
            ->where('updated_at', '>=', $since)
            ->count();

        return [
            'key' => 'provisioning',
            'state' => $failed === 0 ? 'operational' : 'degraded',
            'detail' => $failed > 0 ? ['failed' => $failed] : null,
        ];
    }

    /**
     * Whether the last backup of every instance is recent.
     *
     * @return array<string, mixed>
     */
    private function backups(): array
    {
        // Counted from the INSTANCES that need protecting, not from the backup
        // rows that happen to exist. An active instance with no schedule at all
        // has no row — so counting rows would leave it out of the arithmetic
        // entirely and report the estate as protected because the backups that
        // do exist are fine.
        $total = Instance::query()->where('status', 'active')->count();

        if ($total === 0) {
            return ['key' => 'backups', 'state' => 'operational', 'detail' => null];
        }

        $fresh = Carbon::now()->subHours(self::BACKUP_STALE_AFTER_HOURS);

        $protected = Instance::query()
            ->where('status', 'active')
            ->whereHas('backups', fn ($backup) => $backup->where('last_ok_at', '>=', $fresh))
            ->count();

        $unprotected = $total - $protected;

        return [
            'key' => 'backups',
            'state' => match (true) {
                $unprotected === 0 => 'operational',
                $unprotected >= $total => 'down',
                default => 'degraded',
            },
            'detail' => $unprotected > 0 ? ['stale' => $unprotected, 'total' => $total] : null,
        ];
    }
}