CluPilotCloud/app/Http/Controllers/StatusController.php

<?php

namespace App\Http\Controllers;

use App\Models\Instance;
use App\Models\MonitoringTarget;
use App\Models\ProvisioningRun;
use Illuminate\Contracts\View\View;
use Illuminate\Support\Carbon;

/**
 * The public service status page.
 *
 * It lived at /legal/status, next to the imprint and the terms, which is a
 * filing mistake: nothing about the current health of the platform is a legal
 * document. It has its own address now.
 *
 * Every line on it is derived from a record. Where there is no signal the
 * component says "not monitored" rather than "operational" — a status page that
 * reports green because it has nothing to look at is worse than no status page,
 * because someone will believe it.
 *
 * Aggregate only: counts, never customer names, instance addresses or host
 * names. This page is world-readable and the estate is not public information.
 */
class StatusController extends Controller
{
    /** A backup older than this stops counting as current. */
    private const BACKUP_STALE_AFTER_HOURS = 48;

    /**
     * How old a monitoring verdict may be before it stops counting.
     *
     * The sync job runs every five minutes; four missed runs is a monitoring
     * pipeline that has stopped, and a verdict from then says nothing about now.
     */
    private const MONITORING_STALE_AFTER_MINUTES = 20;

    /** How far back a provisioning failure still says something about now. */
    private const PROVISIONING_WINDOW_HOURS = 24;

    public function __invoke(): View
    {
        $components = [
            $this->portal(),
            $this->instances(),
            $this->provisioning(),
            $this->backups(),
        ];

        // The worst individual state is the state of the whole thing. Averaging
        // it would let one outage disappear behind three healthy components.
        $overall = match (true) {
            in_array('down', array_column($components, 'state'), true) => 'down',
            in_array('degraded', array_column($components, 'state'), true) => 'degraded',
            in_array('unknown', array_column($components, 'state'), true) => 'unknown',
            default => 'operational',
        };

        return view('status', [
            'components' => $components,
            'overall' => $overall,
            'checkedAt' => Carbon::now(),
        ]);
    }

    /**
     * The portal and the website.
     *
     * Answering this request is the measurement. There is no honest way for a
     * page to report that the server serving it is down.
     *
     * @return array<string, mixed>
     */
    private function portal(): array
    {
        return [
            'key' => 'portal',
            'state' => 'operational',
            'detail' => null,
        ];
    }

    /**
     * Customer instances, from what monitoring last saw.
     *
     * @return array<string, mixed>
     */
    private function instances(): array
    {
        $total = Instance::query()->where('status', 'active')->count();

        if ($total === 0) {
            return ['key' => 'instances', 'state' => 'operational', 'detail' => null];
        }

        // Only targets belonging to instances that are actually in service. A
        // healthy check on a decommissioned instance says nothing about a live
        // one, and counting it would let coverage look complete when it is not.
        $onActive = fn ($query) => $query->whereHas(
            'instance',
            fn ($instance) => $instance->where('status', 'active'),
        );

        // Only verdicts the sync job has actually refreshed. A row that has
        // never been checked, or was last checked long enough ago that the
        // answer means nothing, is not evidence of health — and this column was
        // for a long time exactly that: written 'up' at provisioning and never
        // touched again.
        $fresh = Carbon::now()->subMinutes(self::MONITORING_STALE_AFTER_MINUTES);
        $checked = fn ($query) => $query->tap($onActive)->where('checked_at', '>=', $fresh);

        $watched = MonitoringTarget::query()->tap($checked)->distinct()->count('instance_id');
        $down = MonitoringTarget::query()->tap($checked)->where('status', '!=', 'up')->distinct()->count('instance_id');

        // Down first, then coverage. An instance nobody is watching is not a
        // healthy instance — it is an unanswered question, and reporting the
        // absence of a check as the absence of a problem is the one thing this
        // page must never do.
        return [
            'key' => 'instances',
            'state' => match (true) {
                $down > 0 && $down >= $watched => 'down',
                $down > 0 => 'degraded',
                $watched < $total => 'unknown',
                default => 'operational',
            },
            'detail' => match (true) {
                $down > 0 => ['down' => $down, 'total' => $watched],
                $watched < $total => ['down' => $total - $watched, 'total' => $total],
                default => null,
            },
        ];
    }

    /**
     * Whether new instances are being delivered.
     *
     * @return array<string, mixed>
     */
    private function provisioning(): array
    {
        $since = Carbon::now()->subHours(self::PROVISIONING_WINDOW_HOURS);

        $failed = ProvisioningRun::query()
            ->where('status', ProvisioningRun::STATUS_FAILED)
            ->where('updated_at', '>=', $since)
            ->count();

        return [
            'key' => 'provisioning',
            'state' => $failed === 0 ? 'operational' : 'degraded',
            'detail' => $failed > 0 ? ['failed' => $failed] : null,
        ];
    }

    /**
     * Whether the last backup of every instance is recent.
     *
     * @return array<string, mixed>
     */
    private function backups(): array
    {
        // Counted from the INSTANCES that need protecting, not from the backup
        // rows that happen to exist. An active instance with no schedule at all
        // has no row — so counting rows would leave it out of the arithmetic
        // entirely and report the estate as protected because the backups that
        // do exist are fine.
        $total = Instance::query()->where('status', 'active')->count();

        if ($total === 0) {
            return ['key' => 'backups', 'state' => 'operational', 'detail' => null];
        }

        $fresh = Carbon::now()->subHours(self::BACKUP_STALE_AFTER_HOURS);

        $protected = Instance::query()
            ->where('status', 'active')
            ->whereHas('backups', fn ($backup) => $backup->where('last_ok_at', '>=', $fresh))
            ->count();

        $unprotected = $total - $protected;

        return [
            'key' => 'backups',
            'state' => match (true) {
                $unprotected === 0 => 'operational',
                $unprotected >= $total => 'down',
                default => 'degraded',
            },
            'detail' => $unprotected > 0 ? ['stale' => $unprotected, 'total' => $total] : null,
        ];
    }
}