CluPilotCloud/app/Support/Readiness/OperationChecks.php

<?php

namespace App\Support\Readiness;

use App\Support\Settings;
use Illuminate\Support\Carbon;

/**
 * Whether anything is actually running this installation, right now — as
 * opposed to every other group in this file, which asks whether it is
 * CONFIGURED to run at all.
 *
 * Queue-driven delivery has no error path for "nobody is picking up the
 * queue": a paid order just sits there, forever, with nothing in the console
 * to tell an operator apart from a quiet afternoon. Two heartbeats because the
 * scheduler and the provisioning worker fail independently — the dangerous
 * case is the scheduler running and enqueueing jobs that nobody executes, and
 * a single heartbeat written by the scheduler alone would call that healthy.
 */
final class OperationChecks
{
    public const GROUP = 'operation';

    /**
     * A heartbeat older than this is as good as none: the process that was
     * writing it has gone quiet. Five minutes is generous slack against a
     * `->everyMinute()` schedule — a deploy restarting the container, a GC
     * pause, one skipped tick — without also covering up a worker that has
     * actually stopped.
     */
    private const STALE_AFTER_MINUTES = 5;

    /**
     * Fix-Runde (Befund 3): a heartbeat this far into the future is as
     * untrustworthy as one from the past — a clock skewed between the web,
     * scheduler and worker containers, or a corrupted value that still
     * happens to parse. Without an upper bound, `now()->addHour()` written
     * once by a confused clock would read as healthy forever, since it would
     * never age past STALE_AFTER_MINUTES from a `greaterThan()`-only check.
     * One minute is generous slack for the same reason five is above:
     * everything that writes these timestamps runs on the same host clock,
     * so more than a minute of "future" is a fault, not jitter.
     */
    private const FUTURE_TOLERANCE_MINUTES = 1;

    /** @return array<int, Check> */
    public static function all(): array
    {
        return [
            new Check(
                key: 'operation.scheduler',
                group: self::GROUP,
                severity: Check::SEVERITY_BLOCKING,
                label: __('readiness.operation.scheduler'),
                breaks: __('readiness.operation.scheduler_breaks'),
                // Fix-Runde (Befund 4): 'integrations' is not a member of
                // Integrations::TABS (['services', 'platform', 'env']) — a
                // link to it would go nowhere. 'env' is where saveEnv() lives,
                // and saveEnv() is the actual remedy: it restarts the
                // scheduler and queue-provisioning workers (see its own
                // docblock), which is exactly what an operator would do about
                // either check in this group.
                tab: 'env',
                satisfied: self::isFresh('heartbeat.scheduler'),
            ),
            new Check(
                key: 'operation.queue_provisioning',
                group: self::GROUP,
                severity: Check::SEVERITY_BLOCKING,
                label: __('readiness.operation.queue_provisioning'),
                breaks: __('readiness.operation.queue_provisioning_breaks'),
                tab: 'env', // Same fix as above; same remedy (saveEnv()).
                // Written by RecordProvisioningHeartbeat, which the scheduler
                // enqueues but only the worker actually runs. A fresh
                // timestamp here is proof of a running worker, not just of a
                // running scheduler.
                satisfied: self::isFresh('heartbeat.queue_provisioning'),
            ),
        ];
    }

    private static function isFresh(string $key): bool
    {
        $recordedAt = Settings::get($key);

        if (blank($recordedAt)) {
            return false;
        }

        // Fix-Runde (Befund 2): reproduced live — a garbled value in the
        // heartbeat row (never written by RecordProvisioningHeartbeat or the
        // scheduler call themselves, but reachable by anyone editing
        // app_settings directly) made Carbon::parse() throw, which propagated
        // out of Readiness::all() and took the ENTIRE readiness page down
        // with it, not just this one check. A page whose only job is to
        // report calmly what is missing must not join the outage itself. An
        // unreadable value is treated exactly like a missing one: not
        // satisfied, nothing thrown.
        try {
            $parsed = Carbon::parse($recordedAt);
        } catch (\Throwable) {
            return false;
        }

        // Not diffInMinutes(): App\Models\Incident::durationMinutes() already
        // documents that it is SIGNED, and `now()->diffInMinutes($recordedAt)`
        // with $recordedAt in the past returns a NEGATIVE number — which
        // would compare as "less than 5" for a heartbeat six hours stale, and
        // report a dead worker as healthy. A direct threshold comparison has
        // no sign to get backwards.
        //
        // Bounded on both sides (Befund 3): stale in the past beyond
        // STALE_AFTER_MINUTES, but ALSO not more than FUTURE_TOLERANCE_MINUTES
        // ahead of now — otherwise a heartbeat from a skewed clock would read
        // as healthy indefinitely, since "in the future" always satisfies a
        // one-sided "not older than five minutes" comparison.
        return $parsed->greaterThan(now()->subMinutes(self::STALE_AFTER_MINUTES))
            && $parsed->lessThanOrEqualTo(now()->addMinutes(self::FUTURE_TOLERANCE_MINUTES));
    }
}