everyMinute()` schedule — a deploy restarting the container, a GC * pause, one skipped tick — without also covering up a worker that has * actually stopped. */ private const STALE_AFTER_MINUTES = 5; /** @return array */ public static function all(): array { return [ new Check( key: 'operation.scheduler', group: self::GROUP, severity: Check::SEVERITY_BLOCKING, label: __('readiness.operation.scheduler'), breaks: __('readiness.operation.scheduler_breaks'), tab: 'integrations', satisfied: self::isFresh('heartbeat.scheduler'), ), new Check( key: 'operation.queue_provisioning', group: self::GROUP, severity: Check::SEVERITY_BLOCKING, label: __('readiness.operation.queue_provisioning'), breaks: __('readiness.operation.queue_provisioning_breaks'), tab: 'integrations', // Written by RecordProvisioningHeartbeat, which the scheduler // enqueues but only the worker actually runs. A fresh // timestamp here is proof of a running worker, not just of a // running scheduler. satisfied: self::isFresh('heartbeat.queue_provisioning'), ), ]; } private static function isFresh(string $key): bool { $recordedAt = Settings::get($key); if (blank($recordedAt)) { return false; } // Not diffInMinutes(): App\Models\Incident::durationMinutes() already // documents that it is SIGNED, and `now()->diffInMinutes($recordedAt)` // with $recordedAt in the past returns a NEGATIVE number — which // would compare as "less than 5" for a heartbeat six hours stale, and // report a dead worker as healthy. A direct threshold comparison has // no sign to get backwards. return Carbon::parse($recordedAt)->greaterThan(now()->subMinutes(self::STALE_AFTER_MINUTES)); } }