portal(), $this->instances(), $this->provisioning(), $this->backups(), ]; // The worst individual state is the state of the whole thing. Averaging // it would let one outage disappear behind three healthy components. $overall = match (true) { in_array('down', array_column($components, 'state'), true) => 'down', in_array('degraded', array_column($components, 'state'), true) => 'degraded', in_array('unknown', array_column($components, 'state'), true) => 'unknown', default => 'operational', }; return view('status', [ 'components' => $components, 'overall' => $overall, 'checkedAt' => Carbon::now(), ]); } /** * The portal and the website. * * Answering this request is the measurement. There is no honest way for a * page to report that the server serving it is down. * * @return array */ private function portal(): array { return [ 'key' => 'portal', 'state' => 'operational', 'detail' => null, ]; } /** * Customer instances, from what monitoring last saw. * * @return array */ private function instances(): array { $total = Instance::query()->where('status', 'active')->count(); if ($total === 0) { return ['key' => 'instances', 'state' => 'operational', 'detail' => null]; } // Only targets belonging to instances that are actually in service. A // healthy check on a decommissioned instance says nothing about a live // one, and counting it would let coverage look complete when it is not. $onActive = fn ($query) => $query->whereHas( 'instance', fn ($instance) => $instance->where('status', 'active'), ); // Only verdicts the sync job has actually refreshed. A row that has // never been checked, or was last checked long enough ago that the // answer means nothing, is not evidence of health — and this column was // for a long time exactly that: written 'up' at provisioning and never // touched again. $fresh = Carbon::now()->subMinutes(self::MONITORING_STALE_AFTER_MINUTES); $checked = fn ($query) => $query->tap($onActive)->where('checked_at', '>=', $fresh); $watched = MonitoringTarget::query()->tap($checked)->distinct()->count('instance_id'); $down = MonitoringTarget::query()->tap($checked)->where('status', '!=', 'up')->distinct()->count('instance_id'); // Down first, then coverage. An instance nobody is watching is not a // healthy instance — it is an unanswered question, and reporting the // absence of a check as the absence of a problem is the one thing this // page must never do. return [ 'key' => 'instances', 'state' => match (true) { $down > 0 && $down >= $watched => 'down', $down > 0 => 'degraded', $watched < $total => 'unknown', default => 'operational', }, 'detail' => match (true) { $down > 0 => ['down' => $down, 'total' => $watched], $watched < $total => ['down' => $total - $watched, 'total' => $total], default => null, }, ]; } /** * Whether new instances are being delivered. * * @return array */ private function provisioning(): array { $since = Carbon::now()->subHours(self::PROVISIONING_WINDOW_HOURS); $failed = ProvisioningRun::query() ->where('status', ProvisioningRun::STATUS_FAILED) ->where('updated_at', '>=', $since) ->count(); return [ 'key' => 'provisioning', 'state' => $failed === 0 ? 'operational' : 'degraded', 'detail' => $failed > 0 ? ['failed' => $failed] : null, ]; } /** * Whether the last backup of every instance is recent. * * @return array */ private function backups(): array { // Counted from the INSTANCES that need protecting, not from the backup // rows that happen to exist. An active instance with no schedule at all // has no row — so counting rows would leave it out of the arithmetic // entirely and report the estate as protected because the backups that // do exist are fine. $total = Instance::query()->where('status', 'active')->count(); if ($total === 0) { return ['key' => 'backups', 'state' => 'operational', 'detail' => null]; } $fresh = Carbon::now()->subHours(self::BACKUP_STALE_AFTER_HOURS); $protected = Instance::query() ->where('status', 'active') ->whereHas('backups', fn ($backup) => $backup->where('last_ok_at', '>=', $fresh)) ->count(); $unprotected = $total - $protected; return [ 'key' => 'backups', 'state' => match (true) { $unprotected === 0 => 'operational', $unprotected >= $total => 'down', default => 'degraded', }, 'detail' => $unprotected > 0 ? ['stale' => $unprotected, 'total' => $total] : null, ]; } }