option('interval')); /** @var array $clients */ $clients = []; do { // Only servers with a PRESENT, non-disabled credential. A credential disabled // between ticks drops the server here, so its cached connection is no longer polled // (the vault refuses a fresh connect, but a reused long-lived session would not // re-check disabled_at). Prune any cached client whose server just left the set so // the revoked server's open session is closed promptly, not left idle until exit. $servers = Server::withActiveCredential()->get(); $activeIds = $servers->pluck('id')->all(); foreach (array_keys($clients) as $id) { if (! in_array($id, $activeIds, true)) { $clients[$id]->disconnect(); unset($clients[$id]); } } if ($servers->isEmpty()) { $this->warn('Keine Server mit aktivem Credential — nichts zu pollen.'); } foreach ($servers as $server) { try { $clients[$server->id] ??= (new SshClient($vault, timeout: 12))->connect($server); $m = $fleet->readMetrics($clients[$server->id]); $fleet->applyMetrics($server, $m); // forceFills the fresh status onto $server broadcast(new MetricsTicked($m['cpu'], $m['mem'], $server->name)); $this->evaluateAlerts($alerts, $server, $m, $server->status); $this->line("ok {$server->name} cpu={$m['cpu']} mem={$m['mem']} disk={$m['disk']} load={$m['load']}"); } catch (Throwable $e) { // drop the (possibly dead) connection so the next tick reconnects if (isset($clients[$server->id])) { $clients[$server->id]->disconnect(); unset($clients[$server->id]); } $server->forceFill(['status' => 'offline'])->save(); $this->evaluateAlerts($alerts, $server, [], 'offline'); // an offline rule fires here $this->warn("offline {$server->name}: ".$e->getMessage()); } } if ($this->option('once')) { break; } sleep($interval); } while (true); foreach ($clients as $client) { $client->disconnect(); } return self::SUCCESS; } /** * Run the alert rules for a freshly-polled server. Best-effort: an alerting/notification error * must NEVER kill the poll loop, so any throwable is caught and logged as a warning line. * * @param array $metrics */ private function evaluateAlerts(AlertEvaluator $alerts, Server $server, array $metrics, string $status): void { try { $alerts->evaluate($server, $metrics, $status); } catch (Throwable $e) { $this->warn("alert eval failed {$server->name}: ".$e->getMessage()); } } }