clusev/app/Services/FleetService.php

653 lines
24 KiB
PHP

<?php
namespace App\Services;
use App\Models\Server;
use App\Support\Ssh\CredentialVault;
use App\Support\Ssh\Sftp;
use App\Support\Ssh\SshClient;
use Illuminate\Support\Facades\Cache;
use InvalidArgumentException;
use phpseclib3\Crypt\PublicKeyLoader;
use RuntimeException;
use Throwable;
/**
* Reads real host facts over SSH and maps raw command output into the shapes the
* Livewire pages already consume. All parsing forces the C locale (the target may
* be localized) and prefers /proc over `free`/`top` for locale-independent numbers.
*
* Every public reader connects, runs one compound command, parses, disconnects.
* Connection/parse failures bubble up as RuntimeException — callers render an
* offline state instead of crashing.
*/
class FleetService
{
private const MARK = '===CLUSEV:';
public function __construct(private readonly CredentialVault $vault) {}
private function client(Server $server): SshClient
{
return (new SshClient($this->vault, timeout: 12))->connect($server);
}
/** Split a marker-delimited compound output into [section => body]. */
private function sections(string $out): array
{
$parts = [];
$current = null;
foreach (preg_split('/\R/', $out) as $line) {
if (str_starts_with($line, self::MARK)) {
$current = rtrim(substr($line, strlen(self::MARK)), '=');
$parts[$current] = [];
continue;
}
if ($current !== null) {
$parts[$current][] = $line;
}
}
return array_map(fn (array $l): string => implode("\n", $l), $parts);
}
// ── Metrics (light — used by the poller and dashboard fallback) ──────
/** @return array{cpu:int,mem:int,disk:int,load:float} */
public function metrics(Server $server): array
{
$ssh = $this->client($server);
try {
return $this->readMetrics($ssh);
} finally {
$ssh->disconnect();
}
}
/**
* Run the metrics command on an already-connected client. The poller reuses
* one long-lived connection per server (a single login, not one per tick).
*
* @return array{cpu:int,mem:int,disk:int,load:float}
*/
public function readMetrics(SshClient $ssh): array
{
$cmd = 'export LC_ALL=C; '
.'echo '.self::MARK.'mem===; grep -E "MemTotal|MemAvailable" /proc/meminfo; '
.'echo '.self::MARK.'stat1===; head -1 /proc/stat; sleep 1; '
.'echo '.self::MARK.'stat2===; head -1 /proc/stat; '
.'echo '.self::MARK.'load===; cat /proc/loadavg; '
.'echo '.self::MARK.'disk===; df -B1 --output=pcent / | tail -1';
$s = $this->sections($ssh->exec($cmd));
return [
'cpu' => $this->cpuPercent($s['stat1'] ?? '', $s['stat2'] ?? ''),
'mem' => $this->memPercent($s['mem'] ?? ''),
'disk' => (int) trim(str_replace('%', '', $s['disk'] ?? '0')),
'load' => (float) (preg_split('/\s+/', trim($s['load'] ?? '0'))[0] ?? 0),
];
}
/** Poll metrics and persist them onto the Server row. */
public function refresh(Server $server): array
{
$m = $this->metrics($server);
$this->applyMetrics($server, $m);
return $m;
}
/** Persist a metrics reading onto the Server row + cache (latest + rolling history). */
public function applyMetrics(Server $server, array $m): void
{
$server->forceFill([
'cpu' => $m['cpu'],
'mem' => $m['mem'],
'disk' => $m['disk'],
'status' => $this->statusFor($m),
'last_seen_at' => now(),
])->save();
Cache::put("metrics:latest:{$server->id}", $m, now()->addMinutes(5));
$history = Cache::get("metrics:history:{$server->id}", []);
$history[] = ['cpu' => $m['cpu'], 'mem' => $m['mem']];
Cache::put("metrics:history:{$server->id}", array_slice($history, -40), now()->addHour());
}
/** Last persisted metrics reading (cache), or null. */
public function latest(Server $server): ?array
{
return Cache::get("metrics:latest:{$server->id}");
}
/**
* Rolling cpu/mem history for sparklines/charts.
*
* @return array{cpu: array<int,int>, mem: array<int,int>}
*/
public function history(Server $server): array
{
$h = Cache::get("metrics:history:{$server->id}", []);
return [
'cpu' => array_map('intval', array_column($h, 'cpu')),
'mem' => array_map('intval', array_column($h, 'mem')),
];
}
private function statusFor(array $m): string
{
return match (true) {
$m['cpu'] >= 90 || $m['mem'] >= 92 || $m['disk'] >= 90 => 'warning',
default => 'online',
};
}
// ── Full host snapshot (Server-Details) ─────────────────────────────
public function snapshot(Server $server): array
{
$ssh = $this->client($server);
$cmd = 'export LC_ALL=C; '
.'echo '.self::MARK.'mem===; grep -E "MemTotal|MemAvailable" /proc/meminfo; '
.'echo '.self::MARK.'stat1===; head -1 /proc/stat; sleep 1; '
.'echo '.self::MARK.'stat2===; head -1 /proc/stat; '
.'echo '.self::MARK.'load===; cat /proc/loadavg; '
.'echo '.self::MARK.'uptime===; cat /proc/uptime; '
.'echo '.self::MARK.'os===; . /etc/os-release 2>/dev/null; echo "$PRETTY_NAME"; uname -r; uname -m; nproc; hostname; systemd-detect-virt 2>/dev/null || echo -; '
.'echo '.self::MARK.'df===; df -B1 -x tmpfs -x devtmpfs -x overlay -x squashfs --output=target,fstype,size,pcent; '
.'echo '.self::MARK.'addr===; ip -o -4 addr show; '
.'echo '.self::MARK.'link===; ip -o link show; '
.'echo '.self::MARK.'netdev===; cat /proc/net/dev; '
.'echo '.self::MARK.'sshd===; grep -Ei "^[[:space:]]*(permitrootlogin|passwordauthentication)" /etc/ssh/sshd_config 2>/dev/null; '
.'echo '.self::MARK.'active===; for u in fail2ban ufw nftables unattended-upgrades; do echo "$u $(systemctl is-active $u 2>/dev/null)"; done; '
.'echo '.self::MARK.'keys===; ssh-keygen -lf ~/.ssh/authorized_keys 2>/dev/null';
try {
$s = $this->sections($ssh->exec($cmd));
} finally {
$ssh->disconnect();
}
$osLines = $this->lines($s['os'] ?? '');
$totalKb = $this->memTotalKb($s['mem'] ?? '');
$volumes = $this->parseVolumes($s['df'] ?? '');
$rootBytes = collect($volumes)->firstWhere('mount', '/')['bytes'] ?? 0;
return [
'metrics' => [
'cpu' => $this->cpuPercent($s['stat1'] ?? '', $s['stat2'] ?? ''),
'mem' => $this->memPercent($s['mem'] ?? ''),
'load' => (float) (preg_split('/\s+/', trim($s['load'] ?? '0'))[0] ?? 0),
],
'identity' => [
'os' => $osLines[0] ?? '—',
'kernel' => $osLines[1] ?? '—',
'arch' => $osLines[2] ?? '—',
'cores' => (int) ($osLines[3] ?? 0),
'hostname' => $osLines[4] ?? '—',
'virt' => $osLines[5] ?? '—',
'ram_gb' => $totalKb > 0 ? round($totalKb / 1048576, 1) : 0,
'disk_gb' => $rootBytes > 0 ? (int) round($rootBytes / 1e9) : 0,
'uptime' => $this->humanUptime((float) (preg_split('/\s+/', trim($s['uptime'] ?? '0'))[0] ?? 0)),
],
'volumes' => $volumes,
'interfaces' => $this->parseInterfaces($s['addr'] ?? '', $s['link'] ?? '', $s['netdev'] ?? ''),
'hardening' => $this->parseHardening($s['sshd'] ?? '', $s['active'] ?? ''),
'sshKeys' => $this->parseKeys($s['keys'] ?? ''),
];
}
// ── systemd services + journal (Services page) ──────────────────────
/** @return array{services:array<int,array>,journal:array<int,array>} */
public function systemd(Server $server, int $journalLines = 25): array
{
$ssh = $this->client($server);
$cmd = 'export LC_ALL=C; '
.'echo '.self::MARK.'units===; systemctl list-units --type=service --all --plain --no-legend --no-pager; '
.'echo '.self::MARK.'files===; systemctl list-unit-files --type=service --no-legend --no-pager; '
.'echo '.self::MARK.'journal===; if [ "$(id -u)" = 0 ]; then SUDO=""; else SUDO="sudo -n"; fi; $SUDO journalctl -n '.(int) $journalLines.' --no-pager -o short-iso 2>&1';
try {
$s = $this->sections($ssh->exec($cmd));
} finally {
$ssh->disconnect();
}
return [
'services' => $this->parseUnits($s['units'] ?? '', $s['files'] ?? ''),
'journal' => $this->parseJournal($s['journal'] ?? ''),
];
}
// ── SFTP-ish directory listing (Files page) ─────────────────────────
/** @return array<int,array{name:string,type:string,size:?int,perms:string,owner:string,modified:string}> */
public function files(Server $server, string $path): array
{
$ssh = $this->client($server);
$safe = "'".str_replace("'", "'\\''", $path)."'";
try {
[$out, $code] = $ssh->run('export LC_ALL=C; ls -la --time-style=long-iso '.$safe.' 2>/dev/null');
} finally {
$ssh->disconnect();
}
if ($code !== 0) {
return [];
}
$entries = [];
foreach ($this->lines($out) as $line) {
if ($line === '' || str_starts_with($line, 'total ') || str_starts_with($line, 'insgesamt ')) {
continue;
}
// perms links owner group size YYYY-MM-DD HH:MM name...
$p = preg_split('/\s+/', $line, 8);
if (count($p) < 8) {
continue;
}
[$perms, , $owner, , $size, $date, $time, $name] = $p;
if ($name === '.' || $name === '..') {
continue;
}
$type = match ($perms[0] ?? '-') {
'd' => 'dir',
'l' => 'link',
default => 'file',
};
// strip a "-> target" suffix from symlink names
if ($type === 'link' && str_contains($name, ' -> ')) {
$name = strstr($name, ' -> ', true);
}
$entries[] = [
'name' => $name,
'type' => $type,
'size' => $type === 'dir' ? null : (int) $size,
'perms' => $perms,
'owner' => $owner,
'modified' => $date.' '.$time,
];
}
usort($entries, fn ($a, $b) => [$a['type'] !== 'dir', $a['name']] <=> [$b['type'] !== 'dir', $b['name']]);
return $entries;
}
// ── control (write) actions — work without sudo, in the SSH user's space ─
/** Append a public key to the SSH user's authorized_keys. */
public function addAuthorizedKey(Server $server, string $publicKey): void
{
$key = trim(preg_replace('/\s+/', ' ', $publicKey) ?? '');
if (! preg_match('#^(ssh-(rsa|ed25519|dss)|ecdsa-sha2-\S+)\s+[A-Za-z0-9+/=]+#', $key)) {
throw new InvalidArgumentException('Ungueltiger SSH-Public-Key.');
}
$ssh = $this->client($server);
try {
// base64-encode so the key never touches the shell unquoted
$b64 = base64_encode($key);
[$out] = $ssh->run(
'umask 077; mkdir -p ~/.ssh && touch ~/.ssh/authorized_keys && chmod 700 ~/.ssh && chmod 600 ~/.ssh/authorized_keys; '
."k=$(printf '%s' '{$b64}' | base64 -d); "
.'grep -qxF "$k" ~/.ssh/authorized_keys || printf "%s\n" "$k" >> ~/.ssh/authorized_keys; echo DONE'
);
if (! str_contains($out, 'DONE')) {
throw new RuntimeException('Schreiben der authorized_keys fehlgeschlagen.');
}
} finally {
$ssh->disconnect();
}
}
/** Remove an authorized key by SHA256 fingerprint, preserving all others. */
public function removeAuthorizedKey(Server $server, string $fingerprint): void
{
$sftp = (new Sftp($this->vault))->connect($server);
try {
$content = $sftp->get('.ssh/authorized_keys');
if (trim($content) === '') {
throw new RuntimeException('authorized_keys leer oder nicht lesbar.');
}
$lines = array_values(array_filter(
array_map('rtrim', preg_split('/\R/', $content)),
fn ($l) => $l !== ''
));
$kept = array_values(array_filter($lines, fn ($l) => $this->keyFingerprint($l) !== $fingerprint));
if (count($kept) === count($lines)) {
return; // nothing matched — never rewrite blindly (no lockout)
}
$sftp->put('.ssh/authorized_keys', $kept === [] ? '' : implode("\n", $kept)."\n");
} finally {
$sftp->disconnect();
}
}
private function keyFingerprint(string $line): ?string
{
try {
return 'SHA256:'.PublicKeyLoader::load($line)->getFingerprint('sha256');
} catch (Throwable) {
return null;
}
}
/** Re-read just the authorized keys (after an add/remove). */
public function sshKeys(Server $server): array
{
$ssh = $this->client($server);
try {
return $this->parseKeys($ssh->exec('ssh-keygen -lf ~/.ssh/authorized_keys 2>/dev/null'));
} finally {
$ssh->disconnect();
}
}
/** Delete a remote file over SFTP (needs write permission; no sudo). */
public function deleteFile(Server $server, string $path): void
{
$sftp = (new Sftp($this->vault))->connect($server);
try {
if (! $sftp->delete($path)) {
throw new RuntimeException('Loeschen fehlgeschlagen (Rechte?).');
}
} finally {
$sftp->disconnect();
}
}
/**
* systemctl start/stop/restart. Runs directly as root, else via `sudo -n`
* (needs a root or nopasswd-sudo credential).
*
* @return array{ok: bool, output: string}
*/
public function serviceAction(Server $server, string $op, string $unit): array
{
if (! in_array($op, ['start', 'stop', 'restart'], true)) {
throw new InvalidArgumentException('Unbekannte Aktion.');
}
if (! preg_match('/^[\w@.:-]+$/', $unit)) {
throw new InvalidArgumentException('Ungueltige Unit.');
}
$ssh = $this->client($server);
try {
[$out, $code] = $ssh->run(
'export LC_ALL=C; if [ "$(id -u)" = 0 ]; then SUDO=""; else SUDO="sudo -n"; fi; '
."\$SUDO systemctl {$op} {$unit} 2>&1"
);
return ['ok' => $code === 0, 'output' => trim($out)];
} finally {
$ssh->disconnect();
}
}
// ── parsers ─────────────────────────────────────────────────────────
private function lines(string $body): array
{
return array_values(array_filter(
array_map('rtrim', preg_split('/\R/', $body)),
fn ($l) => $l !== ''
));
}
private function cpuPercent(string $a, string $b): int
{
$fa = $this->cpuFields($a);
$fb = $this->cpuFields($b);
if (! $fa || ! $fb) {
return 0;
}
$totalA = array_sum($fa);
$totalB = array_sum($fb);
$dTotal = $totalB - $totalA;
$dIdle = ($fb['idle'] + $fb['iowait']) - ($fa['idle'] + $fa['iowait']);
if ($dTotal <= 0) {
return 0;
}
return max(0, min(100, (int) round(100 * ($dTotal - $dIdle) / $dTotal)));
}
private function cpuFields(string $line): array
{
$p = preg_split('/\s+/', trim($line));
if (($p[0] ?? '') !== 'cpu') {
return [];
}
$n = array_map('intval', array_slice($p, 1, 8));
return [
'user' => $n[0] ?? 0, 'nice' => $n[1] ?? 0, 'system' => $n[2] ?? 0,
'idle' => $n[3] ?? 0, 'iowait' => $n[4] ?? 0, 'irq' => $n[5] ?? 0,
'softirq' => $n[6] ?? 0, 'steal' => $n[7] ?? 0,
];
}
private function memTotalKb(string $body): int
{
return (int) (preg_match('/MemTotal:\s+(\d+)/', $body, $m) ? $m[1] : 0);
}
private function memPercent(string $body): int
{
$total = $this->memTotalKb($body);
$avail = (int) (preg_match('/MemAvailable:\s+(\d+)/', $body, $m) ? $m[1] : 0);
return $total > 0 ? max(0, min(100, (int) round(100 * ($total - $avail) / $total))) : 0;
}
private function humanUptime(float $seconds): string
{
$d = intdiv((int) $seconds, 86400);
$h = intdiv((int) $seconds % 86400, 3600);
return $d > 0 ? "{$d}d {$h}h" : "{$h}h";
}
private function parseVolumes(string $body): array
{
$out = [];
foreach ($this->lines($body) as $line) {
// df --output=target,fstype,size,pcent; the mount (target) can contain
// spaces, so pop the three trailing single-token fields and rejoin the rest.
$p = preg_split('/\s+/', trim($line));
if (count($p) < 4 || ! str_starts_with($p[0], '/')) {
continue; // header / noise
}
$pcent = array_pop($p);
$size = array_pop($p);
$fs = array_pop($p);
$mount = implode(' ', $p);
if (! ctype_digit($size)) {
continue;
}
$out[] = [
'mount' => $mount,
'fs' => $fs,
'size' => $this->humanBytes((int) $size),
'bytes' => (int) $size,
'used' => (int) str_replace('%', '', $pcent),
];
}
return $out;
}
private function parseInterfaces(string $addr, string $link, string $netdev): array
{
$ips = [];
foreach ($this->lines($addr) as $line) {
// "2: ens18 inet 10.10.90.162/24 brd ..."
if (preg_match('/^\d+:\s+(\S+)\s+inet\s+([\d.]+)/', $line, $m)) {
$ips[$m[1]][] = $m[2];
}
}
$macs = [];
foreach ($this->lines($link) as $line) {
if (preg_match('/^\d+:\s+([^:@]+)[:@].*link\/\w+\s+([0-9a-f:]{17})/', $line, $m)) {
$macs[trim($m[1])] = $m[2];
}
}
$traffic = [];
foreach ($this->lines($netdev) as $line) {
if (preg_match('/^\s*([^:]+):\s*(\d+)(?:\s+\d+){7}\s+(\d+)/', $line, $m)) {
$traffic[trim($m[1])] = ['rx' => (int) $m[2], 'tx' => (int) $m[3]];
}
}
$out = [];
foreach ($ips as $name => $list) {
$out[] = [
'name' => $name,
'ip' => implode(', ', $list),
'mac' => $macs[$name] ?? '—',
'rx' => isset($traffic[$name]) ? $this->humanBytes($traffic[$name]['rx']) : '—',
'tx' => isset($traffic[$name]) ? $this->humanBytes($traffic[$name]['tx']) : '—',
];
}
return $out;
}
private function parseHardening(string $sshd, string $active): string|array
{
$root = preg_match('/permitrootlogin\s+(\S+)/i', $sshd, $m) ? strtolower($m[1]) : 'yes';
$pwauth = preg_match('/passwordauthentication\s+(\S+)/i', $sshd, $m) ? strtolower($m[1]) : 'yes';
$svc = [];
foreach ($this->lines($active) as $line) {
[$name, $state] = array_pad(preg_split('/\s+/', trim($line)), 2, '');
$svc[$name] = $state;
}
$isActive = fn (string $u) => ($svc[$u] ?? '') === 'active';
$firewall = $isActive('ufw') || $isActive('nftables');
return [
['label' => 'SSH-Root-Login deaktiviert', 'detail' => "PermitRootLogin {$root}", 'status' => $root === 'no' ? 'online' : 'offline'],
['label' => 'SSH-Passwort-Login deaktiviert', 'detail' => "PasswordAuthentication {$pwauth}", 'status' => $pwauth === 'no' ? 'online' : 'warning'],
['label' => 'fail2ban aktiv', 'detail' => 'fail2ban.service · '.($svc['fail2ban'] ?? 'unbekannt'), 'status' => $isActive('fail2ban') ? 'online' : 'offline'],
['label' => 'Firewall aktiv', 'detail' => $firewall ? 'ufw/nftables · aktiv' : 'ufw/nftables · inaktiv', 'status' => $firewall ? 'online' : 'offline'],
['label' => 'Automatische Updates', 'detail' => 'unattended-upgrades · '.($svc['unattended-upgrades'] ?? 'unbekannt'), 'status' => $isActive('unattended-upgrades') ? 'online' : 'warning'],
];
}
private function parseKeys(string $body): array
{
$out = [];
foreach ($this->lines($body) as $line) {
// "3072 SHA256:xxxx comment with spaces (RSA)"
if (! preg_match('/^(\d+)\s+(\S+)\s+(.*?)\s*\(([^)]+)\)\s*$/', $line, $m)) {
continue;
}
$algo = strtolower($m[4]);
$out[] = [
'comment' => trim($m[3]) ?: '—',
'type' => $algo.'-'.$m[1],
'fingerprint' => $m[2],
];
}
return $out;
}
private function parseUnits(string $units, string $files): array
{
$enabled = [];
foreach ($this->lines($files) as $line) {
$p = preg_split('/\s+/', trim($line));
if (count($p) >= 2) {
$enabled[$p[0]] = in_array($p[1], ['enabled', 'static', 'enabled-runtime'], true);
}
}
$out = [];
foreach ($this->lines($units) as $line) {
// UNIT LOAD ACTIVE SUB DESCRIPTION
$p = preg_split('/\s+/', trim($line), 5);
if (count($p) < 5) {
continue;
}
[$unit, $load, $active, $sub, $desc] = $p;
if ($load === 'not-found') {
continue;
}
$out[] = [
'name' => $unit,
'status' => match ($active) {
'active' => 'online',
'failed' => 'offline',
'activating', 'deactivating', 'reloading' => 'warning',
default => 'offline', // inactive/dead
},
'sub' => $sub,
'enabled' => $enabled[$unit] ?? false,
'desc' => $desc,
];
}
return $out;
}
private function parseJournal(string $body): array
{
$out = [];
foreach ($this->lines($body) as $line) {
// "2026-06-12T21:05:26+02:00 host unit[pid]: message"
if (! preg_match('/^(\d{4}-\d\d-\d\dT[\d:]+)\S*\s+\S+\s+([^\s\[]+)(?:\[\d+\])?:\s+(.*)$/', $line, $m)) {
continue;
}
$text = $m[3];
$level = match (true) {
(bool) preg_match('/\b(fail|failed|error|fatal|denied|refused)\b/i', $text) => 'error',
(bool) preg_match('/\b(warn|warning|deprecat)\b/i', $text) => 'warn',
default => 'info',
};
$out[] = [
'time' => str_replace('T', ' ', substr($m[1], 0, 19)),
'unit' => $m[2],
'level' => $level,
'text' => $text,
];
}
if ($out === []) {
$out[] = ['time' => '—', 'unit' => 'journal', 'level' => 'warn', 'text' => 'Systemjournal benötigt erhöhte Rechte (Gruppe adm/systemd-journal) oder sudo.'];
}
return $out;
}
private function humanBytes(int $bytes): string
{
if ($bytes <= 0) {
return '0 B';
}
$units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'];
$i = (int) floor(log($bytes, 1024));
$i = min($i, count($units) - 1);
return round($bytes / (1024 ** $i), $i >= 3 ? 2 : 0).' '.$units[$i];
}
}