80 lines
3.1 KiB
PHP
80 lines
3.1 KiB
PHP
<?php
|
|
|
|
namespace App\Services;
|
|
|
|
use App\Models\AlertIncident;
|
|
use App\Models\AlertRule;
|
|
use App\Models\Server;
|
|
use Illuminate\Database\QueryException;
|
|
|
|
/**
|
|
* The alert state machine. Given a server's fresh reading, opens a firing incident on a new breach
|
|
* (and notifies) and resolves an open incident on recovery (and notifies). Idempotent: a sustained
|
|
* breach with an already-open incident does nothing, so one breach = one alert, not a flood.
|
|
*/
|
|
class AlertEvaluator
|
|
{
|
|
public function __construct(private AlertNotifier $notifier) {}
|
|
|
|
/**
|
|
* @param array<string, int|string> $metrics cpu/mem/disk/load ints (empty when offline)
|
|
* @param string $status the server's fresh status (online|warning|offline)
|
|
*/
|
|
public function evaluate(Server $server, array $metrics, string $status): void
|
|
{
|
|
$rules = AlertRule::where('enabled', true)->get();
|
|
|
|
foreach ($rules as $rule) {
|
|
if (! $rule->targets($server)) {
|
|
continue;
|
|
}
|
|
|
|
// Numeric metrics need a fresh reading; skip them only when the server is truly OFFLINE
|
|
// (stale numbers). A reachable "warning" server still has fresh metrics, so it evaluates.
|
|
if ($rule->metric !== 'offline' && $status === 'offline') {
|
|
continue;
|
|
}
|
|
|
|
$this->transition($rule, $server, $rule->breached($metrics, $status), $rule->reading($metrics));
|
|
}
|
|
}
|
|
|
|
private function transition(AlertRule $rule, Server $server, bool $breached, float $reading): void
|
|
{
|
|
$open = AlertIncident::where('alert_rule_id', $rule->id)
|
|
->where('server_id', $server->id)
|
|
->where('state', 'firing')
|
|
->first();
|
|
|
|
if ($breached && ! $open) {
|
|
try {
|
|
$incident = AlertIncident::create([
|
|
'alert_rule_id' => $rule->id,
|
|
'server_id' => $server->id,
|
|
'state' => 'firing',
|
|
'value' => $reading,
|
|
// Unique while firing → a concurrent tick that also tries to open this incident
|
|
// hits a unique violation instead of creating a duplicate (see the migration).
|
|
'firing_key' => $rule->id.':'.$server->id,
|
|
'started_at' => now(),
|
|
]);
|
|
} catch (QueryException) {
|
|
return; // another tick opened it first — dedup, no duplicate incident/notification
|
|
}
|
|
AlertIncident::forgetFiringCount(); // keep the sidebar badge in step with this new incident
|
|
$this->notifier->notify($incident, resolved: false);
|
|
|
|
return;
|
|
}
|
|
|
|
if (! $breached && $open) {
|
|
// Clear firing_key on resolve so a later re-breach can claim the unique key again.
|
|
$open->update(['state' => 'resolved', 'firing_key' => null, 'resolved_at' => now()]);
|
|
AlertIncident::forgetFiringCount();
|
|
$this->notifier->notify($open, resolved: true);
|
|
}
|
|
|
|
// else: still firing with an open incident, or fine with none → dedup, no-op.
|
|
}
|
|
}
|