196 lines
8.9 KiB
PHP
196 lines
8.9 KiB
PHP
<?php
|
|
|
|
namespace Tests\Feature;
|
|
|
|
use App\Models\AlertIncident;
|
|
use App\Models\AlertRule;
|
|
use App\Models\Server;
|
|
use App\Models\ServerGroup;
|
|
use App\Services\AlertEvaluator;
|
|
use App\Services\AlertNotifier;
|
|
use Illuminate\Database\QueryException;
|
|
use Illuminate\Foundation\Testing\RefreshDatabase;
|
|
use Mockery;
|
|
use Tests\TestCase;
|
|
|
|
class AlertEvaluatorTest extends TestCase
|
|
{
|
|
use RefreshDatabase;
|
|
|
|
protected function tearDown(): void
|
|
{
|
|
Mockery::close();
|
|
parent::tearDown();
|
|
}
|
|
|
|
/** Evaluator with a notifier spy so no real mail/http is sent; returns [$evaluator, $notifier]. */
|
|
private function make(): array
|
|
{
|
|
$notifier = Mockery::mock(AlertNotifier::class);
|
|
|
|
return [new AlertEvaluator($notifier), $notifier];
|
|
}
|
|
|
|
private function server(string $ip = '10.0.0.1'): Server
|
|
{
|
|
return Server::create(['name' => 'box', 'ip' => $ip, 'ssh_port' => 22, 'status' => 'online']);
|
|
}
|
|
|
|
public function test_a_numeric_breach_opens_a_firing_incident_and_notifies(): void
|
|
{
|
|
[$eval, $notifier] = $this->make();
|
|
$notifier->shouldReceive('notify')->once()->with(Mockery::type(AlertIncident::class), false);
|
|
AlertRule::create(['name' => 'CPU', 'metric' => 'cpu', 'comparator' => 'gt', 'threshold' => 80, 'scope_type' => 'all', 'enabled' => true]);
|
|
$server = $this->server();
|
|
|
|
$eval->evaluate($server, ['cpu' => 95, 'mem' => 10, 'disk' => 10, 'load' => 1], 'online');
|
|
|
|
$this->assertDatabaseHas('alert_incidents', ['server_id' => $server->id, 'state' => 'firing', 'value' => 95]);
|
|
}
|
|
|
|
public function test_a_sustained_breach_does_not_open_a_second_incident(): void
|
|
{
|
|
[$eval, $notifier] = $this->make();
|
|
$notifier->shouldReceive('notify')->once(); // fired once, deduped after
|
|
AlertRule::create(['name' => 'CPU', 'metric' => 'cpu', 'comparator' => 'gt', 'threshold' => 80, 'scope_type' => 'all', 'enabled' => true]);
|
|
$server = $this->server();
|
|
|
|
$eval->evaluate($server, ['cpu' => 95], 'online');
|
|
$eval->evaluate($server, ['cpu' => 96], 'online'); // still breached → dedup
|
|
|
|
$this->assertSame(1, AlertIncident::where('state', 'firing')->count());
|
|
}
|
|
|
|
public function test_recovery_resolves_the_incident_and_notifies(): void
|
|
{
|
|
[$eval, $notifier] = $this->make();
|
|
$notifier->shouldReceive('notify')->once()->with(Mockery::type(AlertIncident::class), false);
|
|
$notifier->shouldReceive('notify')->once()->with(Mockery::type(AlertIncident::class), true);
|
|
AlertRule::create(['name' => 'CPU', 'metric' => 'cpu', 'comparator' => 'gt', 'threshold' => 80, 'scope_type' => 'all', 'enabled' => true]);
|
|
$server = $this->server();
|
|
|
|
$eval->evaluate($server, ['cpu' => 95], 'online'); // fire
|
|
$eval->evaluate($server, ['cpu' => 20], 'online'); // recover
|
|
|
|
$this->assertSame(0, AlertIncident::where('state', 'firing')->count());
|
|
$this->assertDatabaseHas('alert_incidents', ['state' => 'resolved']);
|
|
}
|
|
|
|
public function test_lt_comparator_fires_below_the_threshold(): void
|
|
{
|
|
[$eval, $notifier] = $this->make();
|
|
$notifier->shouldReceive('notify')->once();
|
|
AlertRule::create(['name' => 'Disk free low', 'metric' => 'disk', 'comparator' => 'lt', 'threshold' => 10, 'scope_type' => 'all', 'enabled' => true]);
|
|
$server = $this->server();
|
|
|
|
$eval->evaluate($server, ['disk' => 5], 'online');
|
|
|
|
$this->assertDatabaseHas('alert_incidents', ['state' => 'firing', 'value' => 5]);
|
|
}
|
|
|
|
public function test_offline_rule_fires_when_not_online_and_skips_numeric(): void
|
|
{
|
|
[$eval, $notifier] = $this->make();
|
|
$notifier->shouldReceive('notify')->once()->with(Mockery::type(AlertIncident::class), false);
|
|
AlertRule::create(['name' => 'Down', 'metric' => 'offline', 'scope_type' => 'all', 'enabled' => true]);
|
|
AlertRule::create(['name' => 'CPU', 'metric' => 'cpu', 'comparator' => 'gt', 'threshold' => 1, 'scope_type' => 'all', 'enabled' => true]);
|
|
$server = $this->server();
|
|
|
|
// Offline poll: stale metrics, status offline → only the offline rule fires, cpu is skipped.
|
|
$eval->evaluate($server, [], 'offline');
|
|
|
|
$this->assertSame(1, AlertIncident::where('state', 'firing')->count());
|
|
$this->assertDatabaseHas('alert_incidents', ['alert_rule_id' => AlertRule::where('metric', 'offline')->first()->id]);
|
|
}
|
|
|
|
public function test_offline_incident_resolves_when_the_server_comes_back(): void
|
|
{
|
|
[$eval, $notifier] = $this->make();
|
|
$notifier->shouldReceive('notify')->twice();
|
|
AlertRule::create(['name' => 'Down', 'metric' => 'offline', 'scope_type' => 'all', 'enabled' => true]);
|
|
$server = $this->server();
|
|
|
|
$eval->evaluate($server, [], 'offline'); // fire
|
|
$eval->evaluate($server, ['cpu' => 5], 'online'); // back online → resolve
|
|
|
|
$this->assertSame(0, AlertIncident::where('state', 'firing')->count());
|
|
}
|
|
|
|
public function test_a_fractional_load_breach_fires_and_is_not_truncated(): void
|
|
{
|
|
[$eval, $notifier] = $this->make();
|
|
$notifier->shouldReceive('notify')->once();
|
|
// Regression: load 1.5 with threshold 1 used to truncate to `1 > 1` = false and never fire.
|
|
AlertRule::create(['name' => 'Load', 'metric' => 'load', 'comparator' => 'gt', 'threshold' => 1, 'scope_type' => 'all', 'enabled' => true]);
|
|
|
|
$eval->evaluate($this->server(), ['cpu' => 5, 'load' => 1.5], 'online');
|
|
|
|
$this->assertDatabaseHas('alert_incidents', ['state' => 'firing', 'value' => 1.5]);
|
|
}
|
|
|
|
public function test_firing_key_is_a_unique_db_invariant(): void
|
|
{
|
|
$rule = AlertRule::create(['name' => 'CPU', 'metric' => 'cpu', 'comparator' => 'gt', 'threshold' => 80, 'scope_type' => 'all', 'enabled' => true]);
|
|
$server = $this->server();
|
|
$key = $rule->id.':'.$server->id;
|
|
AlertIncident::create(['alert_rule_id' => $rule->id, 'server_id' => $server->id, 'state' => 'firing', 'value' => 90, 'firing_key' => $key, 'started_at' => now()]);
|
|
|
|
// A concurrent tick opening the same firing incident hits the unique index → no duplicate.
|
|
$this->expectException(QueryException::class);
|
|
AlertIncident::create(['alert_rule_id' => $rule->id, 'server_id' => $server->id, 'state' => 'firing', 'value' => 91, 'firing_key' => $key, 'started_at' => now()]);
|
|
}
|
|
|
|
public function test_a_resolved_incident_frees_the_firing_key_for_a_re_breach(): void
|
|
{
|
|
[$eval, $notifier] = $this->make();
|
|
$notifier->shouldReceive('notify')->times(3); // fire, resolve, fire again
|
|
AlertRule::create(['name' => 'CPU', 'metric' => 'cpu', 'comparator' => 'gt', 'threshold' => 80, 'scope_type' => 'all', 'enabled' => true]);
|
|
$server = $this->server();
|
|
|
|
$eval->evaluate($server, ['cpu' => 95], 'online'); // fire
|
|
$eval->evaluate($server, ['cpu' => 10], 'online'); // resolve → firing_key nulled
|
|
$eval->evaluate($server, ['cpu' => 95], 'online'); // re-breach can claim the key again
|
|
|
|
$this->assertSame(1, AlertIncident::where('state', 'firing')->count());
|
|
$this->assertSame(2, AlertIncident::count()); // one resolved + one new firing
|
|
}
|
|
|
|
public function test_a_disabled_rule_never_fires(): void
|
|
{
|
|
[$eval, $notifier] = $this->make();
|
|
$notifier->shouldReceive('notify')->never();
|
|
AlertRule::create(['name' => 'CPU', 'metric' => 'cpu', 'comparator' => 'gt', 'threshold' => 1, 'scope_type' => 'all', 'enabled' => false]);
|
|
|
|
$eval->evaluate($this->server(), ['cpu' => 99], 'online');
|
|
|
|
$this->assertDatabaseCount('alert_incidents', 0);
|
|
}
|
|
|
|
public function test_scope_server_only_targets_that_server(): void
|
|
{
|
|
[$eval, $notifier] = $this->make();
|
|
$notifier->shouldReceive('notify')->never();
|
|
$target = $this->server('10.0.0.1');
|
|
$other = $this->server('10.0.0.2');
|
|
AlertRule::create(['name' => 'CPU', 'metric' => 'cpu', 'comparator' => 'gt', 'threshold' => 50, 'scope_type' => 'server', 'scope_id' => $target->id, 'enabled' => true]);
|
|
|
|
$eval->evaluate($other, ['cpu' => 99], 'online'); // not the targeted server → nothing
|
|
|
|
$this->assertDatabaseCount('alert_incidents', 0);
|
|
}
|
|
|
|
public function test_scope_group_targets_only_members(): void
|
|
{
|
|
[$eval, $notifier] = $this->make();
|
|
$notifier->shouldReceive('notify')->once();
|
|
$group = ServerGroup::create(['name' => 'prod', 'color' => 'online']);
|
|
$member = $this->server('10.0.0.1');
|
|
$group->servers()->attach($member->id);
|
|
AlertRule::create(['name' => 'CPU', 'metric' => 'cpu', 'comparator' => 'gt', 'threshold' => 50, 'scope_type' => 'group', 'scope_id' => $group->id, 'enabled' => true]);
|
|
|
|
$eval->evaluate($member, ['cpu' => 99], 'online');
|
|
|
|
$this->assertDatabaseHas('alert_incidents', ['server_id' => $member->id, 'state' => 'firing']);
|
|
}
|
|
}
|