106 lines
3.9 KiB
PHP
106 lines
3.9 KiB
PHP
<?php
|
|
|
|
namespace App\Provisioning\Jobs;
|
|
|
|
use App\Models\InstanceMetric;
|
|
use App\Models\MonitoringTarget;
|
|
use App\Services\Monitoring\MonitoringClient;
|
|
use Illuminate\Bus\Queueable;
|
|
use Illuminate\Contracts\Queue\ShouldQueue;
|
|
use Illuminate\Foundation\Bus\Dispatchable;
|
|
use Illuminate\Queue\InteractsWithQueue;
|
|
use Illuminate\Queue\SerializesModels;
|
|
use Illuminate\Support\Facades\Log;
|
|
use Throwable;
|
|
|
|
/**
|
|
* Asks the monitoring service how each target is actually doing.
|
|
*
|
|
* Without this the status column was a fossil: written as 'up' when the
|
|
* instance was provisioned and never touched again. The console's notices and
|
|
* the public status page both read it, so an outage would have been reported as
|
|
* healthy indefinitely — on a page whose entire purpose is to be believed.
|
|
*
|
|
* A target the service cannot answer for becomes 'unknown', never 'up'. The
|
|
* absence of an answer is not a healthy answer, and checked_at is only advanced
|
|
* when the service really replied — so a page can tell a fresh verdict from a
|
|
* stale one.
|
|
*/
|
|
class SyncMonitoringStatus implements ShouldQueue
|
|
{
|
|
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
|
|
|
/**
|
|
* One tick of the availability figure the customer panel shows.
|
|
*
|
|
* Written per day so thirty days is thirty rows, and so a gap in our own
|
|
* monitoring shows up as a day with fewer checks rather than as downtime
|
|
* that never happened.
|
|
*/
|
|
private function countCheck(MonitoringTarget $target, ?bool $healthy): void
|
|
{
|
|
if ($healthy === null || $target->instance_id === null) {
|
|
return;
|
|
}
|
|
|
|
$metric = InstanceMetric::query()->firstOrCreate(
|
|
['instance_id' => $target->instance_id, 'day' => now()->toDateString()],
|
|
);
|
|
|
|
$metric->increment('checks_total');
|
|
|
|
if ($healthy) {
|
|
$metric->increment('checks_ok');
|
|
}
|
|
}
|
|
|
|
public function handle(MonitoringClient $monitoring): void
|
|
{
|
|
MonitoringTarget::query()
|
|
->whereHas('instance', fn ($instance) => $instance->where('status', 'active'))
|
|
->chunkById(100, function ($targets) use ($monitoring) {
|
|
foreach ($targets as $target) {
|
|
$this->refresh($target, $monitoring);
|
|
}
|
|
});
|
|
}
|
|
|
|
private function refresh(MonitoringTarget $target, MonitoringClient $monitoring): void
|
|
{
|
|
try {
|
|
// health(), not isHealthy(): the boolean answer folds "no monitoring
|
|
// configured" into true and "monitor unreachable" into false, and
|
|
// recording either as a measurement publishes a fleet-wide lie.
|
|
$healthy = $monitoring->health($target->external_id);
|
|
} catch (Throwable $e) {
|
|
// The monitoring service being unreachable says nothing about the
|
|
// instance. Recording 'down' here would raise an alarm about the
|
|
// wrong system; recording 'up' would hide a real one. Leave the
|
|
// previous verdict and let it go stale.
|
|
Log::warning('Monitoring did not answer for a target', [
|
|
'external_id' => $target->external_id,
|
|
'exception' => $e->getMessage(),
|
|
]);
|
|
|
|
return;
|
|
}
|
|
|
|
// Counted before the early return below, so a day's denominator grows
|
|
// only when a real answer arrived. An unanswered check is not a failed
|
|
// one — folding the two together turns a monitoring outage into the
|
|
// customer's outage, on the one figure they are asked to trust.
|
|
$this->countCheck($target, $healthy);
|
|
|
|
if ($healthy === null) {
|
|
// No answer. The previous verdict stays and is allowed to go stale,
|
|
// which readers show as "not monitored" rather than as health.
|
|
return;
|
|
}
|
|
|
|
$target->update([
|
|
'status' => $healthy ? 'up' : 'down',
|
|
'checked_at' => now(),
|
|
]);
|
|
}
|
|
}
|