75 lines
2.8 KiB
PHP
75 lines
2.8 KiB
PHP
<?php
|
|
|
|
namespace App\Provisioning\Jobs;
|
|
|
|
use App\Models\MonitoringTarget;
|
|
use App\Services\Monitoring\MonitoringClient;
|
|
use Illuminate\Bus\Queueable;
|
|
use Illuminate\Contracts\Queue\ShouldQueue;
|
|
use Illuminate\Foundation\Bus\Dispatchable;
|
|
use Illuminate\Queue\InteractsWithQueue;
|
|
use Illuminate\Queue\SerializesModels;
|
|
use Illuminate\Support\Facades\Log;
|
|
use Throwable;
|
|
|
|
/**
|
|
* Asks the monitoring service how each target is actually doing.
|
|
*
|
|
* Without this the status column was a fossil: written as 'up' when the
|
|
* instance was provisioned and never touched again. The console's notices and
|
|
* the public status page both read it, so an outage would have been reported as
|
|
* healthy indefinitely — on a page whose entire purpose is to be believed.
|
|
*
|
|
* A target the service cannot answer for becomes 'unknown', never 'up'. The
|
|
* absence of an answer is not a healthy answer, and checked_at is only advanced
|
|
* when the service really replied — so a page can tell a fresh verdict from a
|
|
* stale one.
|
|
*/
|
|
class SyncMonitoringStatus implements ShouldQueue
|
|
{
|
|
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
|
|
|
public function handle(MonitoringClient $monitoring): void
|
|
{
|
|
MonitoringTarget::query()
|
|
->whereHas('instance', fn ($instance) => $instance->where('status', 'active'))
|
|
->chunkById(100, function ($targets) use ($monitoring) {
|
|
foreach ($targets as $target) {
|
|
$this->refresh($target, $monitoring);
|
|
}
|
|
});
|
|
}
|
|
|
|
private function refresh(MonitoringTarget $target, MonitoringClient $monitoring): void
|
|
{
|
|
try {
|
|
// health(), not isHealthy(): the boolean answer folds "no monitoring
|
|
// configured" into true and "monitor unreachable" into false, and
|
|
// recording either as a measurement publishes a fleet-wide lie.
|
|
$healthy = $monitoring->health($target->external_id);
|
|
} catch (Throwable $e) {
|
|
// The monitoring service being unreachable says nothing about the
|
|
// instance. Recording 'down' here would raise an alarm about the
|
|
// wrong system; recording 'up' would hide a real one. Leave the
|
|
// previous verdict and let it go stale.
|
|
Log::warning('Monitoring did not answer for a target', [
|
|
'external_id' => $target->external_id,
|
|
'exception' => $e->getMessage(),
|
|
]);
|
|
|
|
return;
|
|
}
|
|
|
|
if ($healthy === null) {
|
|
// No answer. The previous verdict stays and is allowed to go stale,
|
|
// which readers show as "not monitored" rather than as health.
|
|
return;
|
|
}
|
|
|
|
$target->update([
|
|
'status' => $healthy ? 'up' : 'down',
|
|
'checked_at' => now(),
|
|
]);
|
|
}
|
|
}
|