79 lines
3.1 KiB
PHP
79 lines
3.1 KiB
PHP
<?php
|
|
|
|
namespace App\Support\Readiness;
|
|
|
|
use App\Support\Settings;
|
|
use Illuminate\Support\Carbon;
|
|
|
|
/**
|
|
* Whether anything is actually running this installation, right now — as
|
|
* opposed to every other group in this file, which asks whether it is
|
|
* CONFIGURED to run at all.
|
|
*
|
|
* Queue-driven delivery has no error path for "nobody is picking up the
|
|
* queue": a paid order just sits there, forever, with nothing in the console
|
|
* to tell an operator apart from a quiet afternoon. Two heartbeats because the
|
|
* scheduler and the provisioning worker fail independently — the dangerous
|
|
* case is the scheduler running and enqueueing jobs that nobody executes, and
|
|
* a single heartbeat written by the scheduler alone would call that healthy.
|
|
*/
|
|
final class OperationChecks
|
|
{
|
|
public const GROUP = 'operation';
|
|
|
|
/**
|
|
* A heartbeat older than this is as good as none: the process that was
|
|
* writing it has gone quiet. Five minutes is generous slack against a
|
|
* `->everyMinute()` schedule — a deploy restarting the container, a GC
|
|
* pause, one skipped tick — without also covering up a worker that has
|
|
* actually stopped.
|
|
*/
|
|
private const STALE_AFTER_MINUTES = 5;
|
|
|
|
/** @return array<int, Check> */
|
|
public static function all(): array
|
|
{
|
|
return [
|
|
new Check(
|
|
key: 'operation.scheduler',
|
|
group: self::GROUP,
|
|
severity: Check::SEVERITY_BLOCKING,
|
|
label: __('readiness.operation.scheduler'),
|
|
breaks: __('readiness.operation.scheduler_breaks'),
|
|
tab: 'integrations',
|
|
satisfied: self::isFresh('heartbeat.scheduler'),
|
|
),
|
|
new Check(
|
|
key: 'operation.queue_provisioning',
|
|
group: self::GROUP,
|
|
severity: Check::SEVERITY_BLOCKING,
|
|
label: __('readiness.operation.queue_provisioning'),
|
|
breaks: __('readiness.operation.queue_provisioning_breaks'),
|
|
tab: 'integrations',
|
|
// Written by RecordProvisioningHeartbeat, which the scheduler
|
|
// enqueues but only the worker actually runs. A fresh
|
|
// timestamp here is proof of a running worker, not just of a
|
|
// running scheduler.
|
|
satisfied: self::isFresh('heartbeat.queue_provisioning'),
|
|
),
|
|
];
|
|
}
|
|
|
|
private static function isFresh(string $key): bool
|
|
{
|
|
$recordedAt = Settings::get($key);
|
|
|
|
if (blank($recordedAt)) {
|
|
return false;
|
|
}
|
|
|
|
// Not diffInMinutes(): App\Models\Incident::durationMinutes() already
|
|
// documents that it is SIGNED, and `now()->diffInMinutes($recordedAt)`
|
|
// with $recordedAt in the past returns a NEGATIVE number — which
|
|
// would compare as "less than 5" for a heartbeat six hours stale, and
|
|
// report a dead worker as healthy. A direct threshold comparison has
|
|
// no sign to get backwards.
|
|
return Carbon::parse($recordedAt)->greaterThan(now()->subMinutes(self::STALE_AFTER_MINUTES));
|
|
}
|
|
}
|