Notice when nobody is picking up the queue

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

app/Provisioning/Jobs/RecordProvisioningHeartbeat.php

@@ -0,0 +1,39 @@
+<?php
+
+namespace App\Provisioning\Jobs;
+
+use App\Support\Settings;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+
+/**
+ * A sign of life from the provisioning worker.
+ *
+ * The scheduler enqueues it, the worker executes it — so it proves exactly what
+ * the scheduler heartbeat (routes/console.php) does NOT prove: that someone is
+ * actually pulling jobs off the `provisioning` queue. The two fail
+ * independently, and "scheduler up, worker down" is the silent case: paid
+ * orders pile up on the queue, and nothing reports anything, because the
+ * scheduler itself is doing exactly what it is supposed to do.
+ *
+ * Written through Settings, not the cache: a heartbeat that vanishes on a Redis
+ * restart would report an outage that never happened, which is worse than no
+ * check at all — nobody trusts it after the second false alarm.
+ */
+class RecordProvisioningHeartbeat implements ShouldQueue
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public function __construct()
+    {
+        $this->onQueue('provisioning');
+    }
+
+    public function handle(): void
+    {
+        Settings::set('heartbeat.queue_provisioning', now()->toIso8601String());
+    }
+}

app/Support/Readiness.php

@@ -6,6 +6,7 @@ use App\Support\Readiness\BillingChecks;
 use App\Support\Readiness\Check;
 use App\Support\Readiness\DeliveryChecks;
 use App\Support\Readiness\OnboardingChecks;
+use App\Support\Readiness\OperationChecks;
 use App\Support\Readiness\ProvisioningChecks;
 
 /**
@@ -36,6 +37,7 @@ final class Readiness
             ...OnboardingChecks::all(),
             ...ProvisioningChecks::all(),
             ...DeliveryChecks::all(),
+            ...OperationChecks::all(),
         ];
     }
 

app/Support/Readiness/OperationChecks.php

@@ -0,0 +1,78 @@
+<?php
+
+namespace App\Support\Readiness;
+
+use App\Support\Settings;
+use Illuminate\Support\Carbon;
+
+/**
+ * Whether anything is actually running this installation, right now — as
+ * opposed to every other group in this file, which asks whether it is
+ * CONFIGURED to run at all.
+ *
+ * Queue-driven delivery has no error path for "nobody is picking up the
+ * queue": a paid order just sits there, forever, with nothing in the console
+ * to tell an operator apart from a quiet afternoon. Two heartbeats because the
+ * scheduler and the provisioning worker fail independently — the dangerous
+ * case is the scheduler running and enqueueing jobs that nobody executes, and
+ * a single heartbeat written by the scheduler alone would call that healthy.
+ */
+final class OperationChecks
+{
+    public const GROUP = 'operation';
+
+    /**
+     * A heartbeat older than this is as good as none: the process that was
+     * writing it has gone quiet. Five minutes is generous slack against a
+     * `->everyMinute()` schedule — a deploy restarting the container, a GC
+     * pause, one skipped tick — without also covering up a worker that has
+     * actually stopped.
+     */
+    private const STALE_AFTER_MINUTES = 5;
+
+    /** @return array<int, Check> */
+    public static function all(): array
+    {
+        return [
+            new Check(
+                key: 'operation.scheduler',
+                group: self::GROUP,
+                severity: Check::SEVERITY_BLOCKING,
+                label: __('readiness.operation.scheduler'),
+                breaks: __('readiness.operation.scheduler_breaks'),
+                tab: 'integrations',
+                satisfied: self::isFresh('heartbeat.scheduler'),
+            ),
+            new Check(
+                key: 'operation.queue_provisioning',
+                group: self::GROUP,
+                severity: Check::SEVERITY_BLOCKING,
+                label: __('readiness.operation.queue_provisioning'),
+                breaks: __('readiness.operation.queue_provisioning_breaks'),
+                tab: 'integrations',
+                // Written by RecordProvisioningHeartbeat, which the scheduler
+                // enqueues but only the worker actually runs. A fresh
+                // timestamp here is proof of a running worker, not just of a
+                // running scheduler.
+                satisfied: self::isFresh('heartbeat.queue_provisioning'),
+            ),
+        ];
+    }
+
+    private static function isFresh(string $key): bool
+    {
+        $recordedAt = Settings::get($key);
+
+        if (blank($recordedAt)) {
+            return false;
+        }
+
+        // Not diffInMinutes(): App\Models\Incident::durationMinutes() already
+        // documents that it is SIGNED, and `now()->diffInMinutes($recordedAt)`
+        // with $recordedAt in the past returns a NEGATIVE number — which
+        // would compare as "less than 5" for a heartbeat six hours stale, and
+        // report a dead worker as healthy. A direct threshold comparison has
+        // no sign to get backwards.
+        return Carbon::parse($recordedAt)->greaterThan(now()->subMinutes(self::STALE_AFTER_MINUTES));
+    }
+}

lang/de/readiness.php

@@ -62,4 +62,11 @@ return [
         'inbound_password' => 'Passwort für eingehende Mail',
         'inbound_password_breaks' => 'Ohne es liefert der Mail-Abruf jedes Mal eine leere Liste. Der Zeitplan läuft unauffällig weiter, aber keine Kundenantwort erreicht je die Inbox der Konsole, und nirgends erscheint ein Fehler.',
     ],
+
+    'operation' => [
+        'scheduler' => 'Zeitplaner läuft',
+        'scheduler_breaks' => 'Ohne ihn läuft kein einziger Zeitplan aus routes/console.php — nicht die Rechnungsarchivierung, nicht die Vertragsprüfung, nicht dieser Herzschlag selbst. Eine bezahlte Bestellung wartet auf TickProvisioning, das nie angestoßen wird, und die Konsole zeigt dieselbe Ruhe wie an einem Tag ohne Bestellungen.',
+        'queue_provisioning' => 'Bereitstellungs-Worker läuft',
+        'queue_provisioning_breaks' => 'Der Zeitplaner kann laufen und Aufträge auf die provisioning-Warteschlange stellen, ohne dass sie je jemand abholt. Eine bezahlte Bestellung bleibt dann liegen — kein Fehler, keine Meldung, einfach nichts, bis jemand von Hand nachsieht.',
+    ],
 ];

lang/en/readiness.php

@@ -62,4 +62,11 @@ return [
         'inbound_password' => 'Inbound mail password',
         'inbound_password_breaks' => 'Without it, fetching mail returns an empty list every time. The schedule keeps running quietly, but no customer reply ever reaches the console inbox, and no error appears anywhere.',
     ],
+
+    'operation' => [
+        'scheduler' => 'Scheduler is running',
+        'scheduler_breaks' => 'Without it, not a single schedule in routes/console.php runs — not invoice archiving, not the contract check, not this heartbeat itself. A paid order waits on TickProvisioning, which is never invoked, and the console shows the same calm as a day with no orders at all.',
+        'queue_provisioning' => 'Provisioning worker is running',
+        'queue_provisioning_breaks' => 'The scheduler can be running and putting jobs on the provisioning queue without anyone ever picking them up. A paid order then just sits there — no error, no notice, simply nothing, until someone happens to look.',
+    ],
 ];

routes/console.php

@@ -4,8 +4,10 @@ use App\Console\TickProvisioning;
 use App\Models\StripePendingEvent;
 use App\Provisioning\Jobs\CollectInstanceTraffic;
 use App\Provisioning\Jobs\PingHosts;
+use App\Provisioning\Jobs\RecordProvisioningHeartbeat;
 use App\Provisioning\Jobs\SyncMonitoringStatus;
 use App\Provisioning\Jobs\SyncVpnPeers;
+use App\Support\Settings;
 use Illuminate\Foundation\Inspiring;
 use Illuminate\Support\Facades\Artisan;
 use Illuminate\Support\Facades\Schedule;
@@ -20,6 +22,22 @@ Schedule::call(fn () => app(TickProvisioning::class)())
     ->name('provisioning-tick')
     ->withoutOverlapping();
 
+// Two signs of life for the readiness page (App\Support\Readiness\
+// OperationChecks). Written through Settings, not the cache: a heartbeat that
+// vanishes on a Redis restart would report an outage that never happened.
+//
+// Two of them, because the scheduler and the provisioning queue worker fail
+// independently. The dangerous case is the second one: the scheduler runs and
+// enqueues jobs (this one included) that nobody ever picks up. A heartbeat
+// only the scheduler writes would call that state healthy.
+Schedule::call(fn () => Settings::set('heartbeat.scheduler', now()->toIso8601String()))
+    ->everyMinute()
+    ->name('heartbeat-scheduler');
+
+Schedule::job(new RecordProvisioningHeartbeat)
+    ->everyMinute()
+    ->name('heartbeat-provisioning');
+
 // Refresh the VPN peer state (handshakes, traffic) so the console does not go
 // stale while nobody has it open. The job itself runs on the provisioning
 // queue — that worker owns wg0.

tests/Feature/Readiness/HeartbeatTest.php

@@ -0,0 +1,50 @@
+<?php
+
+use App\Provisioning\Jobs\RecordProvisioningHeartbeat;
+use App\Support\Readiness;
+use App\Support\Readiness\Check;
+use App\Support\Settings;
+
+function operationCheck(string $key): ?Check
+{
+    return collect(Readiness::all())->firstWhere('key', $key);
+}
+
+/**
+ * „Ohne Worker passiert schlicht nichts, ohne Fehlermeldung." Genau dieser
+ * Zustand war in der Konsole bisher nicht von „alles ruhig" zu unterscheiden.
+ *
+ * Zwei getrennte Herzschläge, weil Zeitplaner und Warteschlangen-Worker
+ * getrennt ausfallen: der Zeitplaner kann laufen und Aufträge einstellen, die
+ * niemand abholt.
+ */
+it('reports a missing heartbeat as not satisfied', function () {
+    expect(operationCheck('operation.scheduler')->satisfied)->toBeFalse();
+});
+
+it('accepts a fresh heartbeat', function () {
+    Settings::set('heartbeat.scheduler', now()->toIso8601String());
+
+    expect(operationCheck('operation.scheduler')->satisfied)->toBeTrue();
+});
+
+it('rejects a heartbeat older than five minutes', function () {
+    Settings::set('heartbeat.scheduler', now()->subMinutes(6)->toIso8601String());
+
+    expect(operationCheck('operation.scheduler')->satisfied)->toBeFalse();
+});
+
+it('proves the worker, not just the scheduler', function () {
+    // Der Zeitplaner läuft, der Bereitstellungs-Worker nicht: der eine
+    // Herzschlag ist frisch, der andere nicht.
+    Settings::set('heartbeat.scheduler', now()->toIso8601String());
+
+    expect(operationCheck('operation.scheduler')->satisfied)->toBeTrue();
+    expect(operationCheck('operation.queue_provisioning')->satisfied)->toBeFalse();
+});
+
+it('records the worker heartbeat when the job runs', function () {
+    (new RecordProvisioningHeartbeat)->handle();
+
+    expect(operationCheck('operation.queue_provisioning')->satisfied)->toBeTrue();
+});