From 6c92aa5dd7e2839b7a1268c8dfd3bfc00d01a6ad Mon Sep 17 00:00:00 2001 From: nexxo Date: Wed, 29 Jul 2026 15:16:48 +0200 Subject: [PATCH] Let an incident be deleted, and start measuring whether the hosts answer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three things, all from trying to actually use the incident page. ── Where the later entries go ─────────────────────────────────────────────── The form only ever creates the first one, and nothing said so — so it looked as though the stage had to be chosen up front and there was only a dropdown for the impact. The field now says what it becomes ("wird untersucht") and where the rest are added: on the incident itself, once it exists. ── Deleting ──────────────────────────────────────────────────────────────── Asked for in order to test the thing at all, and right. A whole incident can now be removed, with its entries, behind a confirmation in the product's own dialog (R23). Deliberately not the same as editing an entry, which there is still no way to do: an update is a statement made at a time, and rewriting one afterwards is what the record exists to prevent. Removing the whole incident is a different act — it is how a test entry, or one published against the wrong service, is taken back, and it leaves nothing half-standing behind. A test holds that distinction so the delete button does not quietly become an exception to it. ── The hosts, which nothing was watching ──────────────────────────────────── `hosts.last_seen_at` drove the health dot in the console and was written exactly once, at onboarding — RegisterCapacity stamped it and that was the only write in the codebase. So every host read "offline" thirty minutes after it was added, permanently, and the dot meant nothing. It is also why host reachability could not appear on the status page: it was never being measured. PingHosts asks every host's API the cheapest question it answers, once a minute, and stamps the timestamp on success. Failure writes nothing on purpose: absence IS the signal, and a second column counting failures would be a second version of the same fact. The status page has a fifth component for it. Counts only — never a host name, never an address; the page is world-readable and the estate is not public information. A host between five and thirty minutes silent reads as degraded, not as healthy: "stale" is one of three answers healthState() gives and it is not the good one. Co-Authored-By: Claude Opus 5 --- VERSION | 2 +- app/Livewire/Admin/ConfirmDeleteIncident.php | 40 ++++++++ app/Livewire/Admin/Incidents.php | 32 +++++++ app/Provisioning/Jobs/PingHosts.php | 68 ++++++++++++++ app/Services/Status/ServiceHealth.php | 44 ++++++++- lang/de/admin_incidents.php | 6 ++ lang/de/status.php | 3 + lang/en/admin_incidents.php | 6 ++ lang/en/status.php | 3 + .../admin/confirm-delete-incident.blade.php | 9 ++ .../views/livewire/admin/incidents.blade.php | 20 +++- routes/console.php | 10 ++ tests/Feature/Admin/IncidentDeleteTest.php | 91 +++++++++++++++++++ tests/Feature/StatusHistoryTest.php | 32 +++++++ 14 files changed, 361 insertions(+), 5 deletions(-) create mode 100644 app/Livewire/Admin/ConfirmDeleteIncident.php create mode 100644 app/Provisioning/Jobs/PingHosts.php create mode 100644 resources/views/livewire/admin/confirm-delete-incident.blade.php create mode 100644 tests/Feature/Admin/IncidentDeleteTest.php diff --git a/VERSION b/VERSION index 90a7f60..7962dcf 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.3.12 +1.3.13 diff --git a/app/Livewire/Admin/ConfirmDeleteIncident.php b/app/Livewire/Admin/ConfirmDeleteIncident.php new file mode 100644 index 0000000..3f1afd4 --- /dev/null +++ b/app/Livewire/Admin/ConfirmDeleteIncident.php @@ -0,0 +1,40 @@ +where('uuid', $uuid)->firstOrFail(); + + $this->uuid = $incident->uuid; + $this->title = $incident->title; + } + + public function confirm(): void + { + $this->dispatch('incident-delete-confirmed', uuid: $this->uuid); + $this->closeModal(); + } + + public function render() + { + return view('livewire.admin.confirm-delete-incident'); + } +} diff --git a/app/Livewire/Admin/Incidents.php b/app/Livewire/Admin/Incidents.php index 3ccb58a..1b715db 100644 --- a/app/Livewire/Admin/Incidents.php +++ b/app/Livewire/Admin/Incidents.php @@ -9,6 +9,7 @@ use App\Support\LocalTime; use Illuminate\Support\Facades\Auth; use Illuminate\Support\Facades\DB; use Livewire\Attributes\Layout; +use Livewire\Attributes\On; use Livewire\Attributes\Validate; use Livewire\Component; @@ -83,6 +84,9 @@ class Incidents extends Component 'created_by' => Auth::guard('operator')->user()?->email, ]); + // The first entry is always "wird untersucht" — that is what a + // first report is. The form says so, so nobody looks for a stage + // selector that is not there. $incident->updates()->create([ 'status' => 'investigating', 'body' => $this->firstUpdate, @@ -139,6 +143,34 @@ class Incidents extends Component $this->dispatch('notify', message: __('admin_incidents.update_posted')); } + /** + * Remove an incident entirely — the whole record, updates and all. + * + * Deliberately different from editing an update, which there is still no + * way to do: an update is a statement made at a time, and rewriting one + * afterwards is what this record exists to prevent. Removing the WHOLE + * incident is a different act. It is how a test entry, or one published by + * mistake against the wrong service, is taken back — and taking it back + * leaves nothing half-standing behind. + * + * Owner-level, and confirmed in a modal (R23), because on a page whose + * whole value is that it is believed, deleting the record is the one action + * that cannot be undone by writing more. + */ + #[On('incident-delete-confirmed')] + public function delete(string $uuid): void + { + $this->authorize('maintenance.manage'); + + Incident::query()->where('uuid', $uuid)->first()?->delete(); + + if ($this->updatingUuid === $uuid) { + $this->cancelUpdate(); + } + + $this->dispatch('notify', message: __('admin_incidents.deleted')); + } + /** @return array */ public function componentOptions(): array { diff --git a/app/Provisioning/Jobs/PingHosts.php b/app/Provisioning/Jobs/PingHosts.php new file mode 100644 index 0000000..c264ad2 --- /dev/null +++ b/app/Provisioning/Jobs/PingHosts.php @@ -0,0 +1,68 @@ +onQueue('provisioning'); + } + + public function handle(ProxmoxClient $proxmox): void + { + $hosts = Host::query() + // Hosts still being onboarded, or already retired, are not a + // statement about the platform's health. + ->whereIn('status', ['active', 'error']) + ->get(); + + foreach ($hosts as $host) { + try { + $proxmox->forHost($host)->nodeStatus($host->node); + + // Only the timestamp. Whether a host is "active" or "error" is + // an operator's decision and a provisioning outcome — a + // heartbeat must not quietly promote a host somebody has + // deliberately taken out of service. + $host->forceFill(['last_seen_at' => now()])->save(); + } catch (Throwable) { + // Deliberately silent, and deliberately not writing anything. + // Absence IS the signal: healthState() reads the age of the + // last successful answer, so a failure is recorded by the + // timestamp not moving. Writing a failure count here would be a + // second version of the same fact. + } + } + } +} diff --git a/app/Services/Status/ServiceHealth.php b/app/Services/Status/ServiceHealth.php index eea6225..266ecc3 100644 --- a/app/Services/Status/ServiceHealth.php +++ b/app/Services/Status/ServiceHealth.php @@ -2,6 +2,7 @@ namespace App\Services\Status; +use App\Models\Host; use App\Models\Instance; use App\Models\MonitoringTarget; use App\Models\ProvisioningRun; @@ -27,7 +28,7 @@ use Illuminate\Support\Carbon; class ServiceHealth { /** The components, in the order they appear. */ - public const COMPONENTS = ['portal', 'instances', 'provisioning', 'backups']; + public const COMPONENTS = ['portal', 'hosts', 'instances', 'provisioning', 'backups']; /** A backup older than this stops counting as current. */ private const BACKUP_STALE_AFTER_HOURS = 48; @@ -50,6 +51,7 @@ class ServiceHealth { return [ $this->portal(), + $this->hosts(), $this->instances(), $this->provisioning(), $this->backups(), @@ -87,6 +89,46 @@ class ServiceHealth return ['key' => 'portal', 'state' => 'operational', 'detail' => null]; } + /** + * The machines the instances run on. + * + * From the heartbeat PingHosts writes, which is the same question + * provisioning asks: can we reach this host's API with the credentials we + * hold. A host that fails that is down as far as this platform is + * concerned, whatever a separate ICMP probe would say about the wire. + * + * Counts only — never a host name, never an address. This page is + * world-readable and the estate is not public information. + * + * @return array + */ + private function hosts(): array + { + $hosts = Host::query()->where('status', 'active')->get(); + $total = $hosts->count(); + + if ($total === 0) { + return ['key' => 'hosts', 'state' => 'operational', 'detail' => null]; + } + + // healthState() has three answers and they are not two. "stale" is a + // host that has not answered for between five and thirty minutes: not + // yet an outage, and absolutely not health. + $reachable = $hosts->filter(fn (Host $host) => $host->healthState() === 'online')->count(); + $offline = $hosts->filter(fn (Host $host) => $host->healthState() === 'offline')->count(); + + return [ + 'key' => 'hosts', + 'state' => match (true) { + $offline >= $total => 'down', + $offline > 0 => 'degraded', + $reachable < $total => 'degraded', + default => 'operational', + }, + 'detail' => $reachable < $total ? ['down' => $total - $reachable, 'total' => $total] : null, + ]; + } + /** * Customer instances, from what monitoring last saw. * diff --git a/lang/de/admin_incidents.php b/lang/de/admin_incidents.php index 24b3072..4a3b997 100644 --- a/lang/de/admin_incidents.php +++ b/lang/de/admin_incidents.php @@ -29,4 +29,10 @@ return [ 'appears_on' => 'Was Sie hier eintragen, erscheint sofort öffentlich auf der Statusseite. Die vier Dienste darüber messen sich selbst — hier steht das, was keine Messung wissen kann.', 'open_status' => 'Statusseite öffnen', + + 'delete' => 'Löschen', + 'delete_title' => 'Störung löschen?', + 'delete_body' => '„:title“ wird samt allen Einträgen entfernt und verschwindet sofort von der Statusseite. Das lässt sich nicht rückgängig machen — eine Korrektur ist sonst immer ein weiterer Eintrag.', + 'deleted' => 'Störung gelöscht.', + 'first_update_hint' => 'Erscheint als erster Eintrag mit dem Stand „Wird untersucht“. Alles Weitere — Ursache gefunden, behoben — tragen Sie danach links bei der Störung nach.', ]; diff --git a/lang/de/status.php b/lang/de/status.php index cec1093..e6d00e5 100644 --- a/lang/de/status.php +++ b/lang/de/status.php @@ -20,6 +20,7 @@ return [ 'component' => [ 'portal' => 'Anmeldung & Kundenbereich', + 'hosts' => 'Rechenzentrum', 'instances' => 'Ihre Cloud', 'provisioning' => 'Neue Bestellungen', 'backups' => 'Sicherungen', @@ -31,12 +32,14 @@ return [ // actually asking about. 'about' => [ 'portal' => 'Ob Anmeldung und Kundenbereich antworten.', + 'hosts' => 'Ob die Maschinen antworten, auf denen die Instanzen laufen.', 'instances' => 'Ob die laufenden Kundeninstanzen erreichbar sind.', 'provisioning' => 'Ob Bestellungen der letzten 24 Stunden vollständig ausgeliefert wurden.', 'backups' => 'Ob jede Instanz eine Sicherung aus den letzten 48 Stunden hat.', ], 'detail' => [ + 'hosts' => ':down von :total Maschinen ohne Antwort.', 'instances' => ':down von :total Instanzen ohne positives Signal.', 'provisioning' => ':failed fehlgeschlagene Bereitstellung(en) in den letzten 24 Stunden.', 'backups' => ':stale von :total Instanzen ohne aktuelle Sicherung.', diff --git a/lang/en/admin_incidents.php b/lang/en/admin_incidents.php index 327d149..81e3553 100644 --- a/lang/en/admin_incidents.php +++ b/lang/en/admin_incidents.php @@ -29,4 +29,10 @@ return [ 'appears_on' => 'What you write here appears publicly on the status page at once. The four components above measure themselves — this is the part no probe can know.', 'open_status' => 'Open the status page', + + 'delete' => 'Delete', + 'delete_title' => 'Delete this incident?', + 'delete_body' => '“:title” is removed with all its entries and disappears from the status page at once. This cannot be undone — every other correction is a further entry.', + 'deleted' => 'Incident deleted.', + 'first_update_hint' => 'Posted as the first entry at stage “Investigating”. Everything after that — identified, resolved — you add on the incident itself, on the left.', ]; diff --git a/lang/en/status.php b/lang/en/status.php index 4d3504f..e58e195 100644 --- a/lang/en/status.php +++ b/lang/en/status.php @@ -20,6 +20,7 @@ return [ 'component' => [ 'portal' => 'Sign-in & customer area', + 'hosts' => 'Datacentre', 'instances' => 'Your cloud', 'provisioning' => 'New orders', 'backups' => 'Backups', @@ -30,12 +31,14 @@ return [ // says nothing about the last 24 hours, which is what a reader is asking. 'about' => [ 'portal' => 'Whether sign-in and the customer area respond.', + 'hosts' => 'Whether the machines the instances run on are answering.', 'instances' => 'Whether the customer instances we run are reachable.', 'provisioning' => 'Whether orders in the last 24 hours were delivered in full.', 'backups' => 'Whether every instance has a backup from the last 48 hours.', ], 'detail' => [ + 'hosts' => ':down of :total machines not answering.', 'instances' => ':down of :total instances without a positive signal.', 'provisioning' => ':failed failed provisioning run(s) in the last 24 hours.', 'backups' => ':stale of :total instances without a current backup.', diff --git a/resources/views/livewire/admin/confirm-delete-incident.blade.php b/resources/views/livewire/admin/confirm-delete-incident.blade.php new file mode 100644 index 0000000..c455f74 --- /dev/null +++ b/resources/views/livewire/admin/confirm-delete-incident.blade.php @@ -0,0 +1,9 @@ +
+

{{ __('admin_incidents.delete_title') }}

+

{{ __('admin_incidents.delete_body', ['title' => $title]) }}

+ +
+ {{ __('admin_incidents.cancel') }} + {{ __('admin_incidents.delete') }} +
+
diff --git a/resources/views/livewire/admin/incidents.blade.php b/resources/views/livewire/admin/incidents.blade.php index 6b00566..8825a72 100644 --- a/resources/views/livewire/admin/incidents.blade.php +++ b/resources/views/livewire/admin/incidents.blade.php @@ -65,9 +65,15 @@ @else - - {{ __('admin_incidents.add_update') }} - +
+ + {{ __('admin_incidents.add_update') }} + + + {{ __('admin_incidents.delete') }} + +
@endif @@ -92,6 +98,13 @@ + + + @endforeach @@ -143,6 +156,7 @@
+

{{ __('admin_incidents.first_update_hint') }}

@error('firstUpdate')

{{ $message }}

@enderror diff --git a/routes/console.php b/routes/console.php index aa2f26d..67d0fb9 100644 --- a/routes/console.php +++ b/routes/console.php @@ -92,3 +92,13 @@ Schedule::command('clupilot:verify-domains') Schedule::command('clupilot:auto-update') ->everyFiveMinutes() ->withoutOverlapping(); + +// Ask every host whether it is still there. +// +// hosts.last_seen_at drove the console's health dot and was written exactly +// once, at onboarding — so every host read "offline" half an hour later, +// permanently. Nothing was measuring host reachability at all, which is also +// why it could not appear on the status page. +Schedule::job(new \App\Provisioning\Jobs\PingHosts) + ->everyMinute() + ->name('host-ping'); diff --git a/tests/Feature/Admin/IncidentDeleteTest.php b/tests/Feature/Admin/IncidentDeleteTest.php new file mode 100644 index 0000000..71a7fef --- /dev/null +++ b/tests/Feature/Admin/IncidentDeleteTest.php @@ -0,0 +1,91 @@ +create([ + 'title' => 'Testeintrag', + 'impact' => 'degraded', + 'components' => ['portal'], + 'started_at' => now()->subHour(), + 'published_at' => now()->subHour(), + ...$attributes, + ]); + + $incident->updates()->create(['status' => 'investigating', 'body' => 'Wird untersucht.']); + + return $incident; +} + +it('removes the incident and every entry under it', function () { + $incident = anIncident(); + + Livewire::actingAs(operator('Owner'), 'operator') + ->test(IncidentConsole::class) + ->call('delete', $incident->uuid); + + expect(Incident::query()->count())->toBe(0) + // The updates go with it. A timeline whose incident is gone is rows + // nobody can reach and nobody will remember to clean up. + ->and(DB::table('incident_updates')->count())->toBe(0); +}); + +it('takes it off the public page at once', function () { + $incident = anIncident(['resolved_at' => now()]); + + $this->get('/status')->assertSee('Testeintrag'); + + Livewire::actingAs(operator('Owner'), 'operator') + ->test(IncidentConsole::class) + ->call('delete', $incident->uuid); + + $this->get('/status')->assertDontSee('Testeintrag'); +}); + +it('asks first, in the product’s own dialog', function () { + // R23: not window.confirm. The modal mutates nothing itself — it dispatches + // to the page component, which is where the capability check already is. + $incident = anIncident(); + + Livewire::actingAs(operator('Owner'), 'operator') + ->test(ConfirmDeleteIncident::class, ['uuid' => $incident->uuid]) + ->assertSee($incident->title) + ->call('confirm') + ->assertDispatched('incident-delete-confirmed'); + + // The modal alone does not delete anything. + expect(Incident::query()->count())->toBe(1); +}); + +it('needs the capability, not merely an operator account', function () { + // Asserted on the page, not on the action: render() authorizes too, so an + // operator without the capability never reaches a button — and a + // Livewire::test() against a component that refuses to render fails with a + // snapshot error that says nothing about permissions. + $incident = anIncident(); + + $this->actingAs(operator('Support'), 'operator') + ->get(route('admin.incidents')) + ->assertForbidden(); + + expect(Incident::query()->count())->toBe(1); +}); + +it('still offers no way to rewrite a single entry', function () { + // The rule the delete button must not quietly become an exception to. + expect(method_exists(IncidentConsole::class, 'editUpdate'))->toBeFalse() + ->and(method_exists(IncidentConsole::class, 'deleteUpdate'))->toBeFalse(); +}); diff --git a/tests/Feature/StatusHistoryTest.php b/tests/Feature/StatusHistoryTest.php index 44de68f..b961801 100644 --- a/tests/Feature/StatusHistoryTest.php +++ b/tests/Feature/StatusHistoryTest.php @@ -173,3 +173,35 @@ it('shows an incident that is not over at the top, not filed under history', fun $page->assertSee(__('status.incident.ongoing')); }); + +it('measures whether the hosts answer at all', function () { + // hosts.last_seen_at drove the console's health dot and was written exactly + // once, at onboarding — so every host read "offline" half an hour later, + // for good. Nothing was measuring host reachability, which is why it could + // not appear here. + App\Models\Host::factory()->create(['status' => 'active', 'last_seen_at' => now()]); + App\Models\Host::factory()->create(['status' => 'active', 'last_seen_at' => now()->subHours(2)]); + + $components = collect(app(App\Services\Status\ServiceHealth::class)->components())->keyBy('key'); + + expect($components['hosts']['state'])->toBe('degraded') + ->and($components['hosts']['detail'])->toBe(['down' => 1, 'total' => 2]); +}); + +it('does not call a host that has stopped answering healthy', function () { + // healthState() has three answers and they are not two: "stale" is between + // five and thirty minutes without a word. Not yet an outage, and certainly + // not health. + App\Models\Host::factory()->create(['status' => 'active', 'last_seen_at' => now()->subMinutes(12)]); + + $components = collect(app(App\Services\Status\ServiceHealth::class)->components())->keyBy('key'); + + expect($components['hosts']['state'])->toBe('degraded'); +}); + +it('names no host on a page the whole internet can read', function () { + // Counts only. The estate is not public information. + App\Models\Host::factory()->create(['status' => 'active', 'name' => 'pve-wien-01', 'last_seen_at' => now()->subDay()]); + + $this->get('/status')->assertOk()->assertDontSee('pve-wien-01'); +});