diff --git a/VERSION b/VERSION index 90a7f60..7962dcf 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.3.12 +1.3.13 diff --git a/app/Livewire/Admin/ConfirmDeleteIncident.php b/app/Livewire/Admin/ConfirmDeleteIncident.php new file mode 100644 index 0000000..3f1afd4 --- /dev/null +++ b/app/Livewire/Admin/ConfirmDeleteIncident.php @@ -0,0 +1,40 @@ +where('uuid', $uuid)->firstOrFail(); + + $this->uuid = $incident->uuid; + $this->title = $incident->title; + } + + public function confirm(): void + { + $this->dispatch('incident-delete-confirmed', uuid: $this->uuid); + $this->closeModal(); + } + + public function render() + { + return view('livewire.admin.confirm-delete-incident'); + } +} diff --git a/app/Livewire/Admin/Incidents.php b/app/Livewire/Admin/Incidents.php index 3ccb58a..1b715db 100644 --- a/app/Livewire/Admin/Incidents.php +++ b/app/Livewire/Admin/Incidents.php @@ -9,6 +9,7 @@ use App\Support\LocalTime; use Illuminate\Support\Facades\Auth; use Illuminate\Support\Facades\DB; use Livewire\Attributes\Layout; +use Livewire\Attributes\On; use Livewire\Attributes\Validate; use Livewire\Component; @@ -83,6 +84,9 @@ class Incidents extends Component 'created_by' => Auth::guard('operator')->user()?->email, ]); + // The first entry is always "wird untersucht" — that is what a + // first report is. The form says so, so nobody looks for a stage + // selector that is not there. $incident->updates()->create([ 'status' => 'investigating', 'body' => $this->firstUpdate, @@ -139,6 +143,34 @@ class Incidents extends Component $this->dispatch('notify', message: __('admin_incidents.update_posted')); } + /** + * Remove an incident entirely — the whole record, updates and all. + * + * Deliberately different from editing an update, which there is still no + * way to do: an update is a statement made at a time, and rewriting one + * afterwards is what this record exists to prevent. Removing the WHOLE + * incident is a different act. It is how a test entry, or one published by + * mistake against the wrong service, is taken back — and taking it back + * leaves nothing half-standing behind. + * + * Owner-level, and confirmed in a modal (R23), because on a page whose + * whole value is that it is believed, deleting the record is the one action + * that cannot be undone by writing more. + */ + #[On('incident-delete-confirmed')] + public function delete(string $uuid): void + { + $this->authorize('maintenance.manage'); + + Incident::query()->where('uuid', $uuid)->first()?->delete(); + + if ($this->updatingUuid === $uuid) { + $this->cancelUpdate(); + } + + $this->dispatch('notify', message: __('admin_incidents.deleted')); + } + /** @return array */ public function componentOptions(): array { diff --git a/app/Provisioning/Jobs/PingHosts.php b/app/Provisioning/Jobs/PingHosts.php new file mode 100644 index 0000000..c264ad2 --- /dev/null +++ b/app/Provisioning/Jobs/PingHosts.php @@ -0,0 +1,68 @@ +onQueue('provisioning'); + } + + public function handle(ProxmoxClient $proxmox): void + { + $hosts = Host::query() + // Hosts still being onboarded, or already retired, are not a + // statement about the platform's health. + ->whereIn('status', ['active', 'error']) + ->get(); + + foreach ($hosts as $host) { + try { + $proxmox->forHost($host)->nodeStatus($host->node); + + // Only the timestamp. Whether a host is "active" or "error" is + // an operator's decision and a provisioning outcome — a + // heartbeat must not quietly promote a host somebody has + // deliberately taken out of service. + $host->forceFill(['last_seen_at' => now()])->save(); + } catch (Throwable) { + // Deliberately silent, and deliberately not writing anything. + // Absence IS the signal: healthState() reads the age of the + // last successful answer, so a failure is recorded by the + // timestamp not moving. Writing a failure count here would be a + // second version of the same fact. + } + } + } +} diff --git a/app/Services/Status/ServiceHealth.php b/app/Services/Status/ServiceHealth.php index eea6225..266ecc3 100644 --- a/app/Services/Status/ServiceHealth.php +++ b/app/Services/Status/ServiceHealth.php @@ -2,6 +2,7 @@ namespace App\Services\Status; +use App\Models\Host; use App\Models\Instance; use App\Models\MonitoringTarget; use App\Models\ProvisioningRun; @@ -27,7 +28,7 @@ use Illuminate\Support\Carbon; class ServiceHealth { /** The components, in the order they appear. */ - public const COMPONENTS = ['portal', 'instances', 'provisioning', 'backups']; + public const COMPONENTS = ['portal', 'hosts', 'instances', 'provisioning', 'backups']; /** A backup older than this stops counting as current. */ private const BACKUP_STALE_AFTER_HOURS = 48; @@ -50,6 +51,7 @@ class ServiceHealth { return [ $this->portal(), + $this->hosts(), $this->instances(), $this->provisioning(), $this->backups(), @@ -87,6 +89,46 @@ class ServiceHealth return ['key' => 'portal', 'state' => 'operational', 'detail' => null]; } + /** + * The machines the instances run on. + * + * From the heartbeat PingHosts writes, which is the same question + * provisioning asks: can we reach this host's API with the credentials we + * hold. A host that fails that is down as far as this platform is + * concerned, whatever a separate ICMP probe would say about the wire. + * + * Counts only — never a host name, never an address. This page is + * world-readable and the estate is not public information. + * + * @return array + */ + private function hosts(): array + { + $hosts = Host::query()->where('status', 'active')->get(); + $total = $hosts->count(); + + if ($total === 0) { + return ['key' => 'hosts', 'state' => 'operational', 'detail' => null]; + } + + // healthState() has three answers and they are not two. "stale" is a + // host that has not answered for between five and thirty minutes: not + // yet an outage, and absolutely not health. + $reachable = $hosts->filter(fn (Host $host) => $host->healthState() === 'online')->count(); + $offline = $hosts->filter(fn (Host $host) => $host->healthState() === 'offline')->count(); + + return [ + 'key' => 'hosts', + 'state' => match (true) { + $offline >= $total => 'down', + $offline > 0 => 'degraded', + $reachable < $total => 'degraded', + default => 'operational', + }, + 'detail' => $reachable < $total ? ['down' => $total - $reachable, 'total' => $total] : null, + ]; + } + /** * Customer instances, from what monitoring last saw. * diff --git a/lang/de/admin_incidents.php b/lang/de/admin_incidents.php index 24b3072..4a3b997 100644 --- a/lang/de/admin_incidents.php +++ b/lang/de/admin_incidents.php @@ -29,4 +29,10 @@ return [ 'appears_on' => 'Was Sie hier eintragen, erscheint sofort öffentlich auf der Statusseite. Die vier Dienste darüber messen sich selbst — hier steht das, was keine Messung wissen kann.', 'open_status' => 'Statusseite öffnen', + + 'delete' => 'Löschen', + 'delete_title' => 'Störung löschen?', + 'delete_body' => '„:title“ wird samt allen Einträgen entfernt und verschwindet sofort von der Statusseite. Das lässt sich nicht rückgängig machen — eine Korrektur ist sonst immer ein weiterer Eintrag.', + 'deleted' => 'Störung gelöscht.', + 'first_update_hint' => 'Erscheint als erster Eintrag mit dem Stand „Wird untersucht“. Alles Weitere — Ursache gefunden, behoben — tragen Sie danach links bei der Störung nach.', ]; diff --git a/lang/de/status.php b/lang/de/status.php index cec1093..e6d00e5 100644 --- a/lang/de/status.php +++ b/lang/de/status.php @@ -20,6 +20,7 @@ return [ 'component' => [ 'portal' => 'Anmeldung & Kundenbereich', + 'hosts' => 'Rechenzentrum', 'instances' => 'Ihre Cloud', 'provisioning' => 'Neue Bestellungen', 'backups' => 'Sicherungen', @@ -31,12 +32,14 @@ return [ // actually asking about. 'about' => [ 'portal' => 'Ob Anmeldung und Kundenbereich antworten.', + 'hosts' => 'Ob die Maschinen antworten, auf denen die Instanzen laufen.', 'instances' => 'Ob die laufenden Kundeninstanzen erreichbar sind.', 'provisioning' => 'Ob Bestellungen der letzten 24 Stunden vollständig ausgeliefert wurden.', 'backups' => 'Ob jede Instanz eine Sicherung aus den letzten 48 Stunden hat.', ], 'detail' => [ + 'hosts' => ':down von :total Maschinen ohne Antwort.', 'instances' => ':down von :total Instanzen ohne positives Signal.', 'provisioning' => ':failed fehlgeschlagene Bereitstellung(en) in den letzten 24 Stunden.', 'backups' => ':stale von :total Instanzen ohne aktuelle Sicherung.', diff --git a/lang/en/admin_incidents.php b/lang/en/admin_incidents.php index 327d149..81e3553 100644 --- a/lang/en/admin_incidents.php +++ b/lang/en/admin_incidents.php @@ -29,4 +29,10 @@ return [ 'appears_on' => 'What you write here appears publicly on the status page at once. The four components above measure themselves — this is the part no probe can know.', 'open_status' => 'Open the status page', + + 'delete' => 'Delete', + 'delete_title' => 'Delete this incident?', + 'delete_body' => '“:title” is removed with all its entries and disappears from the status page at once. This cannot be undone — every other correction is a further entry.', + 'deleted' => 'Incident deleted.', + 'first_update_hint' => 'Posted as the first entry at stage “Investigating”. Everything after that — identified, resolved — you add on the incident itself, on the left.', ]; diff --git a/lang/en/status.php b/lang/en/status.php index 4d3504f..e58e195 100644 --- a/lang/en/status.php +++ b/lang/en/status.php @@ -20,6 +20,7 @@ return [ 'component' => [ 'portal' => 'Sign-in & customer area', + 'hosts' => 'Datacentre', 'instances' => 'Your cloud', 'provisioning' => 'New orders', 'backups' => 'Backups', @@ -30,12 +31,14 @@ return [ // says nothing about the last 24 hours, which is what a reader is asking. 'about' => [ 'portal' => 'Whether sign-in and the customer area respond.', + 'hosts' => 'Whether the machines the instances run on are answering.', 'instances' => 'Whether the customer instances we run are reachable.', 'provisioning' => 'Whether orders in the last 24 hours were delivered in full.', 'backups' => 'Whether every instance has a backup from the last 48 hours.', ], 'detail' => [ + 'hosts' => ':down of :total machines not answering.', 'instances' => ':down of :total instances without a positive signal.', 'provisioning' => ':failed failed provisioning run(s) in the last 24 hours.', 'backups' => ':stale of :total instances without a current backup.', diff --git a/resources/views/livewire/admin/confirm-delete-incident.blade.php b/resources/views/livewire/admin/confirm-delete-incident.blade.php new file mode 100644 index 0000000..c455f74 --- /dev/null +++ b/resources/views/livewire/admin/confirm-delete-incident.blade.php @@ -0,0 +1,9 @@ +
+

{{ __('admin_incidents.delete_title') }}

+

{{ __('admin_incidents.delete_body', ['title' => $title]) }}

+ +
+ {{ __('admin_incidents.cancel') }} + {{ __('admin_incidents.delete') }} +
+
diff --git a/resources/views/livewire/admin/incidents.blade.php b/resources/views/livewire/admin/incidents.blade.php index 6b00566..8825a72 100644 --- a/resources/views/livewire/admin/incidents.blade.php +++ b/resources/views/livewire/admin/incidents.blade.php @@ -65,9 +65,15 @@ @else - - {{ __('admin_incidents.add_update') }} - +
+ + {{ __('admin_incidents.add_update') }} + + + {{ __('admin_incidents.delete') }} + +
@endif @@ -92,6 +98,13 @@ + + + @endforeach @@ -143,6 +156,7 @@
+

{{ __('admin_incidents.first_update_hint') }}

@error('firstUpdate')

{{ $message }}

@enderror diff --git a/routes/console.php b/routes/console.php index aa2f26d..67d0fb9 100644 --- a/routes/console.php +++ b/routes/console.php @@ -92,3 +92,13 @@ Schedule::command('clupilot:verify-domains') Schedule::command('clupilot:auto-update') ->everyFiveMinutes() ->withoutOverlapping(); + +// Ask every host whether it is still there. +// +// hosts.last_seen_at drove the console's health dot and was written exactly +// once, at onboarding — so every host read "offline" half an hour later, +// permanently. Nothing was measuring host reachability at all, which is also +// why it could not appear on the status page. +Schedule::job(new \App\Provisioning\Jobs\PingHosts) + ->everyMinute() + ->name('host-ping'); diff --git a/tests/Feature/Admin/IncidentDeleteTest.php b/tests/Feature/Admin/IncidentDeleteTest.php new file mode 100644 index 0000000..71a7fef --- /dev/null +++ b/tests/Feature/Admin/IncidentDeleteTest.php @@ -0,0 +1,91 @@ +create([ + 'title' => 'Testeintrag', + 'impact' => 'degraded', + 'components' => ['portal'], + 'started_at' => now()->subHour(), + 'published_at' => now()->subHour(), + ...$attributes, + ]); + + $incident->updates()->create(['status' => 'investigating', 'body' => 'Wird untersucht.']); + + return $incident; +} + +it('removes the incident and every entry under it', function () { + $incident = anIncident(); + + Livewire::actingAs(operator('Owner'), 'operator') + ->test(IncidentConsole::class) + ->call('delete', $incident->uuid); + + expect(Incident::query()->count())->toBe(0) + // The updates go with it. A timeline whose incident is gone is rows + // nobody can reach and nobody will remember to clean up. + ->and(DB::table('incident_updates')->count())->toBe(0); +}); + +it('takes it off the public page at once', function () { + $incident = anIncident(['resolved_at' => now()]); + + $this->get('/status')->assertSee('Testeintrag'); + + Livewire::actingAs(operator('Owner'), 'operator') + ->test(IncidentConsole::class) + ->call('delete', $incident->uuid); + + $this->get('/status')->assertDontSee('Testeintrag'); +}); + +it('asks first, in the product’s own dialog', function () { + // R23: not window.confirm. The modal mutates nothing itself — it dispatches + // to the page component, which is where the capability check already is. + $incident = anIncident(); + + Livewire::actingAs(operator('Owner'), 'operator') + ->test(ConfirmDeleteIncident::class, ['uuid' => $incident->uuid]) + ->assertSee($incident->title) + ->call('confirm') + ->assertDispatched('incident-delete-confirmed'); + + // The modal alone does not delete anything. + expect(Incident::query()->count())->toBe(1); +}); + +it('needs the capability, not merely an operator account', function () { + // Asserted on the page, not on the action: render() authorizes too, so an + // operator without the capability never reaches a button — and a + // Livewire::test() against a component that refuses to render fails with a + // snapshot error that says nothing about permissions. + $incident = anIncident(); + + $this->actingAs(operator('Support'), 'operator') + ->get(route('admin.incidents')) + ->assertForbidden(); + + expect(Incident::query()->count())->toBe(1); +}); + +it('still offers no way to rewrite a single entry', function () { + // The rule the delete button must not quietly become an exception to. + expect(method_exists(IncidentConsole::class, 'editUpdate'))->toBeFalse() + ->and(method_exists(IncidentConsole::class, 'deleteUpdate'))->toBeFalse(); +}); diff --git a/tests/Feature/StatusHistoryTest.php b/tests/Feature/StatusHistoryTest.php index 44de68f..b961801 100644 --- a/tests/Feature/StatusHistoryTest.php +++ b/tests/Feature/StatusHistoryTest.php @@ -173,3 +173,35 @@ it('shows an incident that is not over at the top, not filed under history', fun $page->assertSee(__('status.incident.ongoing')); }); + +it('measures whether the hosts answer at all', function () { + // hosts.last_seen_at drove the console's health dot and was written exactly + // once, at onboarding — so every host read "offline" half an hour later, + // for good. Nothing was measuring host reachability, which is why it could + // not appear here. + App\Models\Host::factory()->create(['status' => 'active', 'last_seen_at' => now()]); + App\Models\Host::factory()->create(['status' => 'active', 'last_seen_at' => now()->subHours(2)]); + + $components = collect(app(App\Services\Status\ServiceHealth::class)->components())->keyBy('key'); + + expect($components['hosts']['state'])->toBe('degraded') + ->and($components['hosts']['detail'])->toBe(['down' => 1, 'total' => 2]); +}); + +it('does not call a host that has stopped answering healthy', function () { + // healthState() has three answers and they are not two: "stale" is between + // five and thirty minutes without a word. Not yet an outage, and certainly + // not health. + App\Models\Host::factory()->create(['status' => 'active', 'last_seen_at' => now()->subMinutes(12)]); + + $components = collect(app(App\Services\Status\ServiceHealth::class)->components())->keyBy('key'); + + expect($components['hosts']['state'])->toBe('degraded'); +}); + +it('names no host on a page the whole internet can read', function () { + // Counts only. The estate is not public information. + App\Models\Host::factory()->create(['status' => 'active', 'name' => 'pve-wien-01', 'last_seen_at' => now()->subDay()]); + + $this->get('/status')->assertOk()->assertDontSee('pve-wien-01'); +});