boban
/
CluPilotCloud
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(admin): extract MaintenanceNotifier; atomic provisioning retry; catch-up cancellation for race-delivered announcements 

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
feat/portal-design
nexxo 2026-07-25 16:28:02 +02:00
parent ae1f78f534
commit 8bb93137a3
5 changed files with 123 additions and 83 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
app/Livewire/Admin/Maintenance.php
View File

@ -2,17 +2,12 @@
namespace App\Livewire\Admin;
use App\Mail\MaintenanceAnnouncementMail;
use App\Mail\MaintenanceCancelledMail;
use App\Models\Customer;
use App\Models\Datacenter;
use App\Models\Host;
use App\Models\MaintenanceNotification;
use App\Models\MaintenanceWindow;
use Illuminate\Mail\Mailable;
use App\Services\Maintenance\MaintenanceNotifier;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Mail;
use Livewire\Attributes\Layout;
use Livewire\Attributes\Validate;
use Livewire\Component;
@ -20,9 +15,6 @@ use Livewire\Component;
#[Layout('layouts.admin')]
class Maintenance extends Component
{
/** A pending notification is presumed failed (retryable) after this long. */
private const RETRY_AFTER_MIN = 60;
#[Validate('required|string|max:255')]
public string $title = '';
@ -52,7 +44,7 @@ class Maintenance extends Component
$this->authorize('maintenance.manage');
$window = $this->persist('scheduled');
if ($window !== null) {
$this->sendAnnouncements($window);
app(MaintenanceNotifier::class)->announce($window);
}
}
@ -126,7 +118,7 @@ class Maintenance extends Component
return;
}
$window->update(['state' => 'scheduled', 'published_at' => now()]);
$this->sendAnnouncements($window);
app(MaintenanceNotifier::class)->announce($window);
$this->dispatch('notify', message: __('maintenance.published'));
}
@ -143,7 +135,7 @@ class Maintenance extends Component
if ($window === null || ! in_array($window->derivedState(), ['upcoming', 'active'], true)) {
return;
}
$this->sendAnnouncements($window);
app(MaintenanceNotifier::class)->announce($window);
$this->dispatch('notify', message: __('maintenance.notified'));
}
@ -158,72 +150,17 @@ class Maintenance extends Component
$wasPublished = $window->state === 'scheduled';
$window->update(['state' => 'cancelled', 'cancelled_at' => now()]);
// Customers who received an announcement must be told it's cancelled.
// Customers who received an announcement are told it's cancelled. The
// MessageSending listener suppresses still-pending announcements, and the
// MessageSent listener catches the race (an announcement that delivers
// just as we cancel) by queuing a catch-up cancellation for it.
if ($wasPublished) {
$notified = MaintenanceNotification::query()
->where('maintenance_window_id', $window->id)
->where('event', 'announcement')
->whereNotNull('sent_at') // only customers who actually received it
->pluck('customer_id');
foreach (Customer::query()->whereIn('id', $notified)->get() as $customer) {
$this->deliverOnce($window, $customer, 'cancelled', new MaintenanceCancelledMail($window, $customer));
}
app(MaintenanceNotifier::class)->notifyCancellation($window);
}
$this->dispatch('notify', message: __('maintenance.cancelled'));
}
/** Announce to each affected customer exactly once (ledger-guarded). */
private function sendAnnouncements(MaintenanceWindow $window): void
{
foreach ($window->affectedCustomers() as $customer) {
$this->deliverOnce($window, $customer, 'announcement', new MaintenanceAnnouncementMail($window, $customer));
}
}
/**
* Queue one mail per (window, customer, event). The ledger row is the
* idempotency marker; if dispatch itself fails, drop the row so it retries.
* sent_at stays null until real delivery — we never claim delivery on dispatch.
*/
private function deliverOnce(MaintenanceWindow $window, Customer $customer, string $event, Mailable $mail): void
{
$delivery = MaintenanceNotification::query()->firstOrCreate(
['maintenance_window_id' => $window->id, 'customer_id' => $customer->id, 'event' => $event],
['email' => $customer->email],
);
// Delivered rows (sent_at set by the MessageSent listener) are final.
if ($delivery->sent_at !== null) {
return;
}
// Otherwise the row is pending. Re-queue only when this is a fresh claim
// or the previous attempt is stale enough to be presumed failed — so a
// resend while the job is still in flight does not duplicate the email.
// updated_at is the last-attempt marker (touched on every dispatch).
if (! $delivery->wasRecentlyCreated && $delivery->updated_at->gt(now()->subMinutes(self::RETRY_AFTER_MIN))) {
return;
}
// Claim the row, then dispatch. If dispatch fails, release the claim
// immediately (delete if we created it, else restore the prior attempt
// time) so a resend can retry right away instead of waiting out the TTL.
$wasNew = $delivery->wasRecentlyCreated;
$priorUpdatedAt = $delivery->updated_at;
$delivery->forceFill(['updated_at' => now()])->save();
$mail->notificationId = $delivery->id;
try {
Mail::to($customer->email)
->locale($customer->locale ?: config('app.locale'))
->queue($mail);
} catch (\Throwable $e) {
$wasNew ? $delivery->delete() : $delivery->forceFill(['updated_at' => $priorUpdatedAt])->save();
throw $e;
}
}
public function render()
{
$windows = MaintenanceWindow::query()

19
app/Livewire/Admin/Provisioning.php
View File

@ -27,18 +27,25 @@ class Provisioning extends Component
public function retry(string $uuid): void
{
$this->authorize('provisioning.retry');
$run = ProvisioningRun::query()->where('uuid', $uuid)->first();
if ($run === null || $run->status !== ProvisioningRun::STATUS_FAILED) {
return;
}
$run->update([
// Atomically claim the run: only the request that transitions it out of
// FAILED proceeds, so two concurrent retries can't double-dispatch.
$claimed = ProvisioningRun::query()
->where('uuid', $uuid)
->where('status', ProvisioningRun::STATUS_FAILED)
->update([
'status' => ProvisioningRun::STATUS_RUNNING,
'attempt' => 0,
'next_attempt_at' => now(),
'started_at' => now(), // reset the step timer so it doesn't re-time-out instantly
'error' => null,
]);
if ($claimed === 0) {
return; // already retried by a concurrent request
}
$run = ProvisioningRun::query()->where('uuid', $uuid)->first();
if ($run === null) {
return;
}
// Move the subject out of its error state so the console reflects the retry.
$subject = $run->subject;

5
app/Models/MaintenanceNotification.php
View File

@ -18,4 +18,9 @@ class MaintenanceNotification extends Model
{
return $this->belongsTo(MaintenanceWindow::class, 'maintenance_window_id');
}
public function customer(): BelongsTo
{
return $this->belongsTo(Customer::class);
}
}

21
app/Providers/AppServiceProvider.php
View File

@ -15,7 +15,9 @@ use App\Services\Traefik\SshTraefikWriter;
use App\Services\Traefik\TraefikWriter;
use App\Services\Wireguard\LocalWireguardHub;
use App\Services\Wireguard\WireguardHub;
use App\Mail\MaintenanceCancelledMail;
use App\Models\MaintenanceNotification;
use App\Services\Maintenance\MaintenanceNotifier;
use Illuminate\Mail\Events\MessageSending;
use Illuminate\Mail\Events\MessageSent;
use Illuminate\Support\Facades\Event;
@ -64,13 +66,28 @@ class AppServiceProvider extends ServiceProvider
// Stamp a maintenance-notification ledger row as delivered only once the
// mail is actually sent (the X-CP-Notification header carries the id).
// Until then sent_at stays null → the row is a retryable marker.
// Until then sent_at stays null → the row is a retryable marker. If an
// announcement delivered for an already-cancelled window (the cancel race),
// queue a catch-up cancellation so that customer isn't left mis-informed.
Event::listen(MessageSent::class, function (MessageSent $event) {
$header = $event->message->getHeaders()->get('X-CP-Notification');
if ($header === null) {
return;
}
MaintenanceNotification::query()->whereKey((int) $header->getBodyAsString())->update(['sent_at' => now()]);
$notification = MaintenanceNotification::query()->with(['window', 'customer'])->find((int) $header->getBodyAsString());
if ($notification === null) {
return;
}
$notification->update(['sent_at' => now()]);
if ($notification->event === 'announcement' && $notification->window?->state === 'cancelled' && $notification->customer !== null) {
app(MaintenanceNotifier::class)->deliver(
$notification->window,
$notification->customer,
'cancelled',
new MaintenanceCancelledMail($notification->window, $notification->customer),
);
}
});
}
}

74
app/Services/Maintenance/MaintenanceNotifier.php Normal file
View File

@ -0,0 +1,74 @@
<?php
namespace App\Services\Maintenance;
use App\Mail\MaintenanceAnnouncementMail;
use App\Mail\MaintenanceCancelledMail;
use App\Models\Customer;
use App\Models\MaintenanceNotification;
use App\Models\MaintenanceWindow;
use Illuminate\Mail\Mailable;
use Illuminate\Support\Facades\Mail;
/**
* Ledger-guarded delivery of maintenance emails. Shared by the admin console and
* the mail-event listeners so idempotency and the announcement/cancellation
* pairing are enforced in exactly one place.
*/
class MaintenanceNotifier
{
/** A pending notification is presumed failed (retryable) after this long. */
public const RETRY_AFTER_MIN = 60;
public function announce(MaintenanceWindow $window): void
{
foreach ($window->affectedCustomers() as $customer) {
$this->deliver($window, $customer, 'announcement', new MaintenanceAnnouncementMail($window, $customer));
}
}
/** Notify customers who actually received the announcement that it's cancelled. */
public function notifyCancellation(MaintenanceWindow $window): void
{
$ids = MaintenanceNotification::query()
->where('maintenance_window_id', $window->id)
->where('event', 'announcement')
->whereNotNull('sent_at')
->pluck('customer_id');
foreach (Customer::query()->whereIn('id', $ids)->get() as $customer) {
$this->deliver($window, $customer, 'cancelled', new MaintenanceCancelledMail($window, $customer));
}
}
/** Queue one mail per (window, customer, event), idempotently and retryably. */
public function deliver(MaintenanceWindow $window, Customer $customer, string $event, Mailable $mail): void
{
$delivery = MaintenanceNotification::query()->firstOrCreate(
['maintenance_window_id' => $window->id, 'customer_id' => $customer->id, 'event' => $event],
['email' => $customer->email],
);
if ($delivery->sent_at !== null) {
return; // already delivered — final
}
if (! $delivery->wasRecentlyCreated && $delivery->updated_at->gt(now()->subMinutes(self::RETRY_AFTER_MIN))) {
return; // still pending/in-flight — don't duplicate
}
$wasNew = $delivery->wasRecentlyCreated;
$priorUpdatedAt = $delivery->updated_at;
$delivery->forceFill(['updated_at' => now()])->save();
$mail->notificationId = $delivery->id;
try {
Mail::to($customer->email)
->locale($customer->locale ?: config('app.locale'))
->queue($mail);
} catch (\Throwable $e) {
$wasNew ? $delivery->delete() : $delivery->forceFill(['updated_at' => $priorUpdatedAt])->save();
throw $e;
}
}
}