diff --git a/app/Livewire/Admin/Maintenance.php b/app/Livewire/Admin/Maintenance.php index 70219b0..76cb332 100644 --- a/app/Livewire/Admin/Maintenance.php +++ b/app/Livewire/Admin/Maintenance.php @@ -2,17 +2,12 @@ namespace App\Livewire\Admin; -use App\Mail\MaintenanceAnnouncementMail; -use App\Mail\MaintenanceCancelledMail; -use App\Models\Customer; use App\Models\Datacenter; use App\Models\Host; -use App\Models\MaintenanceNotification; use App\Models\MaintenanceWindow; -use Illuminate\Mail\Mailable; +use App\Services\Maintenance\MaintenanceNotifier; use Illuminate\Support\Carbon; use Illuminate\Support\Facades\DB; -use Illuminate\Support\Facades\Mail; use Livewire\Attributes\Layout; use Livewire\Attributes\Validate; use Livewire\Component; @@ -20,9 +15,6 @@ use Livewire\Component; #[Layout('layouts.admin')] class Maintenance extends Component { - /** A pending notification is presumed failed (retryable) after this long. */ - private const RETRY_AFTER_MIN = 60; - #[Validate('required|string|max:255')] public string $title = ''; @@ -52,7 +44,7 @@ class Maintenance extends Component $this->authorize('maintenance.manage'); $window = $this->persist('scheduled'); if ($window !== null) { - $this->sendAnnouncements($window); + app(MaintenanceNotifier::class)->announce($window); } } @@ -126,7 +118,7 @@ class Maintenance extends Component return; } $window->update(['state' => 'scheduled', 'published_at' => now()]); - $this->sendAnnouncements($window); + app(MaintenanceNotifier::class)->announce($window); $this->dispatch('notify', message: __('maintenance.published')); } @@ -143,7 +135,7 @@ class Maintenance extends Component if ($window === null || ! in_array($window->derivedState(), ['upcoming', 'active'], true)) { return; } - $this->sendAnnouncements($window); + app(MaintenanceNotifier::class)->announce($window); $this->dispatch('notify', message: __('maintenance.notified')); } @@ -158,72 +150,17 @@ class Maintenance extends Component $wasPublished = $window->state === 'scheduled'; $window->update(['state' => 'cancelled', 'cancelled_at' => now()]); - // Customers who received an announcement must be told it's cancelled. + // Customers who received an announcement are told it's cancelled. The + // MessageSending listener suppresses still-pending announcements, and the + // MessageSent listener catches the race (an announcement that delivers + // just as we cancel) by queuing a catch-up cancellation for it. if ($wasPublished) { - $notified = MaintenanceNotification::query() - ->where('maintenance_window_id', $window->id) - ->where('event', 'announcement') - ->whereNotNull('sent_at') // only customers who actually received it - ->pluck('customer_id'); - foreach (Customer::query()->whereIn('id', $notified)->get() as $customer) { - $this->deliverOnce($window, $customer, 'cancelled', new MaintenanceCancelledMail($window, $customer)); - } + app(MaintenanceNotifier::class)->notifyCancellation($window); } $this->dispatch('notify', message: __('maintenance.cancelled')); } - /** Announce to each affected customer exactly once (ledger-guarded). */ - private function sendAnnouncements(MaintenanceWindow $window): void - { - foreach ($window->affectedCustomers() as $customer) { - $this->deliverOnce($window, $customer, 'announcement', new MaintenanceAnnouncementMail($window, $customer)); - } - } - - /** - * Queue one mail per (window, customer, event). The ledger row is the - * idempotency marker; if dispatch itself fails, drop the row so it retries. - * sent_at stays null until real delivery — we never claim delivery on dispatch. - */ - private function deliverOnce(MaintenanceWindow $window, Customer $customer, string $event, Mailable $mail): void - { - $delivery = MaintenanceNotification::query()->firstOrCreate( - ['maintenance_window_id' => $window->id, 'customer_id' => $customer->id, 'event' => $event], - ['email' => $customer->email], - ); - - // Delivered rows (sent_at set by the MessageSent listener) are final. - if ($delivery->sent_at !== null) { - return; - } - // Otherwise the row is pending. Re-queue only when this is a fresh claim - // or the previous attempt is stale enough to be presumed failed — so a - // resend while the job is still in flight does not duplicate the email. - // updated_at is the last-attempt marker (touched on every dispatch). - if (! $delivery->wasRecentlyCreated && $delivery->updated_at->gt(now()->subMinutes(self::RETRY_AFTER_MIN))) { - return; - } - - // Claim the row, then dispatch. If dispatch fails, release the claim - // immediately (delete if we created it, else restore the prior attempt - // time) so a resend can retry right away instead of waiting out the TTL. - $wasNew = $delivery->wasRecentlyCreated; - $priorUpdatedAt = $delivery->updated_at; - $delivery->forceFill(['updated_at' => now()])->save(); - $mail->notificationId = $delivery->id; - - try { - Mail::to($customer->email) - ->locale($customer->locale ?: config('app.locale')) - ->queue($mail); - } catch (\Throwable $e) { - $wasNew ? $delivery->delete() : $delivery->forceFill(['updated_at' => $priorUpdatedAt])->save(); - - throw $e; - } - } - public function render() { $windows = MaintenanceWindow::query() diff --git a/app/Livewire/Admin/Provisioning.php b/app/Livewire/Admin/Provisioning.php index 3bf5538..72725b4 100644 --- a/app/Livewire/Admin/Provisioning.php +++ b/app/Livewire/Admin/Provisioning.php @@ -27,19 +27,26 @@ class Provisioning extends Component public function retry(string $uuid): void { $this->authorize('provisioning.retry'); + // Atomically claim the run: only the request that transitions it out of + // FAILED proceeds, so two concurrent retries can't double-dispatch. + $claimed = ProvisioningRun::query() + ->where('uuid', $uuid) + ->where('status', ProvisioningRun::STATUS_FAILED) + ->update([ + 'status' => ProvisioningRun::STATUS_RUNNING, + 'attempt' => 0, + 'next_attempt_at' => now(), + 'started_at' => now(), // reset the step timer so it doesn't re-time-out instantly + 'error' => null, + ]); + if ($claimed === 0) { + return; // already retried by a concurrent request + } $run = ProvisioningRun::query()->where('uuid', $uuid)->first(); - if ($run === null || $run->status !== ProvisioningRun::STATUS_FAILED) { + if ($run === null) { return; } - $run->update([ - 'status' => ProvisioningRun::STATUS_RUNNING, - 'attempt' => 0, - 'next_attempt_at' => now(), - 'started_at' => now(), // reset the step timer so it doesn't re-time-out instantly - 'error' => null, - ]); - // Move the subject out of its error state so the console reflects the retry. $subject = $run->subject; if ($subject instanceof Host) { diff --git a/app/Models/MaintenanceNotification.php b/app/Models/MaintenanceNotification.php index 986c1e1..14d4d2c 100644 --- a/app/Models/MaintenanceNotification.php +++ b/app/Models/MaintenanceNotification.php @@ -18,4 +18,9 @@ class MaintenanceNotification extends Model { return $this->belongsTo(MaintenanceWindow::class, 'maintenance_window_id'); } + + public function customer(): BelongsTo + { + return $this->belongsTo(Customer::class); + } } diff --git a/app/Providers/AppServiceProvider.php b/app/Providers/AppServiceProvider.php index 99a0ab8..2f8ccdf 100644 --- a/app/Providers/AppServiceProvider.php +++ b/app/Providers/AppServiceProvider.php @@ -15,7 +15,9 @@ use App\Services\Traefik\SshTraefikWriter; use App\Services\Traefik\TraefikWriter; use App\Services\Wireguard\LocalWireguardHub; use App\Services\Wireguard\WireguardHub; +use App\Mail\MaintenanceCancelledMail; use App\Models\MaintenanceNotification; +use App\Services\Maintenance\MaintenanceNotifier; use Illuminate\Mail\Events\MessageSending; use Illuminate\Mail\Events\MessageSent; use Illuminate\Support\Facades\Event; @@ -64,13 +66,28 @@ class AppServiceProvider extends ServiceProvider // Stamp a maintenance-notification ledger row as delivered only once the // mail is actually sent (the X-CP-Notification header carries the id). - // Until then sent_at stays null → the row is a retryable marker. + // Until then sent_at stays null → the row is a retryable marker. If an + // announcement delivered for an already-cancelled window (the cancel race), + // queue a catch-up cancellation so that customer isn't left mis-informed. Event::listen(MessageSent::class, function (MessageSent $event) { $header = $event->message->getHeaders()->get('X-CP-Notification'); if ($header === null) { return; } - MaintenanceNotification::query()->whereKey((int) $header->getBodyAsString())->update(['sent_at' => now()]); + $notification = MaintenanceNotification::query()->with(['window', 'customer'])->find((int) $header->getBodyAsString()); + if ($notification === null) { + return; + } + $notification->update(['sent_at' => now()]); + + if ($notification->event === 'announcement' && $notification->window?->state === 'cancelled' && $notification->customer !== null) { + app(MaintenanceNotifier::class)->deliver( + $notification->window, + $notification->customer, + 'cancelled', + new MaintenanceCancelledMail($notification->window, $notification->customer), + ); + } }); } } diff --git a/app/Services/Maintenance/MaintenanceNotifier.php b/app/Services/Maintenance/MaintenanceNotifier.php new file mode 100644 index 0000000..c171cfe --- /dev/null +++ b/app/Services/Maintenance/MaintenanceNotifier.php @@ -0,0 +1,74 @@ +affectedCustomers() as $customer) { + $this->deliver($window, $customer, 'announcement', new MaintenanceAnnouncementMail($window, $customer)); + } + } + + /** Notify customers who actually received the announcement that it's cancelled. */ + public function notifyCancellation(MaintenanceWindow $window): void + { + $ids = MaintenanceNotification::query() + ->where('maintenance_window_id', $window->id) + ->where('event', 'announcement') + ->whereNotNull('sent_at') + ->pluck('customer_id'); + + foreach (Customer::query()->whereIn('id', $ids)->get() as $customer) { + $this->deliver($window, $customer, 'cancelled', new MaintenanceCancelledMail($window, $customer)); + } + } + + /** Queue one mail per (window, customer, event), idempotently and retryably. */ + public function deliver(MaintenanceWindow $window, Customer $customer, string $event, Mailable $mail): void + { + $delivery = MaintenanceNotification::query()->firstOrCreate( + ['maintenance_window_id' => $window->id, 'customer_id' => $customer->id, 'event' => $event], + ['email' => $customer->email], + ); + + if ($delivery->sent_at !== null) { + return; // already delivered — final + } + if (! $delivery->wasRecentlyCreated && $delivery->updated_at->gt(now()->subMinutes(self::RETRY_AFTER_MIN))) { + return; // still pending/in-flight — don't duplicate + } + + $wasNew = $delivery->wasRecentlyCreated; + $priorUpdatedAt = $delivery->updated_at; + $delivery->forceFill(['updated_at' => now()])->save(); + $mail->notificationId = $delivery->id; + + try { + Mail::to($customer->email) + ->locale($customer->locale ?: config('app.locale')) + ->queue($mail); + } catch (\Throwable $e) { + $wasNew ? $delivery->delete() : $delivery->forceFill(['updated_at' => $priorUpdatedAt])->save(); + + throw $e; + } + } +}