CluPilotCloud/app/Provisioning/Jobs/CollectInstanceTraffic.php

<?php

namespace App\Provisioning\Jobs;

use App\Models\Instance;
use App\Models\InstanceTraffic;
use App\Services\Proxmox\ProxmoxClient;
use App\Services\Traffic\TrafficMeter;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldBeUnique;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Log;
use Throwable;

/**
 * Samples every running instance's network counters and accumulates the month's
 * usage, then acts on the result: warn at the configured thresholds, throttle
 * once the allowance is gone.
 *
 * Proxmox reports netin/netout cumulatively since the VM last started, so the
 * usage is the difference between two samples. A counter that went backwards
 * means the VM restarted — the new value is then the usage since the restart,
 * not a refund.
 *
 * One instance failing must not cost the others their sample, so each is
 * wrapped individually.
 */
class CollectInstanceTraffic implements ShouldBeUnique, ShouldQueue
{
    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;

    /**
     * Comfortably longer than the 15-minute cadence. At exactly one interval the
     * lock would expire as the next run is dispatched, and two collectors
     * sharing a baseline would add the same delta twice — throttling a customer
     * for traffic they never used. The lock is released when the job finishes,
     * so a longer expiry only covers the crash case.
     */
    public int $uniqueFor = 3600;

    public int $tries = 1;

    public function __construct()
    {
        $this->onQueue('provisioning');
    }

    public function handle(ProxmoxClient $proxmox): void
    {
        $instances = Instance::query()
            ->with('host')
            ->whereNotNull('host_id')
            ->whereNotNull('vmid')
            ->where('status', 'active')
            ->get();

        foreach ($instances as $instance) {
            try {
                $this->sample($proxmox, $instance);
            } catch (Throwable $e) {
                Log::warning('traffic sample failed', [
                    'instance' => $instance->uuid, 'error' => $e->getMessage(),
                ]);
            }
        }
    }

    private function sample(ProxmoxClient $proxmox, Instance $instance): void
    {
        $client = $proxmox->forHost($instance->host);
        $status = $client->vmStatus($instance->host->node ?? 'pve', $instance->vmid);

        $netin = (int) ($status['netin'] ?? 0);
        $netout = (int) ($status['netout'] ?? 0);

        $row = InstanceTraffic::query()->firstOrCreate(
            ['instance_id' => $instance->id, 'period' => InstanceTraffic::currentPeriod()],
            ['last_netin' => $netin, 'last_netout' => $netout, 'sampled_at' => now()],
        );

        // Checked on every sample, not only when the row was just created: if
        // the release failed once — a Proxmox blip during the rollover run —
        // tying it to row creation would never try again, and the customer would
        // stay slow for a month they have paid for. The lookup exits
        // immediately when there is nothing to release.
        $this->releasePreviousPeriodThrottle($client, $instance);

        // A fresh row starts from this sample: whatever the VM transferred
        // before we started counting this month is not ours to bill.
        if (! $row->wasRecentlyCreated) {
            $row->rx_bytes += $this->delta($netin, $row->last_netin);
            $row->tx_bytes += $this->delta($netout, $row->last_netout);
            $row->last_netin = $netin;
            $row->last_netout = $netout;
            $row->sampled_at = now();
            $row->save();
        }

        $this->enforce($client, $instance, $row);
    }

    private function releasePreviousPeriodThrottle($client, Instance $instance): void
    {
        $previous = InstanceTraffic::query()
            ->where('instance_id', $instance->id)
            ->where('period', '!=', InstanceTraffic::currentPeriod())
            ->where('throttled', true)
            ->latest('period')
            ->first();

        if ($previous === null) {
            return;
        }

        $client->setNetworkRate($instance->host->node ?? 'pve', $instance->vmid, null);
        $previous->update(['throttled' => false, 'throttled_at' => null]);
        Log::info('throttle released for the new period', ['instance' => $instance->uuid]);
    }

    /** Counter went backwards → the VM restarted, so the current value is the delta. */
    private function delta(int $current, int $previous): int
    {
        return $current >= $previous ? $current - $previous : $current;
    }

    private function enforce($client, Instance $instance, InstanceTraffic $row): void
    {
        $meter = TrafficMeter::for($instance, $row);

        if ($meter->isExhausted() && ! $row->throttled) {
            // Throttle rather than cut off: a slow Nextcloud gets the customer to
            // buy more traffic, a dead one gets a cancellation.
            $kbps = (int) config('provisioning.traffic.throttle_kbps', 2048);
            $client->setNetworkRate($instance->host->node ?? 'pve', $instance->vmid, $kbps / 8 / 1024);

            $row->update(['throttled' => true, 'throttled_at' => now()]);
            Log::info('instance throttled for traffic', ['instance' => $instance->uuid]);

            return;
        }

        // Quota was topped up (or the month rolled over): give the line back.
        if ($row->throttled && ! $meter->isExhausted()) {
            $client->setNetworkRate($instance->host->node ?? 'pve', $instance->vmid, null);
            $row->update(['throttled' => false, 'throttled_at' => null]);

            return;
        }

        // Warn once per threshold crossed, not on every sample.
        foreach ((array) config('provisioning.traffic.warn_percent', [80, 95]) as $threshold) {
            if ($meter->percent() >= $threshold && $row->notified_percent < $threshold) {
                $row->update(['notified_percent' => $threshold]);
                Log::info('traffic threshold reached', [
                    'instance' => $instance->uuid, 'percent' => $meter->percent(),
                ]);
            }
        }
    }
}