onQueue('provisioning'); } public function handle(ProxmoxClient $proxmox): void { $instances = Instance::query() ->with('host') ->whereNotNull('host_id') ->whereNotNull('vmid') ->where('status', 'active') ->get(); foreach ($instances as $instance) { try { $this->sample($proxmox, $instance); } catch (Throwable $e) { Log::warning('traffic sample failed', [ 'instance' => $instance->uuid, 'error' => $e->getMessage(), ]); } } } private function sample(ProxmoxClient $proxmox, Instance $instance): void { $client = $proxmox->forHost($instance->host); $status = $client->vmStatus($instance->host->node ?? 'pve', $instance->vmid); // Missing telemetry is not "zero traffic": treating it as zero resets the // baseline, and the next real sample then adds the whole cumulative // counter again — a customer throttled for traffic counted twice. if (! isset($status['netin'], $status['netout'])) { Log::warning('traffic sample without counters, skipping', [ 'instance' => $instance->uuid, 'status' => $status['status'] ?? null, ]); return; } $netin = (int) $status['netin']; $netout = (int) $status['netout']; $row = InstanceTraffic::query()->firstOrCreate( ['instance_id' => $instance->id, 'period' => InstanceTraffic::currentPeriod()], ['last_netin' => $netin, 'last_netout' => $netout, 'sampled_at' => now()], ); // Checked on every sample, not only when the row was just created: if // the release failed once — a Proxmox blip during the rollover run — // tying it to row creation would never try again, and the customer would // stay slow for a month they have paid for. The lookup exits // immediately when there is nothing to release. $this->releasePreviousPeriodThrottle($client, $instance); // A fresh row starts from this sample: whatever the VM transferred // before we started counting this month is not ours to bill. $rxDelta = 0; $txDelta = 0; if (! $row->wasRecentlyCreated) { $rxDelta = $this->delta($netin, $row->last_netin); $txDelta = $this->delta($netout, $row->last_netout); $row->rx_bytes += $rxDelta; $row->tx_bytes += $txDelta; $row->last_netin = $netin; $row->last_netout = $netout; $row->sampled_at = now(); $row->save(); } $this->enforce($client, $instance, $row); $this->recordDay($client, $instance, $rxDelta ?? 0, $txDelta ?? 0); } /** * Today's row for the customer panel's chart. * * Kept apart from the billing row on purpose: instance_traffic answers "how * much of the allowance is gone this period", which must not be disturbed * by anything drawn on a screen. This answers "what did the last fortnight * look like", and a failure here must never cost someone their allowance — * hence the try/catch around a job that has already done its real work. */ private function recordDay($client, Instance $instance, int $rxDelta, int $txDelta): void { try { // Asked of the probe rather than read here, because the portal now // takes the same reading on demand — a customer who has just deleted // data has to be able to ask again without waiting for this round. // Two implementations of "how full is it" would eventually disagree, // and the downgrade check blocks on the answer. $disk = app(DiskUsageProbe::class)->read($instance); $metric = InstanceMetric::query()->firstOrCreate( ['instance_id' => $instance->id, 'day' => now()->toDateString()], ); // Accumulated, because the sampler runs several times a day. $metric->rx_bytes += max(0, $rxDelta); $metric->tx_bytes += max(0, $txDelta); // Overwritten rather than accumulated: a fill level is a state, not // a flow. And left untouched when the reading failed, so yesterday's // figure stands instead of the chart dropping to zero. if ($disk !== null) { $metric->disk_used_bytes = $disk['used']; $metric->disk_total_bytes = $disk['total']; } $metric->save(); } catch (Throwable $e) { Log::warning('daily metric failed', [ 'instance' => $instance->uuid, 'error' => $e->getMessage(), ]); } } private function releasePreviousPeriodThrottle($client, Instance $instance): void { $previous = InstanceTraffic::query() ->where('instance_id', $instance->id) ->where('period', '!=', InstanceTraffic::currentPeriod()) ->where('throttled', true) ->latest('period') ->first(); if ($previous === null) { return; } $client->setNetworkRate($instance->host->node ?? 'pve', $instance->vmid, null); $previous->update(['throttled' => false, 'throttled_at' => null]); Log::info('throttle released for the new period', ['instance' => $instance->uuid]); } /** Counter went backwards → the VM restarted, so the current value is the delta. */ private function delta(int $current, int $previous): int { return $current >= $previous ? $current - $previous : $current; } private function enforce($client, Instance $instance, InstanceTraffic $row): void { $meter = TrafficMeter::for($instance, $row); if ($meter->isExhausted() && ! $row->throttled) { // Throttle rather than cut off: a slow Nextcloud gets the customer to // buy more traffic, a dead one gets a cancellation. $kbps = (int) config('provisioning.traffic.throttle_kbps', 2048); $client->setNetworkRate($instance->host->node ?? 'pve', $instance->vmid, $kbps / 8 / 1024); $row->update(['throttled' => true, 'throttled_at' => now()]); Log::info('instance throttled for traffic', ['instance' => $instance->uuid]); return; } // Quota was topped up (or the month rolled over): give the line back. if ($row->throttled && ! $meter->isExhausted()) { $client->setNetworkRate($instance->host->node ?? 'pve', $instance->vmid, null); $row->update(['throttled' => false, 'throttled_at' => null]); return; } // Warn once per threshold crossed, not on every sample. foreach ((array) config('provisioning.traffic.warn_percent', [80, 95]) as $threshold) { if ($meter->percent() >= $threshold && $row->notified_percent < $threshold) { $row->update(['notified_percent' => $threshold]); Log::info('traffic threshold reached', [ 'instance' => $instance->uuid, 'percent' => $meter->percent(), ]); } } } }