226 lines
8.9 KiB
PHP
226 lines
8.9 KiB
PHP
<?php
|
|
|
|
namespace App\Provisioning\Jobs;
|
|
|
|
use App\Models\Instance;
|
|
use App\Models\InstanceMetric;
|
|
use App\Models\InstanceTraffic;
|
|
use App\Services\Provisioning\DiskUsageProbe;
|
|
use App\Services\Proxmox\ProxmoxClient;
|
|
use App\Services\Traffic\TrafficMeter;
|
|
use Illuminate\Bus\Queueable;
|
|
use Illuminate\Contracts\Queue\ShouldBeUnique;
|
|
use Illuminate\Contracts\Queue\ShouldQueue;
|
|
use Illuminate\Foundation\Bus\Dispatchable;
|
|
use Illuminate\Queue\InteractsWithQueue;
|
|
use Illuminate\Queue\SerializesModels;
|
|
use Illuminate\Support\Facades\Log;
|
|
use Throwable;
|
|
|
|
/**
|
|
* Samples every running instance's network counters and accumulates the month's
|
|
* usage, then acts on the result: warn at the configured thresholds, throttle
|
|
* once the allowance is gone.
|
|
*
|
|
* Proxmox reports netin/netout cumulatively since the VM last started, so the
|
|
* usage is the difference between two samples. A counter that went backwards
|
|
* means the VM restarted — the new value is then the usage since the restart,
|
|
* not a refund.
|
|
*
|
|
* One instance failing must not cost the others their sample, so each is
|
|
* wrapped individually.
|
|
*/
|
|
class CollectInstanceTraffic implements ShouldBeUnique, ShouldQueue
|
|
{
|
|
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
|
|
|
|
/**
|
|
* Comfortably longer than the 15-minute cadence. At exactly one interval the
|
|
* lock would expire as the next run is dispatched, and two collectors
|
|
* sharing a baseline would add the same delta twice — throttling a customer
|
|
* for traffic they never used. The lock is released when the job finishes,
|
|
* so a longer expiry only covers the crash case.
|
|
*/
|
|
public int $uniqueFor = 3600;
|
|
|
|
public int $tries = 1;
|
|
|
|
public function __construct()
|
|
{
|
|
$this->onQueue('provisioning');
|
|
}
|
|
|
|
public function handle(ProxmoxClient $proxmox): void
|
|
{
|
|
$instances = Instance::query()
|
|
->with('host')
|
|
->whereNotNull('host_id')
|
|
->whereNotNull('vmid')
|
|
->where('status', 'active')
|
|
->get();
|
|
|
|
foreach ($instances as $instance) {
|
|
try {
|
|
$this->sample($proxmox, $instance);
|
|
} catch (Throwable $e) {
|
|
Log::warning('traffic sample failed', [
|
|
'instance' => $instance->uuid, 'error' => $e->getMessage(),
|
|
]);
|
|
}
|
|
}
|
|
}
|
|
|
|
private function sample(ProxmoxClient $proxmox, Instance $instance): void
|
|
{
|
|
$client = $proxmox->forHost($instance->host);
|
|
$status = $client->vmStatus($instance->host->node ?? 'pve', $instance->vmid);
|
|
|
|
// Missing telemetry is not "zero traffic": treating it as zero resets the
|
|
// baseline, and the next real sample then adds the whole cumulative
|
|
// counter again — a customer throttled for traffic counted twice.
|
|
if (! isset($status['netin'], $status['netout'])) {
|
|
Log::warning('traffic sample without counters, skipping', [
|
|
'instance' => $instance->uuid, 'status' => $status['status'] ?? null,
|
|
]);
|
|
|
|
return;
|
|
}
|
|
|
|
$netin = (int) $status['netin'];
|
|
$netout = (int) $status['netout'];
|
|
|
|
$row = InstanceTraffic::query()->firstOrCreate(
|
|
['instance_id' => $instance->id, 'period' => InstanceTraffic::currentPeriod()],
|
|
['last_netin' => $netin, 'last_netout' => $netout, 'sampled_at' => now()],
|
|
);
|
|
|
|
// Checked on every sample, not only when the row was just created: if
|
|
// the release failed once — a Proxmox blip during the rollover run —
|
|
// tying it to row creation would never try again, and the customer would
|
|
// stay slow for a month they have paid for. The lookup exits
|
|
// immediately when there is nothing to release.
|
|
$this->releasePreviousPeriodThrottle($client, $instance);
|
|
|
|
// A fresh row starts from this sample: whatever the VM transferred
|
|
// before we started counting this month is not ours to bill.
|
|
$rxDelta = 0;
|
|
$txDelta = 0;
|
|
|
|
if (! $row->wasRecentlyCreated) {
|
|
$rxDelta = $this->delta($netin, $row->last_netin);
|
|
$txDelta = $this->delta($netout, $row->last_netout);
|
|
$row->rx_bytes += $rxDelta;
|
|
$row->tx_bytes += $txDelta;
|
|
$row->last_netin = $netin;
|
|
$row->last_netout = $netout;
|
|
$row->sampled_at = now();
|
|
$row->save();
|
|
}
|
|
|
|
$this->enforce($client, $instance, $row);
|
|
$this->recordDay($client, $instance, $rxDelta ?? 0, $txDelta ?? 0);
|
|
}
|
|
|
|
/**
|
|
* Today's row for the customer panel's chart.
|
|
*
|
|
* Kept apart from the billing row on purpose: instance_traffic answers "how
|
|
* much of the allowance is gone this period", which must not be disturbed
|
|
* by anything drawn on a screen. This answers "what did the last fortnight
|
|
* look like", and a failure here must never cost someone their allowance —
|
|
* hence the try/catch around a job that has already done its real work.
|
|
*/
|
|
private function recordDay($client, Instance $instance, int $rxDelta, int $txDelta): void
|
|
{
|
|
try {
|
|
// Asked of the probe rather than read here, because the portal now
|
|
// takes the same reading on demand — a customer who has just deleted
|
|
// data has to be able to ask again without waiting for this round.
|
|
// Two implementations of "how full is it" would eventually disagree,
|
|
// and the downgrade check blocks on the answer.
|
|
$disk = app(DiskUsageProbe::class)->read($instance);
|
|
|
|
$metric = InstanceMetric::query()->firstOrCreate(
|
|
['instance_id' => $instance->id, 'day' => now()->toDateString()],
|
|
);
|
|
|
|
// Accumulated, because the sampler runs several times a day.
|
|
$metric->rx_bytes += max(0, $rxDelta);
|
|
$metric->tx_bytes += max(0, $txDelta);
|
|
|
|
// Overwritten rather than accumulated: a fill level is a state, not
|
|
// a flow. And left untouched when the reading failed, so yesterday's
|
|
// figure stands instead of the chart dropping to zero.
|
|
if ($disk !== null) {
|
|
$metric->disk_used_bytes = $disk['used'];
|
|
$metric->disk_total_bytes = $disk['total'];
|
|
}
|
|
|
|
$metric->save();
|
|
} catch (Throwable $e) {
|
|
Log::warning('daily metric failed', [
|
|
'instance' => $instance->uuid, 'error' => $e->getMessage(),
|
|
]);
|
|
}
|
|
}
|
|
|
|
private function releasePreviousPeriodThrottle($client, Instance $instance): void
|
|
{
|
|
$previous = InstanceTraffic::query()
|
|
->where('instance_id', $instance->id)
|
|
->where('period', '!=', InstanceTraffic::currentPeriod())
|
|
->where('throttled', true)
|
|
->latest('period')
|
|
->first();
|
|
|
|
if ($previous === null) {
|
|
return;
|
|
}
|
|
|
|
$client->setNetworkRate($instance->host->node ?? 'pve', $instance->vmid, null);
|
|
$previous->update(['throttled' => false, 'throttled_at' => null]);
|
|
Log::info('throttle released for the new period', ['instance' => $instance->uuid]);
|
|
}
|
|
|
|
/** Counter went backwards → the VM restarted, so the current value is the delta. */
|
|
private function delta(int $current, int $previous): int
|
|
{
|
|
return $current >= $previous ? $current - $previous : $current;
|
|
}
|
|
|
|
private function enforce($client, Instance $instance, InstanceTraffic $row): void
|
|
{
|
|
$meter = TrafficMeter::for($instance, $row);
|
|
|
|
if ($meter->isExhausted() && ! $row->throttled) {
|
|
// Throttle rather than cut off: a slow Nextcloud gets the customer to
|
|
// buy more traffic, a dead one gets a cancellation.
|
|
$kbps = (int) config('provisioning.traffic.throttle_kbps', 2048);
|
|
$client->setNetworkRate($instance->host->node ?? 'pve', $instance->vmid, $kbps / 8 / 1024);
|
|
|
|
$row->update(['throttled' => true, 'throttled_at' => now()]);
|
|
Log::info('instance throttled for traffic', ['instance' => $instance->uuid]);
|
|
|
|
return;
|
|
}
|
|
|
|
// Quota was topped up (or the month rolled over): give the line back.
|
|
if ($row->throttled && ! $meter->isExhausted()) {
|
|
$client->setNetworkRate($instance->host->node ?? 'pve', $instance->vmid, null);
|
|
$row->update(['throttled' => false, 'throttled_at' => null]);
|
|
|
|
return;
|
|
}
|
|
|
|
// Warn once per threshold crossed, not on every sample.
|
|
foreach ((array) config('provisioning.traffic.warn_percent', [80, 95]) as $threshold) {
|
|
if ($meter->percent() >= $threshold && $row->notified_percent < $threshold) {
|
|
$row->update(['notified_percent' => $threshold]);
|
|
Log::info('traffic threshold reached', [
|
|
'instance' => $instance->uuid, 'percent' => $meter->percent(),
|
|
]);
|
|
}
|
|
}
|
|
}
|
|
}
|