fix(engine): fail on unresolvable pipeline step; make worker the WG hub

- RunRunner catches step-resolution errors (removed/renamed pipeline) and
  fails the run terminally instead of looping forever every tick.
- queue-provisioning worker gains NET_ADMIN + tun + persistent wireguard
  volume + published UDP port so LocalWireguardHub can manage wg0.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
feat/portal-design
nexxo 2026-07-25 10:53:40 +02:00
parent 00b8897b9d
commit 93191f5de9
4 changed files with 45 additions and 13 deletions

View File

@ -61,7 +61,15 @@ class RunRunner
$run->status = ProvisioningRun::STATUS_RUNNING;
$run->save();
$step = $this->registry->resolve($run->pipeline, $run->current_step);
// A missing/renamed pipeline or step (e.g. after a deploy) is fatal, not
// a transient error — fail terminally instead of looping forever.
try {
$step = $this->registry->resolve($run->pipeline, $run->current_step);
} catch (Throwable $e) {
$this->failRun($run, 'pipeline_resolution', 'cannot resolve step: '.$e->getMessage());
return;
}
$timedOut = $run->started_at !== null
&& $run->started_at->copy()->addSeconds($step->maxDuration())->isPast();
@ -83,7 +91,7 @@ class RunRunner
StepResult::ADVANCE => $this->onAdvance($run, $step),
StepResult::RETRY => $this->onRetry($run, $step, $result),
StepResult::POLL => $this->onPoll($run, $step, $result),
StepResult::FAIL => $this->failRun($run, $step, $result->reason),
StepResult::FAIL => $this->failRun($run, $step->key(), $result->reason),
};
}
@ -96,7 +104,7 @@ class RunRunner
$run->finished_at = now();
$run->attempt = 0;
$run->save();
$this->record($run, $step, 'advanced', null);
$this->record($run, $step->key(), 'advanced', null);
return;
}
@ -106,7 +114,7 @@ class RunRunner
$run->started_at = now();
$run->status = ProvisioningRun::STATUS_RUNNING;
$run->save();
$this->record($run, $step, 'advanced', null);
$this->record($run, $step->key(), 'advanced', null);
AdvanceRunJob::dispatch($run->uuid);
}
@ -116,7 +124,7 @@ class RunRunner
$run->attempt += 1;
if ($run->attempt >= $run->max_attempts) {
$this->failRun($run, $step, $result->reason);
$this->failRun($run, $step->key(), $result->reason);
return;
}
@ -125,7 +133,7 @@ class RunRunner
$run->next_attempt_at = now()->addSeconds($result->afterSeconds);
$run->started_at = now(); // fresh per-attempt timer, else a timed-out step re-times-out
$run->save();
$this->record($run, $step, 'retry', $result->reason);
$this->record($run, $step->key(), 'retry', $result->reason);
}
/** Poll: wait and re-run without consuming the retry budget (step owns its deadline). */
@ -135,15 +143,15 @@ class RunRunner
$run->next_attempt_at = now()->addSeconds($result->afterSeconds);
$run->started_at = now();
$run->save();
$this->record($run, $step, 'info', $result->reason);
$this->record($run, $step->key(), 'info', $result->reason);
}
private function failRun(ProvisioningRun $run, ProvisioningStep $step, string $reason): void
private function failRun(ProvisioningRun $run, string $stepKey, string $reason): void
{
$run->status = ProvisioningRun::STATUS_FAILED;
$run->error = $reason;
$run->save();
$this->record($run, $step, 'failed', $reason);
$this->record($run, $stepKey, 'failed', $reason);
// Let the subject react (e.g. a Host moves to the 'error' status).
$subject = $run->subject;
@ -152,16 +160,16 @@ class RunRunner
}
}
private function record(ProvisioningRun $run, ProvisioningStep $step, string $outcome, ?string $message): void
private function record(ProvisioningRun $run, string $stepKey, string $outcome, ?string $message): void
{
$run->events()->create([
'step' => $step->key(),
'step' => $stepKey,
'attempt' => $run->attempt,
'outcome' => $outcome,
'message' => $message,
]);
StepAdvanced::dispatch($run->uuid, $step->key(), $outcome, $run->current_step, $run->status);
StepAdvanced::dispatch($run->uuid, $stepKey, $outcome, $run->current_step, $run->status);
}
private function backoff(int $attempt): int

View File

@ -54,12 +54,25 @@ services:
# Dedicated worker for long-running provisioning steps (own timeout, single try;
# the DB state machine owns retries). Separate from the fast default queue.
# It also acts as the WireGuard hub (LocalWireguardHub runs `wg set wg0` here),
# so it needs NET_ADMIN, the tun device, a persistent wg config, and the WG
# UDP port published on the host for peers to reach. Set CLUPILOT_WG_ENDPOINT
# to <vm-public-ip>:${WG_HUB_PORT} and CLUPILOT_WG_HUB_PUBKEY to wg0's key.
queue-provisioning:
image: clupilot-app:dev
restart: unless-stopped
command: php artisan queue:work provisioning --queue=provisioning --tries=1 --timeout=2100 --sleep=3
cap_add:
- NET_ADMIN
devices:
- /dev/net/tun:/dev/net/tun
sysctls:
- net.ipv4.ip_forward=1
ports:
- "${WG_HUB_PORT:-51820}:51820/udp"
volumes:
- .:/var/www/html
- wireguard:/etc/wireguard
depends_on:
- app
- redis
@ -105,3 +118,4 @@ services:
volumes:
db-data:
redis-data:
wireguard:

View File

@ -214,7 +214,7 @@ Routen englisch (R13). Alle Texte DE+EN identische Keys (R16). Nur Token-Utiliti
## 11. Watch-Items
1. **WG-Hub-Schreibweg aus dem Container:** App läuft im Docker-`app`-Container, der WG-Hub auf der VM. Realen Schreib-/Reload-Weg festlegen (gemounteter Config-Pfad + `wg syncconf`, oder Hub-Service). v1.0 hinter `WireguardHub` abstrahiert; reale Verdrahtung markiert.
1. **WG-Hub-Schreibweg aus dem Container:** Gelöst — der `queue-provisioning`-Worker-Container ist der WG-Hub (`cap_add: NET_ADMIN`, `/dev/net/tun`, persistentes `wireguard`-Volume, UDP-Port veröffentlicht). `LocalWireguardHub` führt `wg set wg0` dort aus. Vor echtem Betrieb: `CLUPILOT_WG_ENDPOINT`=`<vm-ip>:51820` und `CLUPILOT_WG_HUB_PUBKEY`=wg0-Key setzen, wg0 im Container hochfahren.
2. **Debian-Kernel-Entfernung** darf den Host nicht bricken — dokumentierte Befehlssequenz, **echte** Verifikation nötig (mocked baut die Sequenz nach).
3. **Reboot-Polling-Deadlines** großzügig (Reboot + Kernelwechsel).
4. **`apt full-upgrade`-Dauer** → großzügiges Step-Timeout.

View File

@ -146,6 +146,16 @@ it('re-executes a step after a timeout retry instead of timing out again', funct
expect($run->fresh()->status)->toBe('completed');
});
it('fails terminally when the pipeline step cannot be resolved', function () {
bindPipeline(['test' => [FakeAdvanceStep::class]]);
$run = ProvisioningRun::factory()->create(['pipeline' => 'test', 'current_step' => 9]);
app(RunRunner::class)->advance($run);
expect($run->fresh()->status)->toBe('failed')
->and($run->fresh()->events()->where('step', 'pipeline_resolution')->exists())->toBeTrue();
});
it('is a no-op while the run lock is held', function () {
bindPipeline(['test' => [FakeAdvanceStep::class, FakeAdvanceStep::class]]);
$run = ProvisioningRun::factory()->create(['pipeline' => 'test', 'current_step' => 0]);