fix(engine): fail on unresolvable pipeline step; make worker the WG hub
- RunRunner catches step-resolution errors (removed/renamed pipeline) and fails the run terminally instead of looping forever every tick. - queue-provisioning worker gains NET_ADMIN + tun + persistent wireguard volume + published UDP port so LocalWireguardHub can manage wg0. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>feat/portal-design
parent
00b8897b9d
commit
93191f5de9
|
|
@ -61,7 +61,15 @@ class RunRunner
|
||||||
$run->status = ProvisioningRun::STATUS_RUNNING;
|
$run->status = ProvisioningRun::STATUS_RUNNING;
|
||||||
$run->save();
|
$run->save();
|
||||||
|
|
||||||
$step = $this->registry->resolve($run->pipeline, $run->current_step);
|
// A missing/renamed pipeline or step (e.g. after a deploy) is fatal, not
|
||||||
|
// a transient error — fail terminally instead of looping forever.
|
||||||
|
try {
|
||||||
|
$step = $this->registry->resolve($run->pipeline, $run->current_step);
|
||||||
|
} catch (Throwable $e) {
|
||||||
|
$this->failRun($run, 'pipeline_resolution', 'cannot resolve step: '.$e->getMessage());
|
||||||
|
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
$timedOut = $run->started_at !== null
|
$timedOut = $run->started_at !== null
|
||||||
&& $run->started_at->copy()->addSeconds($step->maxDuration())->isPast();
|
&& $run->started_at->copy()->addSeconds($step->maxDuration())->isPast();
|
||||||
|
|
@ -83,7 +91,7 @@ class RunRunner
|
||||||
StepResult::ADVANCE => $this->onAdvance($run, $step),
|
StepResult::ADVANCE => $this->onAdvance($run, $step),
|
||||||
StepResult::RETRY => $this->onRetry($run, $step, $result),
|
StepResult::RETRY => $this->onRetry($run, $step, $result),
|
||||||
StepResult::POLL => $this->onPoll($run, $step, $result),
|
StepResult::POLL => $this->onPoll($run, $step, $result),
|
||||||
StepResult::FAIL => $this->failRun($run, $step, $result->reason),
|
StepResult::FAIL => $this->failRun($run, $step->key(), $result->reason),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -96,7 +104,7 @@ class RunRunner
|
||||||
$run->finished_at = now();
|
$run->finished_at = now();
|
||||||
$run->attempt = 0;
|
$run->attempt = 0;
|
||||||
$run->save();
|
$run->save();
|
||||||
$this->record($run, $step, 'advanced', null);
|
$this->record($run, $step->key(), 'advanced', null);
|
||||||
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
@ -106,7 +114,7 @@ class RunRunner
|
||||||
$run->started_at = now();
|
$run->started_at = now();
|
||||||
$run->status = ProvisioningRun::STATUS_RUNNING;
|
$run->status = ProvisioningRun::STATUS_RUNNING;
|
||||||
$run->save();
|
$run->save();
|
||||||
$this->record($run, $step, 'advanced', null);
|
$this->record($run, $step->key(), 'advanced', null);
|
||||||
|
|
||||||
AdvanceRunJob::dispatch($run->uuid);
|
AdvanceRunJob::dispatch($run->uuid);
|
||||||
}
|
}
|
||||||
|
|
@ -116,7 +124,7 @@ class RunRunner
|
||||||
$run->attempt += 1;
|
$run->attempt += 1;
|
||||||
|
|
||||||
if ($run->attempt >= $run->max_attempts) {
|
if ($run->attempt >= $run->max_attempts) {
|
||||||
$this->failRun($run, $step, $result->reason);
|
$this->failRun($run, $step->key(), $result->reason);
|
||||||
|
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
@ -125,7 +133,7 @@ class RunRunner
|
||||||
$run->next_attempt_at = now()->addSeconds($result->afterSeconds);
|
$run->next_attempt_at = now()->addSeconds($result->afterSeconds);
|
||||||
$run->started_at = now(); // fresh per-attempt timer, else a timed-out step re-times-out
|
$run->started_at = now(); // fresh per-attempt timer, else a timed-out step re-times-out
|
||||||
$run->save();
|
$run->save();
|
||||||
$this->record($run, $step, 'retry', $result->reason);
|
$this->record($run, $step->key(), 'retry', $result->reason);
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Poll: wait and re-run without consuming the retry budget (step owns its deadline). */
|
/** Poll: wait and re-run without consuming the retry budget (step owns its deadline). */
|
||||||
|
|
@ -135,15 +143,15 @@ class RunRunner
|
||||||
$run->next_attempt_at = now()->addSeconds($result->afterSeconds);
|
$run->next_attempt_at = now()->addSeconds($result->afterSeconds);
|
||||||
$run->started_at = now();
|
$run->started_at = now();
|
||||||
$run->save();
|
$run->save();
|
||||||
$this->record($run, $step, 'info', $result->reason);
|
$this->record($run, $step->key(), 'info', $result->reason);
|
||||||
}
|
}
|
||||||
|
|
||||||
private function failRun(ProvisioningRun $run, ProvisioningStep $step, string $reason): void
|
private function failRun(ProvisioningRun $run, string $stepKey, string $reason): void
|
||||||
{
|
{
|
||||||
$run->status = ProvisioningRun::STATUS_FAILED;
|
$run->status = ProvisioningRun::STATUS_FAILED;
|
||||||
$run->error = $reason;
|
$run->error = $reason;
|
||||||
$run->save();
|
$run->save();
|
||||||
$this->record($run, $step, 'failed', $reason);
|
$this->record($run, $stepKey, 'failed', $reason);
|
||||||
|
|
||||||
// Let the subject react (e.g. a Host moves to the 'error' status).
|
// Let the subject react (e.g. a Host moves to the 'error' status).
|
||||||
$subject = $run->subject;
|
$subject = $run->subject;
|
||||||
|
|
@ -152,16 +160,16 @@ class RunRunner
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private function record(ProvisioningRun $run, ProvisioningStep $step, string $outcome, ?string $message): void
|
private function record(ProvisioningRun $run, string $stepKey, string $outcome, ?string $message): void
|
||||||
{
|
{
|
||||||
$run->events()->create([
|
$run->events()->create([
|
||||||
'step' => $step->key(),
|
'step' => $stepKey,
|
||||||
'attempt' => $run->attempt,
|
'attempt' => $run->attempt,
|
||||||
'outcome' => $outcome,
|
'outcome' => $outcome,
|
||||||
'message' => $message,
|
'message' => $message,
|
||||||
]);
|
]);
|
||||||
|
|
||||||
StepAdvanced::dispatch($run->uuid, $step->key(), $outcome, $run->current_step, $run->status);
|
StepAdvanced::dispatch($run->uuid, $stepKey, $outcome, $run->current_step, $run->status);
|
||||||
}
|
}
|
||||||
|
|
||||||
private function backoff(int $attempt): int
|
private function backoff(int $attempt): int
|
||||||
|
|
|
||||||
|
|
@ -54,12 +54,25 @@ services:
|
||||||
|
|
||||||
# Dedicated worker for long-running provisioning steps (own timeout, single try;
|
# Dedicated worker for long-running provisioning steps (own timeout, single try;
|
||||||
# the DB state machine owns retries). Separate from the fast default queue.
|
# the DB state machine owns retries). Separate from the fast default queue.
|
||||||
|
# It also acts as the WireGuard hub (LocalWireguardHub runs `wg set wg0` here),
|
||||||
|
# so it needs NET_ADMIN, the tun device, a persistent wg config, and the WG
|
||||||
|
# UDP port published on the host for peers to reach. Set CLUPILOT_WG_ENDPOINT
|
||||||
|
# to <vm-public-ip>:${WG_HUB_PORT} and CLUPILOT_WG_HUB_PUBKEY to wg0's key.
|
||||||
queue-provisioning:
|
queue-provisioning:
|
||||||
image: clupilot-app:dev
|
image: clupilot-app:dev
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
command: php artisan queue:work provisioning --queue=provisioning --tries=1 --timeout=2100 --sleep=3
|
command: php artisan queue:work provisioning --queue=provisioning --tries=1 --timeout=2100 --sleep=3
|
||||||
|
cap_add:
|
||||||
|
- NET_ADMIN
|
||||||
|
devices:
|
||||||
|
- /dev/net/tun:/dev/net/tun
|
||||||
|
sysctls:
|
||||||
|
- net.ipv4.ip_forward=1
|
||||||
|
ports:
|
||||||
|
- "${WG_HUB_PORT:-51820}:51820/udp"
|
||||||
volumes:
|
volumes:
|
||||||
- .:/var/www/html
|
- .:/var/www/html
|
||||||
|
- wireguard:/etc/wireguard
|
||||||
depends_on:
|
depends_on:
|
||||||
- app
|
- app
|
||||||
- redis
|
- redis
|
||||||
|
|
@ -105,3 +118,4 @@ services:
|
||||||
volumes:
|
volumes:
|
||||||
db-data:
|
db-data:
|
||||||
redis-data:
|
redis-data:
|
||||||
|
wireguard:
|
||||||
|
|
|
||||||
|
|
@ -214,7 +214,7 @@ Routen englisch (R13). Alle Texte DE+EN identische Keys (R16). Nur Token-Utiliti
|
||||||
|
|
||||||
## 11. Watch-Items
|
## 11. Watch-Items
|
||||||
|
|
||||||
1. **WG-Hub-Schreibweg aus dem Container:** App läuft im Docker-`app`-Container, der WG-Hub auf der VM. Realen Schreib-/Reload-Weg festlegen (gemounteter Config-Pfad + `wg syncconf`, oder Hub-Service). v1.0 hinter `WireguardHub` abstrahiert; reale Verdrahtung markiert.
|
1. **WG-Hub-Schreibweg aus dem Container:** Gelöst — der `queue-provisioning`-Worker-Container ist der WG-Hub (`cap_add: NET_ADMIN`, `/dev/net/tun`, persistentes `wireguard`-Volume, UDP-Port veröffentlicht). `LocalWireguardHub` führt `wg set wg0` dort aus. Vor echtem Betrieb: `CLUPILOT_WG_ENDPOINT`=`<vm-ip>:51820` und `CLUPILOT_WG_HUB_PUBKEY`=wg0-Key setzen, wg0 im Container hochfahren.
|
||||||
2. **Debian-Kernel-Entfernung** darf den Host nicht bricken — dokumentierte Befehlssequenz, **echte** Verifikation nötig (mocked baut die Sequenz nach).
|
2. **Debian-Kernel-Entfernung** darf den Host nicht bricken — dokumentierte Befehlssequenz, **echte** Verifikation nötig (mocked baut die Sequenz nach).
|
||||||
3. **Reboot-Polling-Deadlines** großzügig (Reboot + Kernelwechsel).
|
3. **Reboot-Polling-Deadlines** großzügig (Reboot + Kernelwechsel).
|
||||||
4. **`apt full-upgrade`-Dauer** → großzügiges Step-Timeout.
|
4. **`apt full-upgrade`-Dauer** → großzügiges Step-Timeout.
|
||||||
|
|
|
||||||
|
|
@ -146,6 +146,16 @@ it('re-executes a step after a timeout retry instead of timing out again', funct
|
||||||
expect($run->fresh()->status)->toBe('completed');
|
expect($run->fresh()->status)->toBe('completed');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it('fails terminally when the pipeline step cannot be resolved', function () {
|
||||||
|
bindPipeline(['test' => [FakeAdvanceStep::class]]);
|
||||||
|
$run = ProvisioningRun::factory()->create(['pipeline' => 'test', 'current_step' => 9]);
|
||||||
|
|
||||||
|
app(RunRunner::class)->advance($run);
|
||||||
|
|
||||||
|
expect($run->fresh()->status)->toBe('failed')
|
||||||
|
->and($run->fresh()->events()->where('step', 'pipeline_resolution')->exists())->toBeTrue();
|
||||||
|
});
|
||||||
|
|
||||||
it('is a no-op while the run lock is held', function () {
|
it('is a no-op while the run lock is held', function () {
|
||||||
bindPipeline(['test' => [FakeAdvanceStep::class, FakeAdvanceStep::class]]);
|
bindPipeline(['test' => [FakeAdvanceStep::class, FakeAdvanceStep::class]]);
|
||||||
$run = ProvisioningRun::factory()->create(['pipeline' => 'test', 'current_step' => 0]);
|
$run = ProvisioningRun::factory()->create(['pipeline' => 'test', 'current_step' => 0]);
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue