CluPilotCloud/app/Services/Deployment/UpdateChannel.php

604 lines
24 KiB
PHP

<?php
namespace App\Services\Deployment;
use Illuminate\Support\Carbon;
use Illuminate\Support\Facades\File;
use Illuminate\Support\Facades\Lang;
use Throwable;
/**
* The panel's side of "update this installation".
*
* The panel cannot update anything itself, and it should not be able to. It
* runs as www-data inside a container; the update runs on the host, as the
* service account, and restarts the very container the request came from. A
* button that shelled out would be a button that kills its own web server
* mid-response — and it would need the app to hold host-level credentials,
* which is a far worse thing to own than an out-of-date checkout.
*
* So this is a mailbox, not a command:
*
* panel → writes a request into storage/app/deploy/
* agent → (deploy/update-agent.sh, on the host, on a timer) consumes it,
* runs deploy/update.sh, writes back what happened
* panel → reads the result
*
* The directory is inside the bind-mounted checkout, so both sides see it
* without anything being exposed over the network.
*
* The request carries a KIND, not a flag: "check" (look only — the agent
* already fetches on every tick regardless, this just answers sooner than the
* next one would on its own), "run" (apply it, downtime and all), or
* "restart" (queue, queue-provisioning, scheduler, reverb only — picking up a
* freshly-saved .env without a deployment). Three different things an
* operator can ask for, not one request with a modifier — an operator who
* only wants to know whether anything is new must not be routed through a
* real deployment to find out, and one who only changed a credential must not
* be routed through one either.
*
* Everything here degrades to "unknown" rather than throwing. A broken or
* absent status file must leave the settings page readable — it is the page an
* operator opens when something is already wrong.
*/
final class UpdateChannel
{
/** Written by the panel, consumed by the agent. */
private const REQUEST = 'deploy/update-request.json';
/** A request the agent only checks against, and never acts on. */
private const KIND_CHECK = 'check';
/**
* A request the agent actually applies.
*
* Also the fallback for a request with no `kind` at all — every request
* meant this before "check" existed, so an older panel's request (or one
* still in flight across a deploy of this very feature) must keep meaning
* it rather than silently doing nothing.
*/
private const KIND_RUN = 'run';
/**
* A request to restart the four workers that only read .env at their own
* startup — queue, queue-provisioning, scheduler, reverb — and nothing
* else. Not a deployment: no fetch is consulted, no maintenance mode, no
* release is involved. `app` is deliberately never in that list — see
* update-agent.sh's handling of this kind for why.
*/
private const KIND_RESTART = 'restart';
/**
* A request to mint an SSH key that may do exactly one thing: read the
* invoice archive over rsync.
*
* The panel cannot do any of it. Generating a keypair, installing the public
* half restricted to one directory and handing over the private half all
* happen on the HOST, in the service account's own home — and the panel is
* www-data inside a container that owns none of that. So it asks, like it
* asks for everything else here.
*/
private const KIND_ARCHIVE_KEY = 'archive-key';
/**
* Die Hostnamen aus der Konsole in die Proxy-Konfiguration übernehmen.
*
* Getrennt von `run`, weil hier NICHTS aktualisiert wird: kein Wartungsmodus,
* kein Neustart, kein Anfassen des Auscheckens. Der Agent ruft eine feste
* Befehlszeile des root-eigenen Helfers auf, der die Liste liest, sie gegen
* ein strenges Muster prüft und daraus seine eigene Vorlage füllt.
*
* Die Liste, die er liest, kommt aus `php artisan clupilot:proxy-hosts` —
* NAMEN, keine Caddy-Blöcke. Dürfte diese Anwendung fertige Konfiguration
* schreiben, die ein root-Helfer ungeprüft einbaut, hätte sie damit alles,
* was der Proxy kann. `install-agent.sh` sagt es kürzer: eine Freigabe ist
* nur etwas wert, wenn der Inhaber nicht ändern kann, was sie gewährt.
*/
private const KIND_PROXY_HOSTS = 'proxy-hosts';
/**
* Where the agent leaves a freshly minted key, once.
*
* Read and deleted by the panel in one go: the private half exists on disk
* for the seconds between the agent writing it and somebody looking at it,
* and not a minute longer. Same shape as the initial admin password an
* instance holds until it is acknowledged.
*/
private const ARCHIVE_KEY = 'deploy/archive-key.json';
/** Written by the agent after every check and every run. */
private const STATUS = 'deploy/update-status.json';
/**
* The outcome of the last actual run, kept apart from the periodic check.
*
* In one file the next idle check would overwrite a failure a minute
* later, and an operator would usually never learn that the update failed.
*/
private const LAST_RUN = 'deploy/update-last-run.json';
/** Tail of the last run, for the operator to read without shell access. */
private const LOG = 'deploy/update-last-run.log';
/**
* The outcome of the last worker RESTART, kept apart from LAST_RUN.
*
* A restart is not a deployment and update.sh never runs for it, so it
* gets its own tiny record rather than borrowing the deployment's — the
* two must be able to fail independently and be reported independently.
*/
private const RESTART_LAST_RUN = 'deploy/restart-last-run.json';
/**
* The step deploy/update.sh is on, written by update.sh itself.
*
* Read from here rather than from the status file, and that is the whole
* point: the agent writes the status once, then blocks for the length of the
* run. Anything taken from the status file is therefore frozen at "the run
* started" and the step would never advance while it mattered. update.sh
* rewrites this file at every step, so it is the only live source.
*
* Two tab-separated fields: the step key, and when it started.
*/
private const PHASE = 'deploy/update-phase';
/**
* A request older than this is treated as abandoned.
*
* Without it a request written while the agent was stopped would fire the
* moment the agent came back, which could be days later and no longer what
* anyone wanted.
*/
private const REQUEST_EXPIRES_MINUTES = 30;
/**
* How stale the agent's last check may be before it counts as gone.
*
* The timer runs every minute; twenty missed ticks in a row is a stopped
* agent, not a slow one — comfortably more than one bad tick (a hung
* fetch, a loaded host) can account for. Without this, a single status
* file written once — by an agent since disabled, broken or uninstalled —
* leaves the button enabled forever, and pressing it writes a request
* nobody will ever collect while the page cheerfully reports an update in
* progress.
*/
private const AGENT_STALE_AFTER_MINUTES = 20;
/**
* What the panel needs to show, in one read.
*
* @return array<string, mixed>
*/
public function state(): array
{
$release = Release::current();
$status = $this->readJson(self::STATUS);
$lastRun = $this->readJson(self::LAST_RUN);
$restartLastRun = $this->readJson(self::RESTART_LAST_RUN);
$request = $this->pendingRequest();
$checkedAt = $this->timestamp($status['checked_at'] ?? null);
$agentAlive = $this->agentIsAlive($status);
// Missing on a request an older panel wrote — see KIND_RUN.
$requestKind = $request['kind'] ?? self::KIND_RUN;
// A figure from an agent that has since stopped is not current, and
// "three updates behind" from last week reads exactly like now.
$behind = $agentAlive && isset($status['behind']) ? (int) $status['behind'] : null;
return [
'version' => $release->version,
'commit' => $release->commit,
'source' => $release->source,
'deployed_at' => $release->deployedAt,
// Null means "we do not know", which is not the same as "up to
// date" and must not be shown as it.
'behind' => $behind,
'available' => $behind !== null && $behind > 0,
// The tag an update would install, e.g. "v1.1.0". An update is
// always to a released version now — a commit landing on main is
// not an update, and the console has nothing to say about it.
'target_release' => $agentAlive && ! empty($status['target_release'])
? (string) $status['target_release']
: null,
'remote_commit' => isset($status['remote_commit']) ? (string) $status['remote_commit'] : null,
'checked_at' => $checkedAt,
'agent_seen' => $agentAlive,
// A pending CHECK or RESTART must never read as "running": nothing
// is being deployed, the site never enters maintenance mode, and
// the full-screen overlay (layouts/admin.blade.php) exists only
// for the case where it does. A worker restart used to slip
// through this check by only excluding "check" — any kind that
// was not a check counted as "running" — which would have shown
// an operator saving .env the same maintenance screen as a real
// deployment, for four containers that come back in seconds.
// Matched on KIND_RUN explicitly instead, so a future kind added
// here has to opt IN to the overlay rather than getting it by
// accident.
'running' => $agentAlive && (
($request !== null && $requestKind === self::KIND_RUN)
|| ($status['state'] ?? null) === 'running'
),
// A pending check, still waiting for the agent to pick it up.
'checking' => $agentAlive && $request !== null && $requestKind === self::KIND_CHECK,
// A pending — or in-flight — worker restart. The agent handles a
// restart within the same tick it consumes the request (no
// maintenance mode, no phase file), so this is only ever true for
// the short window before that tick runs; see RESTART_LAST_RUN
// below for what happened once it has.
'restarting' => $agentAlive && $request !== null && $requestKind === self::KIND_RESTART,
'requested_at' => $request !== null ? $this->timestamp($request['requested_at'] ?? null) : null,
'requested_by' => $request['requested_by'] ?? null,
// Is the agent woken the moment a request lands, or does the
// request wait for the next tick?
//
// Reported by the agent rather than decided here: the trigger is a
// systemd path unit on the host (deploy/install-agent.sh), which
// the panel cannot see from inside its container, and a server
// that has not re-run install-agent.sh since this landed still
// waits for the timer. It decides whether the console may say
// "now" or has to name a time — and promising "now" on an
// installation that will take five minutes is the failure this
// whole field exists to avoid.
'instant' => $agentAlive && ($status['request_watch'] ?? null) === true,
// The step the deployment script is on — a key, translated here.
// Read live from the phase file; older agents write none, so its
// absence is normal rather than a fault.
'phase' => $this->phase($this->currentPhase($status)),
'started_at' => $this->timestamp($status['started_at'] ?? null),
'last_state' => $lastRun['state'] ?? null,
'last_phase' => $this->phase($lastRun['phase'] ?? null),
'last_started_at' => $this->timestamp($lastRun['started_at'] ?? null),
'last_finished_at' => $this->timestamp($lastRun['finished_at'] ?? null),
// The run's own failure first; a check-level problem (the
// repository being unreachable) only when the last run was fine.
'last_error' => $this->errorMessage($lastRun) ?? $this->errorMessage($status),
// The last worker restart, independent of the deployment fields
// above — see RESTART_LAST_RUN.
'last_restart_state' => $restartLastRun['state'] ?? null,
'last_restart_finished_at' => $this->timestamp($restartLastRun['finished_at'] ?? null),
'last_restart_error' => $this->errorMessage($restartLastRun),
];
}
/**
* The deployment step, as something a German interface can print.
*
* The script writes a key precisely so this stays translatable. An unknown
* key — a newer script against older translations — yields null rather than
* a raw identifier in the middle of a sentence.
*/
private function phase(mixed $key): ?string
{
if (! is_string($key) || $key === '') {
return null;
}
$line = __('admin_settings.update_phase.'.$key);
return is_string($line) && $line !== 'admin_settings.update_phase.'.$key ? $line : null;
}
/**
* The step key update.sh last announced, or null.
*
* Its own tiny file rather than a field in the status document: update.sh
* runs as the service account on the host and rewrites this at every step,
* while the agent that owns the status document is blocked waiting for it.
*
* Only while a run is genuinely in flight, and only if the step is newer
* than that run. A failed run deliberately LEAVES its file behind so the
* failing step can be named; without these two checks, queueing the next
* update would show it as already at "Migrating the database" — a step
* nothing has started — and hide the one thing the operator needs, which is
* when it will actually begin.
*
* @param array<string, mixed> $status
*/
private function currentPhase(array $status): ?string
{
if (($status['state'] ?? null) !== 'running') {
return null;
}
$path = storage_path('app/'.self::PHASE);
if (! is_file($path)) {
return null;
}
// Deliberately forgiving: this is a progress hint, and a half-written
// line during the moment update.sh replaces it must not throw on a page
// an operator is watching a deployment from.
$parts = explode("\t", (string) @file_get_contents($path), 2);
$key = trim($parts[0] ?? '');
if ($key === '') {
return null;
}
// Left over from a manual run of update.sh on the shell: newer than the
// last agent status, older than this run.
$startedAt = $this->timestamp($status['started_at'] ?? null);
$phaseAt = $this->timestamp(trim($parts[1] ?? ''));
if ($startedAt !== null && $phaseAt !== null && $phaseAt->lt($startedAt)) {
return null;
}
return $key;
}
/*
* There was a nextCheckAt() here, and a checkInterval() feeding it, so the
* console could count down to the agent's next tick. Both are gone.
*
* It could not be made honest. `checked_at + interval` is in the past
* whenever the interval the agent reports is shorter than the timer
* actually installed on that host — which is every server that has not
* re-run install-agent.sh — and the fallback then returned "now + interval"
* afresh on every poll. The operator watched it run down four seconds and
* snap back to a full minute, forever. Reported exactly that way.
*
* It is also no longer needed: the request wakes the agent (the path unit
* in deploy/install-agent.sh), so there is no wait to describe.
*/
/**
* Has the agent checked in recently enough to be considered alive?
*
* @param array<string, mixed> $status
*/
private function agentIsAlive(array $status): bool
{
$checkedAt = $this->timestamp($status['checked_at'] ?? null);
return $checkedAt !== null
&& $checkedAt->gt(Carbon::now()->subMinutes(self::AGENT_STALE_AFTER_MINUTES));
}
/** Is the agent in the middle of a run right now? */
public function isRunning(): bool
{
$status = $this->readJson(self::STATUS);
return ($status['state'] ?? null) === 'running' && $this->agentIsAlive($status);
}
/** Is an update run already asked for, and still current? */
public function pendingRequest(): ?array
{
$request = $this->readJson(self::REQUEST);
if ($request === []) {
return null;
}
$at = $this->timestamp($request['requested_at'] ?? null);
if ($at === null || $at->lt(Carbon::now()->subMinutes(self::REQUEST_EXPIRES_MINUTES))) {
return null;
}
return $request;
}
/**
* Ask the agent to update — an actual run, downtime and all.
*
* Returns false when a request is already outstanding — clicking twice must
* not queue two runs, and the second click is much more likely to be
* impatience than intent.
*/
public function request(string $by): bool
{
return $this->submit($by, self::KIND_RUN);
}
/**
* Ask the agent only to look — never to apply anything.
*
* The agent already fetches on every tick regardless of this; the request
* exists so an operator who asks gets an answer at the moment they ask,
* and so the answer never runs through the "run" branch — no maintenance
* mode, no restart, for a question that was only "is there something new".
*
* A question must not sit in a queue. Where the host runs the path unit
* from deploy/install-agent.sh this file wakes the agent within a second;
* where it does not, the timer is the fallback and the console says so
* rather than counting down to a start that is not happening — see
* 'instant' in state().
*/
public function requestCheck(string $by): bool
{
return $this->submit($by, self::KIND_CHECK);
}
/**
* Den Proxy die Hostnamen aus der Konsole übernehmen lassen.
*
* Damit fällt der Handgriff weg, der `files.clupilot.com` gekostet hat:
* DNS gesetzt, Umgebungsvariable gesetzt, und trotzdem kein Zertifikat,
* weil `/etc/caddy/Caddyfile` von Hand gepflegt wird und niemand daran
* dachte, dass das ein zweiter, getrennter Schritt ist.
*
* Derselbe Aufruf holt auch fehlende Zertifikate: Caddy fordert sie beim
* Neuladen für jeden Namen an, der noch keins hat. Ein eigener Knopf
* „erneuern" wäre daneben — Caddy erneuert von selbst bei zwei Dritteln der
* Laufzeit, und was ein Betreiber wirklich braucht, ist ein zweiter Versuch,
* nachdem eine Ausstellung einmal gescheitert ist.
*/
public function requestProxyHosts(string $by): bool
{
return $this->submit($by, self::KIND_PROXY_HOSTS);
}
/**
* Ask the agent to restart the four workers that only read .env at their
* own startup — never `app`. See update-agent.sh's handling of this kind
* for why `app` is excluded and always will be.
*
* The caller (App\Livewire\Admin\Integrations::saveEnv()) only calls this
* when the agent is known to be alive; a submit() with no agent to
* collect it would sit until it expires and tell the operator nothing.
* Whether the agent is reachable at all is what makes the difference
* between "handled automatically" and "saved, but still on the old
* values" — the operator is told which one happened, never left to guess.
*/
public function requestRestart(string $by): bool
{
return $this->submit($by, self::KIND_RESTART);
}
/**
* Ask the host to mint a collection key for one archive directory.
*
* The path is passed rather than looked up on the host, because the panel
* is the only side that knows which destination the operator pressed the
* button on.
*/
public function requestArchiveKey(string $by, string $path, string $label): bool
{
return $this->submit($by, self::KIND_ARCHIVE_KEY, [
'archive_path' => $path,
'label' => $label,
]);
}
/**
* Take the minted key, if one is waiting.
*
* Taking, not reading: the file is deleted in the same call. A private key
* that stays on disk after somebody has seen it is a private key on disk.
*
* @return array{created_at:string, label:string, path:string, private_key:string}|null
*/
public function takeArchiveKey(): ?array
{
$data = $this->readJson(self::ARCHIVE_KEY);
if ($data === [] || ! isset($data['private_key'])) {
return null;
}
try {
File::delete(storage_path('app/'.self::ARCHIVE_KEY));
} catch (Throwable) {
// Shown once regardless. Failing to delete it is a reason to warn,
// not a reason to withhold the key from the person who asked for it
// — they would simply ask again and leave a second one behind.
}
return $data;
}
/** @param array<string, mixed> $extra */
private function submit(string $by, string $kind, array $extra = []): bool
{
// Both conditions, not just the request file. The agent DELETES the
// request before it starts a run — so between that moment and the end
// of the run there is nothing pending, and a second click from a
// stale tab would queue a whole redundant deployment for the next
// tick. One request slot for all three kinds: the agent handles one
// tick at a time regardless of what it is asked for.
if ($this->pendingRequest() !== null || $this->isRunning()) {
return false;
}
$this->write(self::REQUEST, json_encode([
'requested_at' => Carbon::now()->toIso8601String(),
'requested_by' => $by,
'kind' => $kind,
] + $extra, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES));
return true;
}
/**
* The agent's failure, in the operator's language.
*
* The agent is a shell script and is not translated; it reports a code and
* the translation happens here. Anything unrecognised is passed through
* rather than swallowed — a message nobody has worded yet still beats
* silence when an update is failing.
*/
private function errorMessage(array $status): ?string
{
$code = isset($status['error']) ? trim((string) $status['error']) : '';
if ($code === '') {
return null;
}
$key = 'admin_settings.update_error.'.$code;
if (! Lang::has($key)) {
return $code;
}
return __($key, ['code' => (string) ($status['exit_code'] ?? '?')]);
}
/** The tail of the last run, for showing what went wrong. */
public function lastLog(int $lines = 40): ?string
{
$path = storage_path('app/'.self::LOG);
try {
if (! File::exists($path)) {
return null;
}
$all = preg_split('/\R/', trim((string) File::get($path))) ?: [];
return implode("\n", array_slice($all, -$lines));
} catch (Throwable) {
return null;
}
}
/** @return array<string, mixed> */
private function readJson(string $relative): array
{
try {
$path = storage_path('app/'.$relative);
if (! File::exists($path)) {
return [];
}
$decoded = json_decode((string) File::get($path), true);
return is_array($decoded) ? $decoded : [];
} catch (Throwable) {
return [];
}
}
private function write(string $relative, string $contents): void
{
$path = storage_path('app/'.$relative);
File::ensureDirectoryExists(dirname($path));
File::put($path, $contents);
}
private function timestamp(mixed $value): ?Carbon
{
if (! is_string($value) || $value === '') {
return null;
}
try {
return Carbon::parse($value);
} catch (Throwable) {
return null;
}
}
}