781 lines
39 KiB
Bash
Executable File
781 lines
39 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# CluPilot — pull the latest code and apply it.
|
|
#
|
|
# The checkout is bind-mounted into every container, so a `git merge` swaps the
|
|
# running code instantly. There is no atomic release directory to switch to;
|
|
# what there is, is maintenance mode. So the order is:
|
|
#
|
|
# down → merge → dependencies → migrate → assets → restart → up
|
|
#
|
|
# That trades a short, deliberate outage for never serving code whose schema
|
|
# does not exist yet. If any step fails the site STAYS down: coming back up with
|
|
# new code on an old schema is worse than staying dark until someone looks.
|
|
set -euo pipefail
|
|
|
|
# Das Wurzelverzeichnis des Checkouts. Einmal ermittelt und danach mitgereicht:
|
|
# nach dem Re-Exec gleich darunter liegt $0 in /tmp, und `dirname "$0"/..`
|
|
# zeigte dann auf `/`.
|
|
if [[ -z "${CLUPILOT_ROOT:-}" ]]; then
|
|
CLUPILOT_ROOT="$(cd "$(dirname "$0")/.." && pwd)"
|
|
export CLUPILOT_ROOT
|
|
fi
|
|
cd "$CLUPILOT_ROOT"
|
|
|
|
# --- Sich selbst aus der Schusslinie nehmen, bevor irgendetwas passiert ------
|
|
#
|
|
# Dieses Skript checkt weiter unten einen neuen Stand aus — und ersetzt damit
|
|
# SICH SELBST auf der Platte, weil der Checkout derselbe ist, aus dem es läuft.
|
|
#
|
|
# Bash liest ein Skript aber nicht auf einmal ein, sondern nachlaufend ab einer
|
|
# Byte-Position. Ändert sich die Datei mitten im Lauf, liest bash an derselben
|
|
# Position im NEUEN Text weiter — und die liegt irgendwo mitten in einer
|
|
# anderen Zeile. Im günstigen Fall gibt das einen Syntaxfehler. Im ungünstigen
|
|
# landet die Position VOR dem Punkt, an dem der Lauf schon war, und das Skript
|
|
# führt einen Abschnitt ein zweites Mal aus.
|
|
#
|
|
# Genau das geschah bei v1.3.97: die Freigabe fügte 23 Zeilen oberhalb des
|
|
# Checkouts ein, alles danach rutschte nach hinten, und das Update drehte sich
|
|
# im Kreis. v1.3.96 lief sauber durch — sie hatte diese Datei nicht angefasst.
|
|
# Der Fehler schlägt also nur bei Freigaben zu, die das Update selbst ändern,
|
|
# und war deshalb jahrelang unsichtbar.
|
|
#
|
|
# Die Kopie liegt außerhalb des Checkouts. Ab hier darf `git checkout` mit der
|
|
# Originaldatei machen, was es will.
|
|
if [[ -z "${CLUPILOT_UPDATE_COPY:-}" ]]; then
|
|
CLUPILOT_UPDATE_COPY="$(mktemp /tmp/clupilot-update-XXXXXX.sh)"
|
|
export CLUPILOT_UPDATE_COPY
|
|
cat "$CLUPILOT_ROOT/deploy/update.sh" > "$CLUPILOT_UPDATE_COPY"
|
|
exec bash "$CLUPILOT_UPDATE_COPY" "$@"
|
|
fi
|
|
|
|
# shellcheck source=deploy/lib/release.sh
|
|
. deploy/lib/release.sh
|
|
|
|
BRANCH="${BRANCH:-main}"
|
|
# RELEASE=v1.2.0 moves a pinned server to that tag. Without it, the mode the
|
|
# server was installed in is kept — a production box pinned to a release does
|
|
# not quietly start following main because someone re-ran the updater.
|
|
RELEASE="${RELEASE:-}"
|
|
|
|
# Never as root: the checkout belongs to the service account, and running this
|
|
# as root would leave root-owned files behind that the app cannot write.
|
|
if [[ $EUID -eq 0 ]]; then
|
|
echo "Run this as the service account, not as root:" >&2
|
|
echo " sudo -u clupilot bash $0" >&2
|
|
exit 1
|
|
fi
|
|
STATE_FILE="storage/app/deployed-commit"
|
|
# The root-owned half of an update, and the version of what it can do. Both are
|
|
# fixed by deploy/install-agent.sh; this script only asks. Raising HOST_STEP_NEEDS
|
|
# here is what makes an update tell the operator to run the installer again.
|
|
HOST_STEP=/usr/local/sbin/clupilot-host-step
|
|
# 3 seit `release-update-lock`: ohne den Schritt bleibt der Knopf „Sperre lösen"
|
|
# in der Konsole ein Knopf, der nichts tun kann.
|
|
HOST_STEP_NEEDS=3
|
|
# Which step is running, for the console to show. Written as a KEY, not as the
|
|
# sentence below it: the console is translated and this script is not, so an
|
|
# English line here would surface untranslated in the interface. A run that dies
|
|
# leaves the key of the step it died at, which is the one thing the operator
|
|
# needs and the log alone makes them hunt for.
|
|
PHASE_FILE="storage/app/deploy/update-phase"
|
|
|
|
log() { printf '\n\033[1;34m==>\033[0m %s\n' "$*"; }
|
|
warn() { printf '\033[1;33m !\033[0m %s\n' "$*"; }
|
|
|
|
# A step, announced to the terminal and recorded for the console.
|
|
phase() {
|
|
local key="$1"; shift
|
|
# `|| true`, wie beim Schreiben darunter: gehoert storage/ nach einem
|
|
# frueheren Fehltritt root, scheitert das mkdir — und mit `set -e` starb das
|
|
# ganze Update an seiner ERSTEN Zeile, ohne eine einzige Ausgabe. Von aussen
|
|
# sah das aus wie "haengengeblieben"; in Wahrheit war es nach einer
|
|
# Millisekunde vorbei. Eine fehlende Fortschrittsanzeige ist kein Grund,
|
|
# ein Deployment abzubrechen.
|
|
mkdir -p "$(dirname "$PHASE_FILE")" 2>/dev/null || true
|
|
# Not atomic on purpose: this is a hint for a progress line, and a torn read
|
|
# costs a single poll. Writing a temp file per step would be more moving
|
|
# parts than the thing is worth.
|
|
printf '%s\t%s\n' "$key" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$PHASE_FILE" 2>/dev/null || true
|
|
log "$*"
|
|
}
|
|
|
|
# As the application's own user, not as root.
|
|
#
|
|
# `docker compose exec` runs as root unless told otherwise, and this did — so
|
|
# when `artisan optimize` failed during a deployment it wrote its error into
|
|
# storage/logs/laravel.log AS ROOT. From that moment the application could not
|
|
# append to its own log: Monolog threw on every attempt, and a throw while
|
|
# logging is a 500 on every page that logs, with nothing in the log to say why.
|
|
# It surfaced as a 500 on the VPN config download — one of the few pages that
|
|
# writes a log line on the way through — and looked for days like a fault in
|
|
# the VPN code.
|
|
#
|
|
# docker/entrypoint.sh already drops to www-data for exactly these commands.
|
|
# The deployment simply never did.
|
|
in_app() { docker compose exec -T -u www-data app "$@"; }
|
|
|
|
# Hand back anything an earlier run left owned by root.
|
|
#
|
|
# Not tidiness. That damage is silent and permanent — nothing repairs a
|
|
# root-owned log file on its own, and the symptom surfaces on whichever page
|
|
# happens to log first, which is nowhere near the deployment that caused it. A
|
|
# server already carrying it heals on its next deployment instead of needing
|
|
# somebody to work out what happened.
|
|
#
|
|
# And it is now load-bearing: with in_app unprivileged, a vendor/ or
|
|
# node_modules/ left owned by root would make the very next composer or npm
|
|
# step fail outright.
|
|
normalise_ownership() {
|
|
# find, not `stat` on the directory itself. The first version of this
|
|
# sampled the top-level owner and skipped the recursion when it matched —
|
|
# and node_modules was owned by www-data while node_modules/.vite-temp,
|
|
# left by an earlier root build, was not. The deployment then failed at
|
|
# `npm run build` with EACCES, in maintenance mode, on the very release
|
|
# that was supposed to prevent it. A directory's owner says nothing about
|
|
# its contents.
|
|
#
|
|
# One walk that changes only what is wrong, rather than a detection pass
|
|
# and then a blanket chown -R over thirty thousand files.
|
|
docker compose exec -T -u root app sh -c '
|
|
for d in storage bootstrap/cache vendor node_modules public/build; do
|
|
[ -d "$d" ] || continue
|
|
find "$d" ! -user www-data -exec chown www-data:www-data {} + 2>/dev/null || true
|
|
done
|
|
|
|
# Das Heimatverzeichnis von www-data und die Zwischenspeicher darin.
|
|
#
|
|
# Der Dockerfile vergibt www-data per `usermod -o -u` eine neue Nummer,
|
|
# schreibt aber keine vorhandene Datei um: /var/www blieb bei der alten
|
|
# aus dem Basis-Abbild, und damit ist das Heimatverzeichnis für seinen
|
|
# eigenen Benutzer nicht beschreibbar. npm legt seinen Cache unter
|
|
# ~/.npm an, durfte das nicht und brach ein Deployment mit EACCES ab —
|
|
# errno -13, also Code 243 — mitten im Wartungsmodus. Composer trifft
|
|
# dasselbe und verzeiht es still, weshalb es nur auffiel, als sich zum
|
|
# ersten Mal seit Langem package.json änderte und npm ci überhaupt lief.
|
|
#
|
|
# Der Dockerfile legt beides inzwischen selbst richtig an. Das hier ist
|
|
# für Server, die noch auf einem älteren Abbild stehen — sie sollen
|
|
# sich beim nächsten Deployment heilen, statt darauf zu warten.
|
|
#
|
|
# /var/www nur eine Ebene tief: darunter liegt der ganze Checkout, der
|
|
# oben schon einzeln behandelt wird.
|
|
mkdir -p /var/www/.npm /var/www/.composer 2>/dev/null || true
|
|
find /var/www -maxdepth 0 ! -user www-data -exec chown www-data:www-data {} + 2>/dev/null || true
|
|
for d in /var/www/.npm /var/www/.composer; do
|
|
find "$d" ! -user www-data -exec chown www-data:www-data {} + 2>/dev/null || true
|
|
done
|
|
' >/dev/null 2>&1 || true
|
|
}
|
|
|
|
down=0
|
|
finish() {
|
|
local code=$?
|
|
|
|
# Die Arbeitskopie aus dem Re-Exec oben. Sie liegt in /tmp und wäre sonst
|
|
# nach jedem Lauf eine Datei mehr.
|
|
[[ -n "${CLUPILOT_UPDATE_COPY:-}" ]] && rm -f "$CLUPILOT_UPDATE_COPY"
|
|
|
|
if [[ $code -ne 0 && $down -eq 1 ]]; then
|
|
warn "Update failed. The site is STILL in maintenance mode on purpose."
|
|
warn "Fix the problem and re-run this script, or force it back up with:"
|
|
warn " docker compose exec -u www-data app php artisan up"
|
|
fi
|
|
exit $code
|
|
}
|
|
trap finish EXIT
|
|
|
|
# Which line this server follows, and what it is being moved to.
|
|
mode="$(release_mode)"
|
|
[[ -n "$RELEASE" ]] && mode="release"
|
|
|
|
if [[ "$mode" == "release" ]]; then
|
|
# A pinned server without an explicit RELEASE has nothing to do: staying on
|
|
# the tag is the whole point of being pinned.
|
|
if [[ -z "$RELEASE" ]]; then
|
|
RELEASE="$(release_source)"
|
|
RELEASE="${RELEASE#refs/tags/}"
|
|
[[ -n "$RELEASE" ]] || { echo "Pinned to a release, but no tag recorded. Pass RELEASE=vX.Y.Z." >&2; exit 1; }
|
|
fi
|
|
|
|
phase fetch "Fetching release $RELEASE"
|
|
git fetch --quiet --tags --force origin
|
|
git rev-parse -q --verify "refs/tags/${RELEASE}^{commit}" >/dev/null \
|
|
|| { echo "No such release tag: ${RELEASE}" >&2; exit 1; }
|
|
|
|
target="$(git rev-parse "refs/tags/${RELEASE}^{commit}")"
|
|
source_ref="refs/tags/${RELEASE}"
|
|
else
|
|
phase fetch "Fetching $BRANCH"
|
|
git fetch --quiet origin "$BRANCH"
|
|
target="$(git rev-parse "origin/$BRANCH")"
|
|
source_ref="$BRANCH"
|
|
fi
|
|
|
|
before="$(git rev-parse HEAD)"
|
|
|
|
# Backwards is not an update. The database has already been migrated forward,
|
|
# and older code against a newer schema is the one failure this script exists to
|
|
# prevent — with the added cruelty that the migrations needed to roll back are
|
|
# not in the older checkout at all. A genuine downgrade means restoring the
|
|
# pre-upgrade database snapshot first, deliberately, by hand.
|
|
if release_would_go_backwards "$before" "$target"; then
|
|
cat >&2 <<BACKWARDS
|
|
Refusing to move backwards: ${source_ref} is behind what is deployed.
|
|
|
|
deployed $(git rev-parse --short "$before")
|
|
requested $(git rev-parse --short "$target")
|
|
|
|
The schema has already moved forward, and older code against a newer schema is
|
|
what maintenance mode exists to avoid. To go back: restore the database
|
|
snapshot taken before the upgrade, then install the older release.
|
|
BACKWARDS
|
|
exit 1
|
|
fi
|
|
# What was last deployed successfully — not what is merely checked out. A run
|
|
# that died halfway leaves these different, and the next run finishes the job
|
|
# instead of declaring "already up to date".
|
|
deployed="$(cat "$STATE_FILE" 2>/dev/null || echo '')"
|
|
|
|
# Readiness is re-established after every start, never carried over. `up -d`
|
|
# succeeds once a container is CREATED, so one that starts and exits — an
|
|
# unreadable certificate, an address it cannot bind — would leave a previous
|
|
# VPN_READY=true in place and the application still advertising a resolver that
|
|
# is no longer running.
|
|
#
|
|
# A function because there are two ways out of this script: a full deploy, and
|
|
# the shortcut taken when the checkout has not moved. Both start services; both
|
|
# have to record what actually came up.
|
|
reconcile_vpn_readiness() {
|
|
grep -qE '^COMPOSE_PROFILES=.*vpn' .env 2>/dev/null || return 0
|
|
|
|
local host vpn_ready=false
|
|
host="$(sed -n 's/^VPN_INTERNAL_HOST=//p' .env | tail -1)"
|
|
|
|
# ASKED, not assumed. A container can be up and running while the process
|
|
# inside it listens in a network namespace that was torn down underneath it
|
|
# — nothing errors, the address simply refuses connections, and a
|
|
# container-state check reports everything fine.
|
|
#
|
|
# Asked at the address the gateway actually serves. It binds to the hub
|
|
# address alone (see docker/caddy/vpn.Caddyfile), so nothing has ever
|
|
# listened on 127.0.0.1 — the previous probe could only ever fail, and it
|
|
# did: VPN_READY stayed false on a perfectly healthy tunnel, and because of
|
|
# that the application withheld the resolver from every client config it
|
|
# issued. The console was then unreachable over the VPN for weeks, with the
|
|
# deployment printing a warning that read like a gateway fault.
|
|
#
|
|
# Over plain HTTP on the health port, not HTTPS: the TLS site matches on the
|
|
# console's hostname, so a request to the bare address presents no SNI, gets
|
|
# no certificate and fails the handshake — which looks exactly like the
|
|
# outage this is meant to detect. Caddy refuses to start at all if the
|
|
# certificate is unreadable, so a health port that answers still proves the
|
|
# whole gateway loaded.
|
|
# The same variable compose hands the gateway, so the two cannot drift.
|
|
local hub port health
|
|
hub="$(sed -n 's/^CLUPILOT_WG_HUB_ADDRESS=//p' .env | tail -1)"
|
|
hub="${hub:-10.66.0.1}"
|
|
port="$(sed -n 's/^VPN_HEALTH_PORT=//p' .env | tail -1)"
|
|
health="http://${hub}:${port:-8081}/healthz"
|
|
|
|
# Long enough for a restart to finish. These containers are restarted twice
|
|
# in a deployment — once by `up -d`, then again after the hub they live
|
|
# inside — and `docker compose exec` into one that is still coming up fails
|
|
# outright rather than waiting. Twenty seconds was not enough for that, and
|
|
# the run then reported a healthy gateway as down.
|
|
for _ in $(seq 1 30); do
|
|
if docker compose exec -T vpn-gateway sh -c \
|
|
"wget -q --spider '$health' 2>/dev/null || wget -q -O /dev/null '$health' 2>/dev/null" >/dev/null 2>&1; then
|
|
vpn_ready=true
|
|
break
|
|
fi
|
|
sleep 2
|
|
done
|
|
|
|
if [[ "$vpn_ready" != "true" ]]; then
|
|
printf '\033[1;33m !\033[0m %s\n' "The tunnel gateway is not answering on ${host} — clients will not be given its resolver."
|
|
fi
|
|
|
|
if [[ "$(sed -n 's/^VPN_READY=//p' .env | tail -1)" != "$vpn_ready" ]]; then
|
|
sed -i '/^VPN_READY=/d' .env
|
|
printf 'VPN_READY=%s\n' "$vpn_ready" >> .env
|
|
in_app php artisan config:clear >/dev/null 2>&1 || true
|
|
in_app php artisan config:cache >/dev/null 2>&1 || true
|
|
fi
|
|
}
|
|
|
|
# The VPN gateway and resolver live behind a compose profile, and behind a
|
|
# certificate. Enabling the profile on the hostname alone would start a Caddy
|
|
# with an empty tls directive — it fails to load, the internal console never
|
|
# comes up, and the application is meanwhile handing out client configs that
|
|
# point at a resolver nobody started. So: both, or neither, and BEFORE the
|
|
# services are brought up rather than after.
|
|
if ! grep -qE '^VPN_INTERNAL_HOST=..' .env 2>/dev/null; then
|
|
# Cleared. Leaving the profile in place keeps both services running against
|
|
# a placeholder hostname and stale certificate paths, so "empty disables it"
|
|
# would not be true.
|
|
if grep -qE '^COMPOSE_PROFILES=.*vpn' .env 2>/dev/null; then
|
|
# Rebuilt from the remaining entries rather than cut out in place:
|
|
# deleting "vpn" from "ci,vpn" by substitution leaves "ci," and Compose
|
|
# reads the empty entry as a profile.
|
|
profiles="$(sed -n 's/^COMPOSE_PROFILES=//p' .env | tail -1)"
|
|
# `|| true`: with only "vpn" in the list, grep finds nothing and exits
|
|
# 1 — which under `set -e` aborts the update mid-flight, in maintenance
|
|
# mode, with the services it was about to stop still running.
|
|
profiles="$(printf '%s' "$profiles" | tr ',' '\n' | grep -vx 'vpn' | grep -v '^$' | paste -sd, - || true)"
|
|
sed -i '/^COMPOSE_PROFILES=/d' .env
|
|
[[ -n "$profiles" ]] && printf 'COMPOSE_PROFILES=%s\n' "$profiles" >> .env
|
|
docker compose --profile vpn stop vpn-dns vpn-gateway >/dev/null 2>&1 || true
|
|
docker compose --profile vpn rm -f vpn-dns vpn-gateway >/dev/null 2>&1 || true
|
|
# The application caches its configuration, so clearing .env alone
|
|
# would leave it issuing configs that name a resolver just stopped.
|
|
in_app php artisan config:clear >/dev/null 2>&1 || true
|
|
log "Disabled the vpn compose profile — VPN_INTERNAL_HOST is empty"
|
|
fi
|
|
elif grep -qE '^VPN_INTERNAL_HOST=..' .env 2>/dev/null; then
|
|
vpn_cert="$(sed -n 's/^VPN_CERT_PATH=//p' .env | tail -1)"
|
|
vpn_key="$(sed -n 's/^VPN_KEY_PATH=//p' .env | tail -1)"
|
|
|
|
vpn_host="$(sed -n 's/^VPN_INTERNAL_HOST=//p' .env | tail -1)"
|
|
|
|
# The certificate has to belong to the CURRENT hostname. After a rename the
|
|
# old paths are still set and still exist, and the gateway would start
|
|
# serving the previous name's certificate to clients resolving the new one.
|
|
if [[ -n "$vpn_cert" && "$(basename "$vpn_cert")" != "${vpn_host}.crt" ]]; then
|
|
sed -i '/^VPN_CERT_PATH=/d;/^VPN_KEY_PATH=/d;/^VPN_READY=/d' .env
|
|
vpn_cert=''; vpn_key=''
|
|
in_app php artisan config:clear >/dev/null 2>&1 || true
|
|
printf '\033[1;33m !\033[0m %s\n' "The tunnel certificate does not match VPN_INTERNAL_HOST — cleared."
|
|
fi
|
|
|
|
if [[ -z "$vpn_cert" || -z "$vpn_key" ]]; then
|
|
# Both or neither: leaving the profile on with empty tls paths starts a
|
|
# Caddy that cannot load its configuration and crashes forever.
|
|
if grep -qE '^COMPOSE_PROFILES=.*vpn' .env 2>/dev/null; then
|
|
docker compose --profile vpn stop vpn-dns vpn-gateway >/dev/null 2>&1 || true
|
|
profiles="$(sed -n 's/^COMPOSE_PROFILES=//p' .env | tail -1)"
|
|
profiles="$(printf '%s' "$profiles" | tr ',' '\n' | grep -vx 'vpn' | grep -v '^$' | paste -sd, - || true)"
|
|
sed -i '/^COMPOSE_PROFILES=/d' .env
|
|
[[ -n "$profiles" ]] && printf 'COMPOSE_PROFILES=%s\n' "$profiles" >> .env
|
|
sed -i '/^VPN_READY=/d' .env
|
|
fi
|
|
fi
|
|
|
|
if [[ -n "$vpn_cert" && -n "$vpn_key" ]]; then
|
|
if ! grep -qE '^COMPOSE_PROFILES=.*vpn' .env; then
|
|
if grep -qE '^COMPOSE_PROFILES=' .env; then
|
|
sed -i 's/^COMPOSE_PROFILES=\(.*\)$/COMPOSE_PROFILES=\1,vpn/' .env
|
|
else
|
|
printf 'COMPOSE_PROFILES=vpn\n' >> .env
|
|
fi
|
|
log "Enabled the vpn compose profile"
|
|
# Started here, not left to the deploy below: the update may exit
|
|
# early as already deployed, and the profile would then be marked
|
|
# on with nothing running behind it.
|
|
docker compose --profile vpn up -d vpn-dns vpn-gateway >/dev/null 2>&1 || true
|
|
reconcile_vpn_readiness
|
|
fi
|
|
else
|
|
printf '\033[1;33m !\033[0m %s\n' "VPN_INTERNAL_HOST is set but the certificate paths are not."
|
|
printf ' %s\n' "The console is not reachable inside the tunnel. Run: sudo bash $(pwd)/deploy/install-agent.sh"
|
|
fi
|
|
fi
|
|
|
|
if [[ "$before" == "$target" && "$deployed" == "$target" ]]; then
|
|
# Nothing to deploy, but possibly something to decide: pinning a server to
|
|
# a release that happens to be the commit it is already on is a real and
|
|
# sensible thing to do. Taking this exit without recording it would leave
|
|
# the machine looking pinned while the next plain update quietly walks it
|
|
# back onto the branch.
|
|
# The manifest is checked too, not just the mode: if writing it failed at
|
|
# the end of the last run — a full disk is enough — nothing else would ever
|
|
# repair it, and the console would report the previous release forever
|
|
# while every later update declared itself already done.
|
|
if [[ "$(release_mode)" != "$mode" || "$(release_source)" != "$source_ref" \
|
|
|| "$(release_manifest_commit)" != "$before" ]]; then
|
|
log "Already on this commit — recording it as $mode ($source_ref)"
|
|
# And actually pin it. A checkout still attached to the branch is not
|
|
# pinned, whatever the manifest claims: the next thing that touches git
|
|
# moves it, and the machine drifts off the release it is supposed to be
|
|
# nailed to.
|
|
if [[ "$mode" == "release" ]]; then
|
|
git checkout --quiet --detach "$target"
|
|
fi
|
|
release_remember "$mode" "$source_ref"
|
|
release_write_manifest "$before" "$source_ref" "$mode"
|
|
else
|
|
log "Already up to date ($(git rev-parse --short HEAD), ${source_ref})"
|
|
fi
|
|
|
|
# ABER: derselbe Commit heisst nicht, dass der letzte Lauf FERTIG geworden
|
|
# ist. Bricht ein Update mittendrin ab, steht der Checkout schon auf dem
|
|
# Ziel — und ein zweiter Aufruf ging bis hierher davon aus, es sei alles
|
|
# erledigt, und tat gar nichts. Genau so stand ein Server eine Stunde lang
|
|
# halb unten, waehrend `bash deploy/update.sh` freundlich "Already up to
|
|
# date" meldete.
|
|
#
|
|
# Deshalb wird hier nicht der Commit gefragt, sondern der Zustand: laeuft
|
|
# jeder Dienst, und ist der Wartungsmodus aus? Wenn nein, laeuft der Rest
|
|
# dieses Skripts trotzdem und richtet es.
|
|
unhealthy=""
|
|
soll_services="$(docker compose config --services 2>/dev/null | sort || true)"
|
|
ist_services="$(docker compose ps --services --status running 2>/dev/null | sort || true)"
|
|
if [[ -n "$soll_services" ]]; then
|
|
fehlende="$(comm -23 <(printf '%s\n' "$soll_services") <(printf '%s\n' "$ist_services") | tr '\n' ' ' | sed 's/ *$//')"
|
|
[[ -n "$fehlende" ]] && unhealthy="es fehlen Dienste: $fehlende"
|
|
fi
|
|
if [[ -z "$unhealthy" ]] && docker compose exec -T -u www-data app test -f storage/framework/down >/dev/null 2>&1; then
|
|
unhealthy="der Wartungsmodus ist noch an"
|
|
fi
|
|
|
|
if [[ -z "$unhealthy" ]]; then
|
|
exit 0
|
|
fi
|
|
|
|
warn "Derselbe Stand, aber $unhealthy — der letzte Lauf ist offenbar abgebrochen."
|
|
warn "Ich fahre den Rest trotzdem, statt 'fertig' zu melden."
|
|
fi
|
|
|
|
# What the change checks below compare against. After a failed run the checkout
|
|
# is already at the target, so diffing against it would compare a commit with
|
|
# itself and skip the very steps that did not finish — the base has to be the
|
|
# last commit we actually deployed.
|
|
base="$before"
|
|
if [[ -n "$deployed" ]] && git cat-file -e "${deployed}^{commit}" 2>/dev/null; then
|
|
base="$deployed"
|
|
fi
|
|
|
|
if [[ "$before" != "$target" ]]; then
|
|
log "Updating $(git rev-parse --short "$before") → $(git rev-parse --short "$target")"
|
|
git --no-pager log --oneline "$before..$target" | sed 's/^/ /'
|
|
else
|
|
warn "Code is current but the last update did not finish — repeating the steps."
|
|
fi
|
|
|
|
# Before the first unprivileged command, not after: if an earlier run left the
|
|
# log or the vendor directory owned by root, everything below fails on it.
|
|
normalise_ownership
|
|
|
|
phase maintenance_on "Enabling maintenance mode"
|
|
in_app php artisan down --retry=60 >/dev/null || warn "Could not enable maintenance mode (continuing)"
|
|
down=1
|
|
|
|
if [[ "$mode" == "release" ]]; then
|
|
# Detached on purpose: a release is a fixed point, not a line to follow.
|
|
# `git merge --ff-only` would be meaningless here, and on a detached HEAD it
|
|
# is how a pinned server silently rejoins main.
|
|
phase checkout "Checking out $source_ref"
|
|
git checkout --quiet --detach "$target"
|
|
else
|
|
phase checkout "Checking out $BRANCH"
|
|
git merge --quiet --ff-only "origin/$BRANCH"
|
|
fi
|
|
after="$(git rev-parse HEAD)"
|
|
|
|
# The image is only rebuilt when its definition changed — minutes versus seconds.
|
|
if ! git diff --quiet "$base" "$after" -- docker/ 2>/dev/null; then
|
|
phase image "Rebuilding the image"
|
|
# `terminal` und `vpn-hub` mitbauen, nicht nur `app`: `docker compose up -d` weiter unten
|
|
# baut nur Images, die es noch GAR NICHT gibt. Beim ersten Ausrollen fällt
|
|
# das nicht auf, danach nie wieder — eine Änderung an docker/terminal/
|
|
# sähe ausgeliefert aus, und es liefe das alte Image.
|
|
docker compose build --quiet app terminal vpn-hub
|
|
# Recreate now, not at the end: everything below runs INSIDE this container,
|
|
# and an update that changes the PHP runtime would otherwise install and
|
|
# migrate under the old one.
|
|
docker compose up -d --force-recreate app
|
|
for _ in $(seq 1 30); do
|
|
in_app php -v >/dev/null 2>&1 && break
|
|
sleep 2
|
|
done
|
|
fi
|
|
|
|
# vendor/ and node_modules/ live in the bind mount, so they shadow whatever the
|
|
# image contains: rebuilding the image does NOT update them. Install explicitly
|
|
# whenever a lockfile moved, or the migration below runs against stale packages.
|
|
if ! git diff --quiet "$base" "$after" -- composer.json composer.lock 2>/dev/null || [[ ! -d vendor ]]; then
|
|
phase composer "Installing PHP dependencies"
|
|
in_app composer install --no-interaction --no-dev --prefer-dist --no-progress --optimize-autoloader
|
|
fi
|
|
|
|
if ! git diff --quiet "$base" "$after" -- package.json package-lock.json 2>/dev/null || [[ ! -d node_modules ]]; then
|
|
phase npm "Installing JS dependencies"
|
|
in_app npm ci --no-fund --no-audit
|
|
fi
|
|
|
|
phase migrate "Applying migrations"
|
|
in_app php artisan migrate --force
|
|
|
|
phase assets "Rebuilding assets"
|
|
in_app npm run build
|
|
|
|
# Before the restarts, not after: a service that starts while the old cache is
|
|
# still on disk loads it and keeps those values for the life of its process.
|
|
phase caches "Rebuilding caches"
|
|
# optimize:clear first, then optimize: a half-warm cache from before the update
|
|
# is what produces a page styled with assets that no longer exist.
|
|
in_app php artisan optimize:clear >/dev/null
|
|
in_app php artisan optimize >/dev/null
|
|
|
|
# Existing installations keep whatever .env already said, so safer defaults in
|
|
# compose protect new installs only. A backend published on 0.0.0.0 is reachable
|
|
# from the internet even with UFW closed — Docker publishes ahead of it — and
|
|
# reaching a backend directly skips every hostname and address rule the proxy
|
|
# enforces, including the one keeping the console private.
|
|
#
|
|
# Rewritten only where a reverse proxy is actually in front. A development box
|
|
# without one genuinely needs the port exposed, and silently taking it away
|
|
# would look like the machine had broken.
|
|
# ACTIVE, not merely installed. A retired package still on disk is not a proxy,
|
|
# and rebinding a development box to loopback because caddy happens to be
|
|
# installed makes it unreachable with no obvious cause.
|
|
proxy_running=0
|
|
systemctl is-active --quiet caddy 2>/dev/null && proxy_running=1
|
|
systemctl is-active --quiet nginx 2>/dev/null && proxy_running=1
|
|
# And actually holding 443 — a service that is up but proxying something else
|
|
# entirely is no reason to take this application off the network.
|
|
if [[ $proxy_running -eq 1 ]] && ! ss -ltn 2>/dev/null | grep -qE ':443\s'; then
|
|
proxy_running=0
|
|
fi
|
|
|
|
if [[ $proxy_running -eq 1 ]]; then
|
|
for var in APP_PORT REVERB_HOST_PORT; do
|
|
value="$(sed -n "s/^${var}=//p" .env | tail -1)"
|
|
|
|
# A bare number: published on every interface.
|
|
if [[ "$value" =~ ^[0-9]+$ ]]; then
|
|
sed -i "s|^${var}=.*|${var}=127.0.0.1:${value}|" .env
|
|
printf '\033[1;33m !\033[0m %s\n' "${var} was published on all interfaces — bound to 127.0.0.1:${value}."
|
|
printf ' %s\n' "Reaching it directly bypassed the reverse proxy entirely. Set ${var}=${value} again if that was deliberate."
|
|
fi
|
|
done
|
|
fi
|
|
|
|
# Again, at the end. The first call heals what a previous run left; this one
|
|
# heals what THIS run made — `git checkout` rewrites the tree as the service
|
|
# account, and anything a root process wrote into storage while the old build
|
|
# was serving would otherwise stay root-owned. A compiled Blade view in that
|
|
# state answers every request for it with 500: touch() with an explicit mtime
|
|
# needs ownership, and the framework does exactly that on recompile.
|
|
normalise_ownership
|
|
|
|
phase restart "Restarting services"
|
|
# Vorher/nachher merken: nur wenn `up -d` den Tunnel-Container WIRKLICH neu baut
|
|
# (neues Image, geänderte Konfiguration), ist sein Netz-Namensraum ein anderer —
|
|
# und nur dann müssen die Nachbarn hinterher und die gemerkten Ströme weg. Ein
|
|
# blosses „lief schon" lässt beides in Ruhe.
|
|
# EINMALIGE Umstellung: das Compose-Netz bekommt ein erklärtes Subnetz, damit
|
|
# vpn-hub eine feste Adresse haben kann (siehe docker-compose.yml, ganz unten).
|
|
#
|
|
# Docker kann ein bestehendes Netz nicht umdefinieren — es muss neu angelegt
|
|
# werden, und das geht nur, wenn KEIN Container mehr daranhängt. `docker compose
|
|
# up -d` allein scheitert daran mit "network … has active endpoints" und lässt
|
|
# den Stapel halb unten stehen. Beim Bauen genau so passiert; deshalb macht das
|
|
# Deployment es geordnet, statt darüber zu stolpern.
|
|
#
|
|
# Nach dieser einen Umstellung stimmen die Werte überein und der Block tut nie
|
|
# wieder etwas.
|
|
net_migrated=false
|
|
want_subnet="$(sed -n 's/^CLUPILOT_NET_SUBNET=//p' .env 2>/dev/null | tail -1)"
|
|
want_subnet="${want_subnet:-172.18.0.0/16}"
|
|
net_name="${COMPOSE_PROJECT_NAME:-$(basename "$PWD")}_default"
|
|
have_subnet="$(docker network inspect "$net_name" --format '{{range .IPAM.Config}}{{.Subnet}}{{end}}' 2>/dev/null || true)"
|
|
|
|
if [[ -n "$have_subnet" && "$have_subnet" != "$want_subnet" ]]; then
|
|
log "Das Netz bekommt ein festes Subnetz ($want_subnet) — dafür muss der Stapel einmal ganz herunter."
|
|
docker compose down --remove-orphans || true
|
|
net_migrated=true
|
|
fi
|
|
|
|
hub_before="$(docker compose ps -q vpn-hub 2>/dev/null || true)"
|
|
# Nach einer Netz-Umstellung NEU ERZEUGEN, nicht nur starten: ein Container, der
|
|
# nur neu gestartet wird, haengt weiter am alten Netz. Alle laufen dann, und
|
|
# trotzdem loest kein Name mehr auf — das teuerste Fehlerbild dieses Tages.
|
|
if [[ "$net_migrated" == true ]]; then
|
|
docker compose up -d --force-recreate
|
|
else
|
|
docker compose up -d
|
|
fi
|
|
hub_after="$(docker compose ps -q vpn-hub 2>/dev/null || true)"
|
|
|
|
# Seit der Tunnel einen eigenen Container mit eigenem, selten wechselndem Abbild
|
|
# hat, sollten diese beiden Werte bei einem gewoehnlichen Update GLEICH sein.
|
|
# Sind sie es nicht, wurde der Namensraum neu gebaut — dann muessen die
|
|
# Mitbewohner hinterher und die gemerkten Stroeme weg.
|
|
#
|
|
# Als `if` geschrieben, nicht als `[[ … ]] && hub_rebuilt=true`. Nicht weil
|
|
# Letzteres bräche — bash nimmt die linke Seite einer `&&`-Liste ausdrücklich
|
|
# von `set -e` aus, nachgeprüft —, sondern weil man an dieser Stelle nicht erst
|
|
# nachschlagen sollen muss, ob es bricht.
|
|
hub_rebuilt=false
|
|
if [[ -n "$hub_before" && "$hub_before" != "$hub_after" ]]; then
|
|
hub_rebuilt=true
|
|
fi
|
|
# Nach der Netz-Umstellung war der Hub vorher schon weg, der Vergleich oben
|
|
# greift also nicht — die gemerkten Ströme zeigen aber trotzdem auf die alte,
|
|
# automatisch vergebene Adresse und müssen weg.
|
|
if [[ "$net_migrated" == true ]]; then
|
|
hub_rebuilt=true
|
|
fi
|
|
# Workers hold their PHP classes for the life of the process; without this they
|
|
# keep running the code from before the update.
|
|
#
|
|
# queue-provisioning steht mit Absicht NICHT in dieser Liste. In seinem
|
|
# Netz-Namensraum lebt wg0, und `docker compose restart` baut den Namensraum neu
|
|
# auf — jede WireGuard-Sitzung reisst dabei ab, die des Betreibers am Telefon
|
|
# wie die jedes Hosts. Fuer ein Update, das nur PHP-Code aendert, ist das ein
|
|
# absurder Preis.
|
|
#
|
|
# Seit der Arbeiter dort in einer Schleife laeuft (docker/provisioning-worker.sh)
|
|
# geht es billiger: `queue:restart` setzt ein Signal, der Arbeiter beendet sich
|
|
# nach dem laufenden Auftrag, und die Schleife startet ihn mit dem neuen Code neu.
|
|
# Der Container bleibt stehen, wg0 bleibt oben, niemand merkt etwas.
|
|
#
|
|
# `|| true`: laeuft der Container gerade nicht, ist das kein Grund, das
|
|
# Deployment abzubrechen — `docker compose up -d` oben hat ihn dann ohnehin
|
|
# frisch gestartet, und ein frischer Prozess hat den neuen Code schon.
|
|
docker compose restart queue scheduler reverb
|
|
docker compose exec -T queue-provisioning php artisan queue:restart >/dev/null 2>&1 || true
|
|
|
|
# Die Mitbewohner des Namensraums — aber nur, wenn er wirklich neu ist.
|
|
#
|
|
# Ein Prozess bleibt in dem Namensraum, in dem er gestartet ist. Wurde vpn-hub
|
|
# neu gebaut, lauschen Terminal-Bruecke, interner DNS und internes Gateway in
|
|
# einem, den es nicht mehr gibt. Nichts meldet dabei einen Fehler — im
|
|
# Gegenteil: `docker compose ps` sagt weiter „healthy", weil die Lebendpruefung
|
|
# ueber die Loopback-Adresse INNERHALB des verwaisten Namensraums laeuft. Nach
|
|
# aussen antwortet nginx mit 502.
|
|
#
|
|
# Vorher stand das hier bedingungslos. Das war richtig, solange der Namensraum
|
|
# dem Arbeiter gehoerte und bei jedem Update neu entstand; seit vpn-hub ist es
|
|
# der Ausnahmefall, und ein Neustart „nur zur Sicherheit" ist eine
|
|
# Unterbrechung ohne Anlass.
|
|
if [[ "$hub_rebuilt" == true ]]; then
|
|
docker compose restart terminal
|
|
fi
|
|
|
|
# Die gemerkten UDP-Ströme, wenn der Tunnel-Container neu gebaut wurde.
|
|
#
|
|
# Der Kernel merkt sich laufende UDP-Ströme samt Ziel. Ein neu gebauter Container
|
|
# bekommt eine neue Adresse im Compose-Netz, die Weiterleitung für den
|
|
# WireGuard-Port wird neu geschrieben — die gemerkten Einträge zeigen aber weiter
|
|
# auf den alten. Und sie verfallen nicht: WireGuard schickt alle 25 Sekunden ein
|
|
# Lebenszeichen und hält den kaputten Eintrag damit am Leben.
|
|
#
|
|
# Genau das hat einen Host nach einem Update dauerhaft draussen gelassen, während
|
|
# ein Telefon nach Aus- und Einschalten sofort wieder drin war — ein Client mit
|
|
# neuem Quellport bekommt einen frischen Eintrag, ein Host mit festem nicht.
|
|
#
|
|
# Braucht Root auf dem Wirt; dieses Skript läuft als Dienstbenutzer. Deshalb
|
|
# `sudo -n` (fragt nicht nach einem Passwort) und, wenn das nicht darf, eine
|
|
# deutliche Zeile statt eines stillen Fehlschlags.
|
|
if [[ "$hub_rebuilt" == true ]]; then
|
|
wg_port="$(grep -m1 '^WG_HUB_PORT=' .env 2>/dev/null | cut -d= -f2- | tr -d '"'"'"' ' || true)"
|
|
wg_port="${wg_port:-51820}"
|
|
|
|
if sudo -n conntrack -D -p udp --dport "$wg_port" >/dev/null 2>&1; then
|
|
log "Gemerkte UDP-Ströme auf Port $wg_port verworfen — die Tunnel bauen sich neu auf."
|
|
else
|
|
warn "Der Tunnel-Container wurde neu gebaut. Bestehende WireGuard-Sitzungen"
|
|
warn "zeigen jetzt auf den alten Container und kommen von allein NICHT zurück."
|
|
warn "Auf dem Wirt einmal ausführen:"
|
|
warn " sudo conntrack -D -p udp --dport $wg_port"
|
|
fi
|
|
fi
|
|
|
|
# AFTER the hub, always. Both VPN services live in the provisioning container's
|
|
# network namespace, and a process holds the namespace it started in — so once
|
|
# the hub is restarted they are listening inside one that no longer exists.
|
|
# Nothing errors; connections to the tunnel address are simply refused, which
|
|
# looks exactly like the gateway never having worked.
|
|
if [[ "$hub_rebuilt" == true ]] && grep -qE '^COMPOSE_PROFILES=.*vpn' .env 2>/dev/null; then
|
|
# And only once wg0 is back. The hub brings the interface up as part of its
|
|
# start command, so restarting these the instant the container is "started"
|
|
# has them binding an address that does not exist yet.
|
|
for _ in $(seq 1 30); do
|
|
docker compose exec -T queue-provisioning ip -4 addr show wg0 2>/dev/null | grep -q 'inet ' && break
|
|
sleep 2
|
|
done
|
|
docker compose --profile vpn restart vpn-dns vpn-gateway >/dev/null 2>&1 || true
|
|
fi
|
|
|
|
reconcile_vpn_readiness
|
|
|
|
# ── Host packages the update can install itself ──────────────────────────────
|
|
# This script cannot install anything on the host: it runs as the service
|
|
# account. What it can do is ask the root-owned helper that install-agent.sh put
|
|
# at a path the service account cannot write to, for one fixed command line that
|
|
# sudoers permits by name. See the long note in deploy/install-agent.sh for why
|
|
# it is built that way and not as sudo on this script.
|
|
#
|
|
# Deliberately after the restart and never fatal. rsync is needed by whoever
|
|
# collects the invoice archive, some minutes or hours from now — it is not worth
|
|
# a failed deployment, and an update that stops here would be a far bigger
|
|
# problem than a backup that has to wait for the next run.
|
|
if ! command -v rsync >/dev/null 2>&1; then
|
|
if [[ -x "$HOST_STEP" ]] && sudo -n "$HOST_STEP" ensure-rsync >/dev/null 2>&1; then
|
|
log "Installed rsync on the host (the invoice archive is collected over ssh)"
|
|
else
|
|
warn "rsync is not installed on this host — the invoice archive cannot be collected."
|
|
fi
|
|
fi
|
|
|
|
phase maintenance_off "Leaving maintenance mode"
|
|
in_app php artisan up >/dev/null
|
|
down=0
|
|
|
|
printf '%s' "$after" > "$STATE_FILE"
|
|
|
|
# Only now: the manifest is what the console reports, and it must mean "this
|
|
# came up". Written after `artisan up`, atomically, so a reader never catches it
|
|
# half-finished.
|
|
release_remember "$mode" "$source_ref"
|
|
release_write_manifest "$after" "$source_ref" "$mode"
|
|
|
|
# The panel's update button needs a host-side timer, and this script cannot
|
|
# install it: it runs as the service account and /etc/systemd is root's. Say so
|
|
# once per update rather than leaving the button disabled without explanation.
|
|
agent_hint=''
|
|
if ! systemctl list-unit-files clupilot-update-agent.timer >/dev/null 2>&1 \
|
|
|| ! systemctl is-enabled --quiet clupilot-update-agent.timer 2>/dev/null; then
|
|
agent_hint="the panel's update button stays disabled"
|
|
fi
|
|
|
|
# The proxy's console allowlist has to be wired once, as root, and an update
|
|
# never runs as root — so an existing installation would keep its hard-coded
|
|
# list and everything the owner changes in the console would do nothing.
|
|
# A host helper from before a step was added to it. Nothing is broken, but the
|
|
# new step silently does nothing — which is worse than an error, because the
|
|
# next person to look assumes it ran.
|
|
if [[ -z "$agent_hint" ]]; then
|
|
# `|| true` on the read, not on the whole test: an installation that has
|
|
# never had the helper reports 0 and lands in the same hint.
|
|
host_step_have="$( [[ -x "$HOST_STEP" ]] && "$HOST_STEP" contract 2>/dev/null || true )"
|
|
[[ "$host_step_have" =~ ^[0-9]+$ ]] || host_step_have=0
|
|
|
|
if (( host_step_have < HOST_STEP_NEEDS )); then
|
|
agent_hint="the update cannot install host packages such as rsync by itself"
|
|
fi
|
|
fi
|
|
|
|
if [[ -z "$agent_hint" ]] && command -v caddy >/dev/null 2>&1 \
|
|
&& [[ -f /etc/caddy/Caddyfile ]] \
|
|
&& ! grep -q 'clupilot-console-allow.conf' /etc/caddy/Caddyfile 2>/dev/null; then
|
|
agent_hint="the console's access list does not reach the reverse proxy"
|
|
fi
|
|
|
|
# Der Wächter. Ohne ihn bleibt ein Stapel, den ein abgebrochenes Update halb
|
|
# unten liegen ließ, genau so liegen — bis jemand nachsieht. Genau das ist
|
|
# einmal passiert und hat eine Stunde gekostet.
|
|
if [[ -z "$agent_hint" ]] \
|
|
&& ! systemctl list-unit-files clupilot-watchdog.timer >/dev/null 2>&1; then
|
|
agent_hint="der Wächter läuft nicht — ein abgebrochenes Update heilt dann nicht von allein"
|
|
fi
|
|
|
|
if [[ -n "$agent_hint" ]]; then
|
|
printf '\033[1;33m !\033[0m %s\n' "One-time setup missing — $agent_hint."
|
|
printf ' %s\n' "Run once: sudo bash $(pwd)/deploy/install-agent.sh"
|
|
fi
|
|
|
|
log "Done — $(release_version) on $(git rev-parse --short HEAD) (${source_ref})"
|