#!/usr/bin/env bash # # Installs the update agent's systemd timer. Root, once per machine, idempotent. # # Separate from install.sh because the installed base never runs install.sh # again — existing servers upgrade through deploy/update.sh, which runs as the # service account and cannot write to /etc/systemd. Without this as its own # entry point, every server that already exists would show the update button # permanently disabled. # # sudo bash deploy/install-agent.sh # set -euo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" APP_USER="${APP_USER:-clupilot}" [[ $EUID -eq 0 ]] || { echo "Please run as root: sudo bash deploy/install-agent.sh" >&2; exit 1; } id -u "$APP_USER" >/dev/null 2>&1 || { echo "No such account: $APP_USER" >&2; exit 1; } install -o "$APP_USER" -g "$APP_USER" -d "$ROOT/storage/app/deploy" chmod +x "$ROOT/deploy/update-agent.sh" # ── The few things an update needs root for ────────────────────────────────── # An update runs as the service account. Everything root-owned on the host is # therefore out of its reach, and each such thing turns into "log into every # server once and paste this" — which is exactly the work this project exists to # remove. # # The obvious fix is to let the service account run THIS script under sudo. That # would be handing it root outright: it owns the checkout, so it can rewrite the # very script it is allowed to run privileged. A grant is only worth anything if # the holder cannot change what it grants. # # So the privileged part is written OUT of the checkout, to a root-owned path, # from a here-document rather than copied — the service account has no way to # influence a single byte of it. What it may then do as root is one fixed # command line, enumerated in sudoers, and nothing else. # # CONTRACT is the version of that fixed set. deploy/update.sh compares it and # says so when this script has to be run again — a new step added here is # useless on a server still carrying the old helper, and silently useless is the # failure mode worth spending a line on. HOST_STEP=/usr/local/sbin/clupilot-host-step cat > "$HOST_STEP" <<'EOF' #!/usr/bin/env bash # # The root-owned half of a CluPilot update. Installed by deploy/install-agent.sh # and never read from the application checkout — see the note there. # # One argument, from the list below. Nothing takes free-form input. set -euo pipefail CONTRACT=1 case "${1:-}" in contract) echo "$CONTRACT" ;; ensure-rsync) # rsync has to be on the HOST: a backup that collects the invoice # archive connects by ssh, and sshd starts `rsync --server` out here, # not in a container. rrsync comes with it, and that is what locks the # collecting key to one directory. command -v rsync >/dev/null 2>&1 && exit 0 command -v apt-get >/dev/null 2>&1 || { echo "no apt-get on this host" >&2; exit 3; } export DEBIAN_FRONTEND=noninteractive apt-get update -qq >/dev/null 2>&1 || true apt-get install -y -qq rsync >/dev/null command -v rsync >/dev/null 2>&1 || { echo "rsync still missing after install" >&2; exit 4; } ;; *) # Refused by name. sudoers already permits only the exact command lines # above, so this is the second of two locks, not the only one. echo "unknown step: ${1:-}" >&2 exit 64 ;; esac EOF chown root:root "$HOST_STEP" chmod 0755 "$HOST_STEP" # The command line in full, not just the path: sudoers matches arguments too, so # written this way the grant is for `clupilot-host-step ensure-rsync` and for # nothing else the file might ever learn to do. cat > /etc/sudoers.d/clupilot-host-step </dev/null || { echo " ! sudoers fragment rejected — removing it. Updates will ask for rsync by hand." rm -f /etc/sudoers.d/clupilot-host-step } # And straight away, through the same helper rather than beside it — one # implementation, exercised on every install instead of only when an update # happens to find rsync missing. if ! command -v rsync >/dev/null 2>&1; then echo " Installing rsync (the invoice archive is collected over ssh)" "$HOST_STEP" ensure-rsync \ || echo " ! Could not install rsync — the invoice archive cannot be collected until it is there." fi cat > /etc/systemd/system/clupilot-update-agent.service < /etc/systemd/system/clupilot-update-agent.service.d/rate-limit.conf <<'EOF' [Unit] StartLimitIntervalSec=0 EOF cat > /etc/systemd/system/clupilot-update-agent.timer <<'EOF' [Unit] Description=Check for CluPilot updates and run requested ones [Timer] # Every minute: a `git fetch` against one remote is not a nuisance at this rate, # and this is now only the FALLBACK — the path unit below wakes the agent the # moment somebody asks for something. What the timer still decides is how fresh # the "is there a new version" reading is on a console nobody is looking at. OnBootSec=3min OnUnitActiveSec=1min AccuracySec=5s [Install] WantedBy=timers.target EOF # ── Answering a question at the moment it is asked ─────────────────────────── # The timer above decides how often the agent looks on its own. It must not also # decide how long someone waits after pressing a button. # # "Nach Aktualisierungen suchen" is a QUESTION. Routed through the timer it was # answered on the next tick — up to five minutes later on a server whose timer # still had the old interval — and the console, having nothing else to show, # counted down to it. An operator who asked what was new was told an update # would start in 3:44. Nothing was starting; nothing had been asked to start. # # So the request file gets its own trigger. The panel writes it inside the # container, the checkout is bind-mounted, and the host sees the write on the # same inode — the agent is woken within a second, for a check and for a real # run alike. # # PathChanged, deliberately, not PathExists: PathExists re-arms as soon as the # service goes inactive, so a request the agent could not consume — it holds a # lock while an update runs — would restart it in a tight loop for the length of # that update. PathChanged fires once, when the file is written and closed. A # trigger missed because the agent was busy is picked up by the timer. cat > /etc/systemd/system/clupilot-update-agent.path </dev/null 2>&1 && [[ -f "$CADDYFILE" ]]; then echo " Wiring the console allowlist into Caddy" # Seed it before anything imports it: an import of a missing file makes the # whole proxy config invalid, and that takes the public site down too. ALLOWLIST_READY=0 if [[ -s "$ALLOWFILE" ]] && grep -q '@allowed remote_ip .' "$ALLOWFILE"; then ALLOWLIST_READY=1 else # Compose discovers its own file from the working directory — naming it # here guessed wrong once. # -u www-data, like every other artisan call: exec defaults to root, and # a command that logs would leave storage/logs owned by root — after # which the application cannot append to its own log at all, and every # page that logs answers 500. That is not hypothetical; it happened. if ( cd "$ROOT" && runuser -u "$APP_USER" -- docker compose exec -T -u www-data app \ php artisan clupilot:console-access caddy ) > "$ALLOWFILE.new" 2>/dev/null \ && grep -q '@allowed remote_ip .' "$ALLOWFILE.new"; then mv -f "$ALLOWFILE.new" "$ALLOWFILE" ALLOWLIST_READY=1 else # NO fallback matcher here. A loopback-only list looks like a valid # allowlist, passes validation, and locks out every remote operator # the moment Caddy reloads — because the application happened to be # unavailable for the ten seconds this ran. rm -f "$ALLOWFILE.new" echo " ! Could not read the console allowlist from the application." echo " Leaving the proxy configuration untouched." fi fi [[ -f "$ALLOWFILE" ]] && { chown "$APP_USER":root "$ALLOWFILE"; chmod 0644 "$ALLOWFILE"; } # Replace a hard-coded matcher with the import, once. Backed up and # validated first — a broken Caddyfile is an outage of everything, not just # of the console. if [[ "$ALLOWLIST_READY" == "1" ]] && ! grep -q "import $ALLOWFILE" "$CADDYFILE"; then # Exactly one, or none of them. An unqualified substitution would # rewrite a matcher belonging to some other site — and the result would # still validate, so the first sign of it would be that site's access # control quietly changing. matches="$(grep -cE '^[[:space:]]*@allowed remote_ip ' "$CADDYFILE" || true)" if [[ "$matches" == "1" ]]; then cp -a "$CADDYFILE" "$CADDYFILE.clupilot.bak" sed -i "s|^\([[:space:]]*\)@allowed remote_ip .*|\1import $ALLOWFILE|" "$CADDYFILE" if caddy validate --config "$CADDYFILE" --adapter caddyfile >/dev/null 2>&1; then if systemctl reload caddy >/dev/null 2>&1; then echo " Console allowlist is now managed from the console." else # Handed to the agent. Without this marker the file already # matches what the agent would generate, so nothing would # ever trigger another reload and the proxy would keep the # old hard-coded list indefinitely. : > "$ROOT/storage/app/deploy/.caddy-reload-pending" chown "$APP_USER":"$APP_USER" "$ROOT/storage/app/deploy/.caddy-reload-pending" echo " ! Caddy did not reload — the agent retries within a minute." fi else echo " ! Caddy rejected the change — restoring the previous config." mv -f "$CADDYFILE.clupilot.bak" "$CADDYFILE" fi else # Said out loud rather than assumed. Nothing to replace, or more # than one candidate and no way to tell which belongs to the # console — either way, carrying on silently would leave the # console's allowlist a decoration while the proxy kept its own. echo " ! Found $matches single-line '@allowed remote_ip' entries in $CADDYFILE (expected exactly 1)." echo " Put this inside the console's site block yourself, then reload Caddy:" echo " import $ALLOWFILE" fi fi # Narrow on purpose: one command, no arguments, nothing else. cat > /etc/sudoers.d/clupilot-caddy-reload </dev/null || rm -f /etc/sudoers.d/clupilot-caddy-reload fi # ── The console inside the tunnel ──────────────────────────────────────────── # The gateway serves the console on the WireGuard hub address using the # certificate the public Caddy already obtains for that same hostname — a # certificate is bound to the NAME, not to the address serving it. Finding it # has to happen here, as root: the storage is Caddy's and the paths contain the # ACME directory, which is not something to ask an operator to type. VPN_HOST="$(sed -n 's/^VPN_INTERNAL_HOST=//p' "$ROOT/.env" 2>/dev/null | tail -1)" if [[ -z "$VPN_HOST" ]]; then # Cleared. Reconciled here too, because the updater exits early when the # checkout has not moved — so without this there may be no path at all that # stops the services, and clients would keep being handed a resolver that # is no longer meant to exist. runuser -u "$APP_USER" -- sh -c " cd '$ROOT' if grep -qE '^COMPOSE_PROFILES=.*vpn' .env 2>/dev/null || grep -qE '^VPN_READY=' .env 2>/dev/null; then docker compose --profile vpn stop vpn-dns vpn-gateway >/dev/null 2>&1 || true docker compose --profile vpn rm -f vpn-dns vpn-gateway >/dev/null 2>&1 || true sed -i '/^VPN_CERT_PATH=/d;/^VPN_KEY_PATH=/d;/^VPN_READY=/d' .env profiles=\"\$(sed -n 's/^COMPOSE_PROFILES=//p' .env | tail -1)\" profiles=\"\$(printf '%s' \"\$profiles\" | tr ',' '\n' | grep -vx 'vpn' | grep -v '^\$' | paste -sd, - || true)\" sed -i '/^COMPOSE_PROFILES=/d' .env [ -n \"\$profiles\" ] && printf 'COMPOSE_PROFILES=%s\n' \"\$profiles\" >> .env docker compose exec -T -u www-data app php artisan config:clear >/dev/null 2>&1 || true echo ' Tunnel gateway disabled — VPN_INTERNAL_HOST is empty.' fi " || true fi if [[ -n "$VPN_HOST" ]]; then # From the project's .env, because that is the file Compose reads for the # mount. Looking only at the default would report "no certificate" on an # installation whose storage is somewhere else. CADDY_DATA="$(sed -n 's/^CADDY_DATA_DIR=//p' "$ROOT/.env" 2>/dev/null | tail -1)" CADDY_DATA="${CADDY_DATA:-${CADDY_DATA_DIR:-/var/lib/caddy/.local/share/caddy}}" # -print -quit, not | head -1: under `set -o pipefail` a find that is still # producing output when head exits returns SIGPIPE, and the whole agent # installation aborts — on the perfectly ordinary case of one certificate # existing under two issuer directories. cert="$(find "$CADDY_DATA/certificates" -name "${VPN_HOST}.crt" -print -quit 2>/dev/null)" key="${cert%.crt}.key" if [[ -n "$cert" && -f "$key" ]]; then # Paths as the gateway container sees them: the storage is mounted at /certs. rel="${cert#$CADDY_DATA/}" runuser -u "$APP_USER" -- sh -c " cd '$ROOT' sed -i '/^VPN_CERT_PATH=/d;/^VPN_KEY_PATH=/d' .env printf 'VPN_CERT_PATH=/certs/%s\n' '$rel' >> .env printf 'VPN_KEY_PATH=/certs/%s\n' '${rel%.crt}.key' >> .env " # No chmod. The gateway container runs as root and reads these through # the read-only mount already; making the private key world-readable # would let every local account copy the console's TLS identity. # Enabled AND started here. The updater exits early when the checkout # has not moved, so leaving this to "the next deploy" leaves the # gateway stopped and the success message a lie. runuser -u "$APP_USER" -- sh -c " cd '$ROOT' grep -qE '^COMPOSE_PROFILES=.*vpn' .env \ || { grep -qE '^COMPOSE_PROFILES=' .env \ && sed -i 's/^COMPOSE_PROFILES=\\(.*\\)\$/COMPOSE_PROFILES=\\1,vpn/' .env \ || printf 'COMPOSE_PROFILES=vpn\n' >> .env; } docker compose --profile vpn up -d vpn-dns vpn-gateway " >/dev/null 2>&1 || true # `up -d` succeeds once the containers are CREATED. One that starts and # exits — Caddy unable to read the certificate, or to bind the hub # address — still counts as success there, so the state is checked. sleep 3 vpn_started=0 if runuser -u "$APP_USER" -- sh -c "cd '$ROOT' && docker compose --profile vpn ps --status running --services" 2>/dev/null \ | grep -q '^vpn-gateway$' \ && runuser -u "$APP_USER" -- sh -c "cd '$ROOT' && docker compose --profile vpn ps --status running --services" 2>/dev/null \ | grep -q '^vpn-dns$'; then vpn_started=1 fi # The marker, and only on success. Certificate paths alone would report # the tunnel ready after a failed start, and the application would hand # out configs naming a resolver that never came up. runuser -u "$APP_USER" -- sh -c " cd '$ROOT' sed -i '/^VPN_READY=/d' .env printf 'VPN_READY=%s\n' '$([[ $vpn_started -eq 1 ]] && echo true || echo false)' >> .env " if [[ $vpn_started -ne 1 ]]; then echo " ! Could not start the tunnel gateway — check: docker compose --profile vpn up -d" fi # The application caches its configuration. Without rebuilding it here, # vpn_ready stays false and client configs keep omitting the tunnel # resolver — the gateway running but nothing pointed at it. runuser -u "$APP_USER" -- sh -c " cd '$ROOT' docker compose exec -T -u www-data app php artisan config:clear >/dev/null 2>&1 docker compose exec -T -u www-data app php artisan config:cache >/dev/null 2>&1 " || true # Only when it is true. Announcing success while the gateway failed to # start is worse than silence: it sends the operator looking somewhere # else entirely. if [[ $vpn_started -eq 1 ]]; then echo " Console reachable inside the tunnel at https://$VPN_HOST" fi else # Nothing usable for the CURRENT hostname. Leaving the old paths, the # profile and the readiness marker in place would have the application # advertise the renamed endpoint as ready while the gateway still # served the previous name. runuser -u "$APP_USER" -- sh -c " cd '$ROOT' sed -i '/^VPN_CERT_PATH=/d;/^VPN_KEY_PATH=/d;/^VPN_READY=/d' .env docker compose --profile vpn stop vpn-dns vpn-gateway >/dev/null 2>&1 || true # And the profile with it, or the next ordinary `up -d` starts a # Caddy with empty tls paths that can only crash. profiles=\"\$(sed -n 's/^COMPOSE_PROFILES=//p' .env | tail -1)\" profiles=\"\$(printf '%s' \"\$profiles\" | tr ',' '\n' | grep -vx 'vpn' | grep -v '^\$' | paste -sd, - || true)\" sed -i '/^COMPOSE_PROFILES=/d' .env [ -n \"\$profiles\" ] && printf 'COMPOSE_PROFILES=%s\n' \"\$profiles\" >> .env docker compose exec -T -u www-data app php artisan config:clear >/dev/null 2>&1 || true " || true echo " ! No certificate for $VPN_HOST in $CADDY_DATA yet." echo " Reach https://$VPN_HOST publicly once so Caddy obtains it, then run this again." fi fi systemctl daemon-reload systemctl enable --now clupilot-update-agent.timer >/dev/null systemctl enable --now clupilot-update-agent.path >/dev/null # Run it once now, so the panel has a status to show instead of "never reported # in" for the first minute. systemctl start clupilot-update-agent.service || true echo "Update agent installed and running (clupilot-update-agent.timer)."