#!/usr/bin/env bash # # The host side of the panel's update button. # # The panel cannot run an update: it is www-data inside a container, and the # update restarts that container. So it writes a request into the checkout and # this agent — running on the host as the service account, on a timer — picks it # up, runs deploy/update.sh, and writes back what happened. # # It also answers the question the panel cannot answer on its own: is there # anything to update? That needs a `git fetch`, which needs credentials the # application deliberately does not have. # # Installed by deploy/install-agent.sh (as root, once). Runs every few minutes. # set -Eeuo pipefail ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT" # release_mode / release_source, so this agent and update.sh agree on what this # installation is following rather than each deciding for itself. # shellcheck source=lib/release.sh . "$ROOT/deploy/lib/release.sh" STATE_DIR="$ROOT/storage/app/deploy" REQUEST="$STATE_DIR/update-request.json" STATUS="$STATE_DIR/update-status.json" RUNLOG="$STATE_DIR/update-last-run.log" # The step update.sh is on, written by update.sh itself. Read back here so a run # that failed reports WHERE it failed instead of only that it did. PHASE_FILE="$STATE_DIR/update-phase" # The outcome of the last RUN, kept apart from the periodic check. Written into # one file, the next idle tick a minute later would overwrite a failure with # "idle" — and an operator would usually never see that the update failed. LASTRUN="$STATE_DIR/update-last-run.json" # The outcome of the last worker RESTART — its own file, not folded into # LASTRUN: a restart is not a deployment (no fetch, no maintenance mode, no # update.sh), and the two must be able to fail independently. RESTARTLAST="$STATE_DIR/restart-last-run.json" # Where a freshly generated collection key is handed to the panel, once. Written # 0600 by this agent, read and deleted by the panel — the same shape as the # initial admin password an instance holds until somebody notes it down. ARCHIVE_KEY="$STATE_DIR/archive-key.json" LOCK="$STATE_DIR/.agent.lock" # The reverse proxy's console allowlist, generated from the one the owner keeps # in the console. Without this the proxy has its own hard-coded list that runs # FIRST, so everything added in the console has no effect at all — and when the # owner's address changes they are turned away before reaching the page that # would have let them fix it. ALLOWFILE="${CONSOLE_ALLOW_FILE:-/etc/caddy/clupilot-console-allow.conf}" # A request older than this is abandoned. The panel already refuses to show one # as pending; the agent has to agree, or a request written while the agent was # stopped fires whenever the agent next starts — which could be days later. REQUEST_EXPIRES_MINUTES=30 # How often systemd starts this agent — see OnUnitActiveSec in install-agent.sh. # Reported so the console can say WHEN a queued update will start rather than # "in a few minutes", which is the whole reason an operator sits there wondering # whether anything is happening at all. CHECK_INTERVAL_MINUTES=1 # Can this agent be woken the moment a request lands, or only on the next tick? # # Installed by install-agent.sh as clupilot-update-agent.path. It is reported # rather than assumed because it lives on the host: a server that has not run # install-agent.sh since this landed still waits for the timer, and the console # has to say so instead of promising an answer it will not get. Reported by the # agent for the same reason CHECK_INTERVAL_MINUTES is — the panel cannot see # systemd from inside its container. REQUEST_WATCH=false if systemctl is-active --quiet clupilot-update-agent.path 2>/dev/null; then REQUEST_WATCH=true fi mkdir -p "$STATE_DIR" # One agent at a time. Two overlapping runs of update.sh fight over the # checkout, and the loser leaves it half-updated. exec 9>"$LOCK" flock -n 9 || exit 0 sync_console_allowlist() { [[ -w "$(dirname "$ALLOWFILE")" || -w "$ALLOWFILE" ]] || return 0 local generated # -u www-data: exec defaults to root, and this one runs every minute. An # artisan command that logs anything as root leaves storage/logs owned by # root, after which the application cannot append to its own log — and every # page that logs answers 500 with nothing written to say why. generated="$(docker compose exec -T -u www-data app php artisan clupilot:console-access caddy 2>/dev/null)" || return 0 # Never write an empty matcher: in Caddy that matches nothing, and the # console would be unreachable from everywhere including the shell. grep -q '@allowed remote_ip .' <<<"$generated" || return 0 local pending="$STATE_DIR/.caddy-reload-pending" if [[ ! -f "$ALLOWFILE" ]] || ! diff -q <(printf '%s\n' "$generated") "$ALLOWFILE" >/dev/null 2>&1; then printf '%s\n' "$generated" > "$ALLOWFILE" : > "$pending" fi # A reload that failed once must be retried. Without the marker the next # run sees an unchanged file, does nothing, and the proxy keeps the old # list indefinitely — the console and the proxy quietly disagreeing is the # exact failure this whole mechanism exists to prevent. if [[ -f "$pending" ]]; then # Reload, never restart: a restart drops every connection in flight, # including the one belonging to whoever just changed the list. if systemctl reload caddy >/dev/null 2>&1 || sudo -n systemctl reload caddy >/dev/null 2>&1; then rm -f "$pending" fi fi } # The tunnel gateway serves a certificate the PUBLIC Caddy owns and renews. Its # tls directive loads the file once at startup, so after a renewal it would go # on presenting the old one until something restarted it — and eventually serve # an expired certificate to the only people who can reach the console. sync_vpn_certificate() { local stamp seen="$STATE_DIR/.vpn-cert-stamp" path path="$(sed -n 's/^VPN_CERT_PATH=//p' "$ROOT/.env" 2>/dev/null | tail -1)" [[ -n "$path" ]] || return 0 # Read from INSIDE the gateway, not from the host. Caddy's storage belongs # to its own service account and is deliberately left that way — this agent # runs unprivileged and cannot even traverse it, so a host-side check would # silently never fire and the gateway would serve an expired certificate. stamp="$(docker compose exec -T vpn-gateway stat -c %Y "$path" 2>/dev/null | tr -d '\r\n')" [[ -n "$stamp" ]] || return 0 if [[ ! -f "$seen" ]] || [[ "$(cat "$seen" 2>/dev/null)" != "$stamp" ]]; then # Stamped only on success. Recording a restart that did not happen means # the next run considers this certificate handled and never retries — # leaving the gateway serving the old one until the NEXT renewal. if docker compose --profile vpn restart vpn-gateway >/dev/null 2>&1; then printf '%s' "$stamp" > "$seen" fi fi } sync_console_allowlist sync_vpn_certificate MODE="$(release_mode)" BRANCH="$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo main)" TARGET_RELEASE='' json_escape() { printf '%s' "${1-}" | sed 's/\\/\\\\/g; s/"/\\"/g; s/\t/\\t/g' | tr -d '\n\r' } # The step update.sh last announced, if any. Two tab-separated fields: key, time. # # Not copied into the status document below: this agent BLOCKS for the whole # run, so anything it wrote there would be frozen at the first step. The panel # reads the phase file itself, which update.sh keeps current. Here it is only # needed once, to record which step a failed run died at. read_phase() { cut -f1 "$PHASE_FILE" 2>/dev/null | tr -d '\r\n' || true; } write_status() { local state="$1" error="${2-}" cat > "$STATUS.tmp" < "$LASTRUN.tmp" </dev/null || echo '')")", "error": "$(json_escape "$error")", "exit_code": ${EXIT_CODE:-null} } EOF mv -f "$LASTRUN.tmp" "$LASTRUN" } # The outcome of the last worker restart. Survives until the next one. write_restart() { local state="$1" error="${2-}" cat > "$RESTARTLAST.tmp" </dev/null; then # Newest by version order, not by tag date. Through the helper rather than # `| head -1`: head exits after one line, git takes SIGPIPE, and pipefail # ends the agent — see release_newest_tag. NEWEST_TAG="$(release_newest_tag)" NEWEST_VERSION="${NEWEST_TAG#v}" if [[ -n "$NEWEST_TAG" ]] && release_version_gt "$NEWEST_VERSION" "$DEPLOYED_VERSION"; then # How many releases ahead, so the console can say "2 Aktualisierungen" # rather than only that something exists. Both helpers live in # deploy/lib/release.sh so a test can run them — the version that lived # here ended the agent on every tick under `set -e`, and no test in a # PHP suite can reach a bash pipeline. BEHIND="$(release_tags_ahead "$DEPLOYED_VERSION")" TARGET_RELEASE="$NEWEST_TAG" REMOTE_COMMIT="$(git rev-parse "refs/tags/${NEWEST_TAG}^{commit}" 2>/dev/null || echo '')" else # Includes "no tags at all yet": nothing has been released, so there is # nothing to offer, and saying "up to date" is the truth. BEHIND=0 fi else # Reported, not swallowed: "cannot reach the repository" and "already up to # date" look identical from the panel otherwise, and only one is fine. A # code, not a sentence: the panel is translated and this script is not, so a # German string here would surface in the English interface. FETCH_ERROR='repo_unreachable' fi # ── Is an update asked for, and is it still current? ────────────────────── if [[ ! -f "$REQUEST" ]]; then write_status idle "$FETCH_ERROR" exit 0 fi REQUEST_AGE_MINUTES="$(( ( $(date +%s) - $(stat -c %Y "$REQUEST" 2>/dev/null || date +%s) ) / 60 ))" if (( REQUEST_AGE_MINUTES > REQUEST_EXPIRES_MINUTES )); then # Written while this agent was stopped. Running it now is not what was # asked for — it was asked for then. rm -f "$REQUEST" write_status idle "$FETCH_ERROR" exit 0 fi # What was actually asked for: a real run, only the check the block above # already performed, or a worker restart. A distinct field, not a flag folded # into the existing request — "check", "run" and "restart" are three # different things the panel can ask for, not one request with a modifier. # Read before the file is consumed; a request written by an older panel has # no such field and is treated as a run, which is everything a request has # ever meant until now. REQUEST_KIND="$(sed -n 's/.*"kind"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$REQUEST" 2>/dev/null | head -1)" # The whole document, kept for the kinds that carry more than a kind. Read here # for the same reason the kind is: the file is consumed on the next line, and a # field fetched afterwards would be fetched from a file that no longer exists. REQUEST_BODY="$(cat "$REQUEST" 2>/dev/null || true)" # Consumed BEFORE anything below runs, not after: update.sh restarts the # container stack and may well kill this shell with it. A request left in # place would be picked up again on the next tick and update in a loop. rm -f "$REQUEST" # ── A collection key for the invoice archive ────────────────────────────── # Everything an operator otherwise does by hand across three machines: generate # a keypair, install the public half restricted to one directory, hand over the # private half. Done here because it is all on the HOST — the panel is www-data # in a container and owns none of it. # # The restriction is rrsync, which ships with rsync: it parses what the client # asked for, forces read-only and forces the directory. The obvious alternative, # pinning the exact rsync option string in authorized_keys, breaks the moment # either side is a different rsync version — and breaks silently, as a refusal # with no reason given. if [[ "$REQUEST_KIND" == "archive-key" ]]; then ARCHIVE_PATH="$(sed -n 's/.*"archive_path"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' <<<"$REQUEST_BODY" | head -1)" KEY_NAME="$(sed -n 's/.*"label"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' <<<"$REQUEST_BODY" | head -1)" KEY_NAME="${KEY_NAME:-clupilot-archiv}" ARCHIVE_ERROR='' if [[ -z "$ARCHIVE_PATH" || "$ARCHIVE_PATH" != /* ]]; then ARCHIVE_ERROR='archive_path_invalid' elif ! command -v rrsync >/dev/null 2>&1; then # Named, not guessed at. Without rrsync the only way to restrict the key # is the brittle option string, and issuing a key that is NOT restricted # when the panel says it is would be worse than issuing none. ARCHIVE_ERROR='rrsync_missing' else HOME_DIR="$(getent passwd "$(id -un)" | cut -d: -f6)" HOME_DIR="${HOME_DIR:-$HOME}" KEY_FILE="$(mktemp "$STATE_DIR/.newkey.XXXXXX")" rm -f "$KEY_FILE" if ssh-keygen -q -t ed25519 -N '' -C "$KEY_NAME" -f "$KEY_FILE" 2>/dev/null; then mkdir -p "$HOME_DIR/.ssh" chmod 700 "$HOME_DIR/.ssh" touch "$HOME_DIR/.ssh/authorized_keys" chmod 600 "$HOME_DIR/.ssh/authorized_keys" # restrict switches off every forwarding and the pty; the forced # command means this key can do one thing and read one directory. printf 'command="/usr/bin/rrsync -ro %s",restrict %s\n' \ "$ARCHIVE_PATH" "$(cat "$KEY_FILE.pub")" >> "$HOME_DIR/.ssh/authorized_keys" umask 077 printf '{\n "created_at": "%s",\n "label": "%s",\n "path": "%s",\n "private_key": "%s"\n}\n' \ "$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ "$(json_escape "$KEY_NAME")" \ "$(json_escape "$ARCHIVE_PATH")" \ "$(sed ':a;N;$!ba;s/\n/\\n/g' "$KEY_FILE" | sed 's/"/\\"/g')" \ > "$ARCHIVE_KEY" chmod 600 "$ARCHIVE_KEY" rm -f "$KEY_FILE" "$KEY_FILE.pub" else ARCHIVE_ERROR='keygen_failed' rm -f "$KEY_FILE" "$KEY_FILE.pub" fi fi write_status idle "$ARCHIVE_ERROR" exit 0 fi if [[ "$REQUEST_KIND" == "check" ]]; then # Nothing further to do: the fetch and the BEHIND calculation above # already answered it. The operator asked to look, not to touch # anything, so this tick ends exactly like one that found no request — # no maintenance mode, no restart, no phase file. write_status idle "$FETCH_ERROR" exit 0 fi if [[ "$REQUEST_KIND" == "restart" ]]; then # queue, queue-provisioning, scheduler, reverb — and ONLY those four. # `app` is deliberately never in this list: it is the container serving # THIS very request (the panel wrote the request from inside it), so # restarting it would kill the request that asked for the restart before # the agent could even answer it. It is also unnecessary — PHP-FPM # bootstraps a fresh Laravel application on every request it serves, so # `app` already picks up whatever the panel just wrote to .env (and # cleared from the config cache, which the panel also does itself, see # App\Livewire\Admin\Integrations::saveEnv()) without being touched here. # Not a deployment: no fetch was needed for this, no maintenance mode, no # phase file, no update.sh. if docker compose restart queue queue-provisioning scheduler reverb >/dev/null 2>&1; then write_restart succeeded else write_restart failed restart_failed fi write_status idle "$FETCH_ERROR" exit 0 fi # Only ever to a tag, and only to one that is genuinely newer. Checked BEFORE # anything announces a run: a request with nothing to install is answered, not # obeyed. The console does not offer the button in that state, so this is a # stale tab or a forged request — and obeying it would take the site into # maintenance mode to install what is already installed. if [[ -z "$TARGET_RELEASE" ]]; then write_status idle "$FETCH_ERROR" exit 0 fi # The previous run's last step is not this run's first one. Left in place, the # console would show a stale phase for the seconds before update.sh writes its # own — and on a run that dies before writing any, for the whole run. rm -f "$PHASE_FILE" STARTED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)" write_status running "$FETCH_ERROR" set +e RELEASE="$TARGET_RELEASE" "$ROOT/deploy/update.sh" > "$RUNLOG" 2>&1 RESULT=$? set -e FINISHED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)" LOCAL_COMMIT="$(git rev-parse HEAD 2>/dev/null || echo '')" EXIT_CODE="$RESULT" if [[ $RESULT -eq 0 ]]; then BEHIND=0 TARGET_RELEASE='' # Dropped before anything records it: a run that finished has no step it is # on, and the last one it passed would read as "still at Rebuilding assets". rm -f "$PHASE_FILE" write_run succeeded '' write_status idle '' else # The phase file stays: it names the step that failed, and the console shows # it next to the log so the operator does not have to read the log to find # out where it stopped. write_run failed 'update_failed' write_status idle "$FETCH_ERROR" fi exit 0