CluPilotCloud/deploy/update-agent.sh

#!/usr/bin/env bash
#
# The host side of the panel's update button.
#
# The panel cannot run an update: it is www-data inside a container, and the
# update restarts that container. So it writes a request into the checkout and
# this agent — running on the host as the service account, on a timer — picks it
# up, runs deploy/update.sh, and writes back what happened.
#
# It also answers the question the panel cannot answer on its own: is there
# anything to update? That needs a `git fetch`, which needs credentials the
# application deliberately does not have.
#
# Installed by deploy/install-agent.sh (as root, once). Runs every few minutes.
#
set -Eeuo pipefail

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
cd "$ROOT"

# release_mode / release_source, so this agent and update.sh agree on what this
# installation is following rather than each deciding for itself.
# shellcheck source=lib/release.sh
. "$ROOT/deploy/lib/release.sh"

STATE_DIR="$ROOT/storage/app/deploy"
REQUEST="$STATE_DIR/update-request.json"
STATUS="$STATE_DIR/update-status.json"
RUNLOG="$STATE_DIR/update-last-run.log"
# The step update.sh is on, written by update.sh itself. Read back here so a run
# that failed reports WHERE it failed instead of only that it did.
PHASE_FILE="$STATE_DIR/update-phase"
# The outcome of the last RUN, kept apart from the periodic check. Written into
# one file, the next idle tick a minute later would overwrite a failure with
# "idle" — and an operator would usually never see that the update failed.
LASTRUN="$STATE_DIR/update-last-run.json"
LOCK="$STATE_DIR/.agent.lock"
# The reverse proxy's console allowlist, generated from the one the owner keeps
# in the console. Without this the proxy has its own hard-coded list that runs
# FIRST, so everything added in the console has no effect at all — and when the
# owner's address changes they are turned away before reaching the page that
# would have let them fix it.
ALLOWFILE="${CONSOLE_ALLOW_FILE:-/etc/caddy/clupilot-console-allow.conf}"

# A request older than this is abandoned. The panel already refuses to show one
# as pending; the agent has to agree, or a request written while the agent was
# stopped fires whenever the agent next starts — which could be days later.
REQUEST_EXPIRES_MINUTES=30

# How often systemd starts this agent — see OnUnitActiveSec in install-agent.sh.
# Reported so the console can say WHEN a queued update will start rather than
# "in a few minutes", which is the whole reason an operator sits there wondering
# whether anything is happening at all.
CHECK_INTERVAL_MINUTES=1

# Can this agent be woken the moment a request lands, or only on the next tick?
#
# Installed by install-agent.sh as clupilot-update-agent.path. It is reported
# rather than assumed because it lives on the host: a server that has not run
# install-agent.sh since this landed still waits for the timer, and the console
# has to say so instead of promising an answer it will not get. Reported by the
# agent for the same reason CHECK_INTERVAL_MINUTES is — the panel cannot see
# systemd from inside its container.
REQUEST_WATCH=false
if systemctl is-active --quiet clupilot-update-agent.path 2>/dev/null; then
    REQUEST_WATCH=true
fi

mkdir -p "$STATE_DIR"

# One agent at a time. Two overlapping runs of update.sh fight over the
# checkout, and the loser leaves it half-updated.
exec 9>"$LOCK"
flock -n 9 || exit 0

sync_console_allowlist() {
    [[ -w "$(dirname "$ALLOWFILE")" || -w "$ALLOWFILE" ]] || return 0

    local generated
    # -u www-data: exec defaults to root, and this one runs every minute. An
    # artisan command that logs anything as root leaves storage/logs owned by
    # root, after which the application cannot append to its own log — and every
    # page that logs answers 500 with nothing written to say why.
    generated="$(docker compose exec -T -u www-data app php artisan clupilot:console-access caddy 2>/dev/null)" || return 0
    # Never write an empty matcher: in Caddy that matches nothing, and the
    # console would be unreachable from everywhere including the shell.
    grep -q '@allowed remote_ip .' <<<"$generated" || return 0

    local pending="$STATE_DIR/.caddy-reload-pending"

    if [[ ! -f "$ALLOWFILE" ]] || ! diff -q <(printf '%s\n' "$generated") "$ALLOWFILE" >/dev/null 2>&1; then
        printf '%s\n' "$generated" > "$ALLOWFILE"
        : > "$pending"
    fi

    # A reload that failed once must be retried. Without the marker the next
    # run sees an unchanged file, does nothing, and the proxy keeps the old
    # list indefinitely — the console and the proxy quietly disagreeing is the
    # exact failure this whole mechanism exists to prevent.
    if [[ -f "$pending" ]]; then
        # Reload, never restart: a restart drops every connection in flight,
        # including the one belonging to whoever just changed the list.
        if systemctl reload caddy >/dev/null 2>&1 || sudo -n systemctl reload caddy >/dev/null 2>&1; then
            rm -f "$pending"
        fi
    fi
}

# The tunnel gateway serves a certificate the PUBLIC Caddy owns and renews. Its
# tls directive loads the file once at startup, so after a renewal it would go
# on presenting the old one until something restarted it — and eventually serve
# an expired certificate to the only people who can reach the console.
sync_vpn_certificate() {
    local stamp seen="$STATE_DIR/.vpn-cert-stamp" path

    path="$(sed -n 's/^VPN_CERT_PATH=//p' "$ROOT/.env" 2>/dev/null | tail -1)"
    [[ -n "$path" ]] || return 0

    # Read from INSIDE the gateway, not from the host. Caddy's storage belongs
    # to its own service account and is deliberately left that way — this agent
    # runs unprivileged and cannot even traverse it, so a host-side check would
    # silently never fire and the gateway would serve an expired certificate.
    stamp="$(docker compose exec -T vpn-gateway stat -c %Y "$path" 2>/dev/null | tr -d '\r\n')"
    [[ -n "$stamp" ]] || return 0

    if [[ ! -f "$seen" ]] || [[ "$(cat "$seen" 2>/dev/null)" != "$stamp" ]]; then
        # Stamped only on success. Recording a restart that did not happen means
        # the next run considers this certificate handled and never retries —
        # leaving the gateway serving the old one until the NEXT renewal.
        if docker compose --profile vpn restart vpn-gateway >/dev/null 2>&1; then
            printf '%s' "$stamp" > "$seen"
        fi
    fi
}

sync_console_allowlist
sync_vpn_certificate

MODE="$(release_mode)"
BRANCH="$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo main)"
TARGET_RELEASE=''

json_escape() {
    printf '%s' "${1-}" | sed 's/\\/\\\\/g; s/"/\\"/g; s/\t/\\t/g' | tr -d '\n\r'
}

# The step update.sh last announced, if any. Two tab-separated fields: key, time.
#
# Not copied into the status document below: this agent BLOCKS for the whole
# run, so anything it wrote there would be frozen at the first step. The panel
# reads the phase file itself, which update.sh keeps current. Here it is only
# needed once, to record which step a failed run died at.
read_phase() { cut -f1 "$PHASE_FILE" 2>/dev/null | tr -d '\r\n' || true; }

write_status() {
    local state="$1" error="${2-}"
    cat > "$STATUS.tmp" <<EOF
{
  "state": "$(json_escape "$state")",
  "mode": "$(json_escape "$MODE")",
  "checked_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
  "check_interval_minutes": ${CHECK_INTERVAL_MINUTES},
  "request_watch": ${REQUEST_WATCH},
  "started_at": "$(json_escape "${STARTED_AT:-}")",
  "finished_at": "$(json_escape "${FINISHED_AT:-}")",
  "local_commit": "$(json_escape "${LOCAL_COMMIT:-}")",
  "remote_commit": "$(json_escape "${REMOTE_COMMIT:-}")",
  "target_release": "$(json_escape "${TARGET_RELEASE:-}")",
  "behind": ${BEHIND:-null},
  "branch": "$(json_escape "$BRANCH")",
  "error": "$(json_escape "$error")"
}
EOF
    # Replaced atomically: the panel reads this on every settings render, and
    # half a JSON document is exactly the kind of thing that breaks the page at
    # the moment someone needs it.
    mv -f "$STATUS.tmp" "$STATUS"
}

# The outcome of an actual run. Survives every later idle check.
write_run() {
    local state="$1" error="${2-}"
    cat > "$LASTRUN.tmp" <<EOF
{
  "state": "$(json_escape "$state")",
  "started_at": "$(json_escape "${STARTED_AT:-}")",
  "finished_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
  "phase": "$(json_escape "$(read_phase)")",
  "commit": "$(json_escape "$(git rev-parse HEAD 2>/dev/null || echo '')")",
  "error": "$(json_escape "$error")",
  "exit_code": ${EXIT_CODE:-null}
}
EOF
    mv -f "$LASTRUN.tmp" "$LASTRUN"
}

LOCAL_COMMIT="$(git rev-parse HEAD 2>/dev/null || echo '')"
REMOTE_COMMIT=''
BEHIND=null
FETCH_ERROR=''

# ── Is there anything newer? ──────────────────────────────────────────────
#
# One question, whatever this installation is checked out as: is there a TAG
# whose version is higher than the one currently deployed?
#
# It used to be two questions. A pinned server compared tags; everything else
# counted commits on its branch — so a console following main offered an update
# for every commit that landed, verified or not, and the button meant "take
# whatever is on the branch right now". Landing on main and being released are
# different events, and only the second one is a decision somebody made.
#
# Compared by VERSION, not by ancestry or by tag date: a tag can be cut from
# anywhere, and what an operator is being offered is a version number. `sort -V`
# does the comparing, so v1.10.0 correctly beats v1.9.0.
DEPLOYED_VERSION="$(release_manifest_version)"
[[ -n "$DEPLOYED_VERSION" ]] || DEPLOYED_VERSION="$(release_version)"

# version_gt A B — true when A is strictly a higher version than B.
version_gt() {
    [[ "$1" != "$2" ]] && [[ "$(printf '%s\n%s\n' "$1" "$2" | sort -V | tail -1)" == "$1" ]]
}

if git fetch --quiet --tags --force origin 2>/dev/null; then
    # Newest by version order, not by tag date.
    NEWEST_TAG="$(git tag --list 'v*' --sort=-v:refname 2>/dev/null | head -1)"
    NEWEST_VERSION="${NEWEST_TAG#v}"

    if [[ -n "$NEWEST_TAG" ]] && version_gt "$NEWEST_VERSION" "$DEPLOYED_VERSION"; then
        # How many releases ahead, so the console can say "2 Aktualisierungen"
        # rather than only that something exists.
        BEHIND="$(git tag --list 'v*' --sort=-v:refname 2>/dev/null | while read -r t; do
            version_gt "${t#v}" "$DEPLOYED_VERSION" && echo "$t"
        done | wc -l | tr -d ' ')"
        TARGET_RELEASE="$NEWEST_TAG"
        REMOTE_COMMIT="$(git rev-parse "refs/tags/${NEWEST_TAG}^{commit}" 2>/dev/null || echo '')"
    else
        # Includes "no tags at all yet": nothing has been released, so there is
        # nothing to offer, and saying "up to date" is the truth.
        BEHIND=0
    fi
else
    # Reported, not swallowed: "cannot reach the repository" and "already up to
    # date" look identical from the panel otherwise, and only one is fine. A
    # code, not a sentence: the panel is translated and this script is not, so a
    # German string here would surface in the English interface.
    FETCH_ERROR='repo_unreachable'
fi

# ── Is an update asked for, and is it still current? ──────────────────────
if [[ ! -f "$REQUEST" ]]; then
    write_status idle "$FETCH_ERROR"
    exit 0
fi

REQUEST_AGE_MINUTES="$(( ( $(date +%s) - $(stat -c %Y "$REQUEST" 2>/dev/null || date +%s) ) / 60 ))"

if (( REQUEST_AGE_MINUTES > REQUEST_EXPIRES_MINUTES )); then
    # Written while this agent was stopped. Running it now is not what was
    # asked for — it was asked for then.
    rm -f "$REQUEST"
    write_status idle "$FETCH_ERROR"
    exit 0
fi

# What was actually asked for: a real run, or only the check the block above
# already performed. A distinct field, not a flag folded into the existing
# request — "check" and "run" are two different things the panel can ask
# for, not one request with a modifier. Read before the file is consumed;
# a request written by an older panel has no such field and is treated as a
# run, which is everything a request has ever meant until now.
REQUEST_KIND="$(sed -n 's/.*"kind"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$REQUEST" 2>/dev/null | head -1)"

# Consumed BEFORE anything below runs, not after: update.sh restarts the
# container stack and may well kill this shell with it. A request left in
# place would be picked up again on the next tick and update in a loop.
rm -f "$REQUEST"

if [[ "$REQUEST_KIND" == "check" ]]; then
    # Nothing further to do: the fetch and the BEHIND calculation above
    # already answered it. The operator asked to look, not to touch
    # anything, so this tick ends exactly like one that found no request —
    # no maintenance mode, no restart, no phase file.
    write_status idle "$FETCH_ERROR"
    exit 0
fi

# Only ever to a tag, and only to one that is genuinely newer. Checked BEFORE
# anything announces a run: a request with nothing to install is answered, not
# obeyed. The console does not offer the button in that state, so this is a
# stale tab or a forged request — and obeying it would take the site into
# maintenance mode to install what is already installed.
if [[ -z "$TARGET_RELEASE" ]]; then
    write_status idle "$FETCH_ERROR"
    exit 0
fi

# The previous run's last step is not this run's first one. Left in place, the
# console would show a stale phase for the seconds before update.sh writes its
# own — and on a run that dies before writing any, for the whole run.
rm -f "$PHASE_FILE"
STARTED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)"

write_status running "$FETCH_ERROR"

set +e
RELEASE="$TARGET_RELEASE" "$ROOT/deploy/update.sh" > "$RUNLOG" 2>&1
RESULT=$?
set -e

FINISHED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
LOCAL_COMMIT="$(git rev-parse HEAD 2>/dev/null || echo '')"

EXIT_CODE="$RESULT"

if [[ $RESULT -eq 0 ]]; then
    BEHIND=0
    TARGET_RELEASE=''
    # Dropped before anything records it: a run that finished has no step it is
    # on, and the last one it passed would read as "still at Rebuilding assets".
    rm -f "$PHASE_FILE"
    write_run succeeded ''
    write_status idle ''
else
    # The phase file stays: it names the step that failed, and the console shows
    # it next to the log so the operator does not have to read the log to find
    # out where it stopped.
    write_run failed 'update_failed'
    write_status idle "$FETCH_ERROR"
fi

exit 0