291 lines
12 KiB
Bash
Executable File
291 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# The host side of the panel's update button.
|
|
#
|
|
# The panel cannot run an update: it is www-data inside a container, and the
|
|
# update restarts that container. So it writes a request into the checkout and
|
|
# this agent — running on the host as the service account, on a timer — picks it
|
|
# up, runs deploy/update.sh, and writes back what happened.
|
|
#
|
|
# It also answers the question the panel cannot answer on its own: is there
|
|
# anything to update? That needs a `git fetch`, which needs credentials the
|
|
# application deliberately does not have.
|
|
#
|
|
# Installed by deploy/install-agent.sh (as root, once). Runs every few minutes.
|
|
#
|
|
set -Eeuo pipefail
|
|
|
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
cd "$ROOT"
|
|
|
|
# release_mode / release_source, so this agent and update.sh agree on what this
|
|
# installation is following rather than each deciding for itself.
|
|
# shellcheck source=lib/release.sh
|
|
. "$ROOT/deploy/lib/release.sh"
|
|
|
|
STATE_DIR="$ROOT/storage/app/deploy"
|
|
REQUEST="$STATE_DIR/update-request.json"
|
|
STATUS="$STATE_DIR/update-status.json"
|
|
RUNLOG="$STATE_DIR/update-last-run.log"
|
|
# The step update.sh is on, written by update.sh itself. Read back here so a run
|
|
# that failed reports WHERE it failed instead of only that it did.
|
|
PHASE_FILE="$STATE_DIR/update-phase"
|
|
# The outcome of the last RUN, kept apart from the periodic check. Written into
|
|
# one file, the next idle tick five minutes later would overwrite a failure with
|
|
# "idle" — and an operator would usually never see that the update failed.
|
|
LASTRUN="$STATE_DIR/update-last-run.json"
|
|
LOCK="$STATE_DIR/.agent.lock"
|
|
# The reverse proxy's console allowlist, generated from the one the owner keeps
|
|
# in the console. Without this the proxy has its own hard-coded list that runs
|
|
# FIRST, so everything added in the console has no effect at all — and when the
|
|
# owner's address changes they are turned away before reaching the page that
|
|
# would have let them fix it.
|
|
ALLOWFILE="${CONSOLE_ALLOW_FILE:-/etc/caddy/clupilot-console-allow.conf}"
|
|
|
|
# A request older than this is abandoned. The panel already refuses to show one
|
|
# as pending; the agent has to agree, or a request written while the agent was
|
|
# stopped fires whenever the agent next starts — which could be days later.
|
|
REQUEST_EXPIRES_MINUTES=30
|
|
|
|
# How often systemd starts this agent — see OnUnitActiveSec in install-agent.sh.
|
|
# Reported so the console can say WHEN a queued update will start rather than
|
|
# "in a few minutes", which is the whole reason an operator sits there wondering
|
|
# whether anything is happening at all.
|
|
CHECK_INTERVAL_MINUTES=5
|
|
|
|
mkdir -p "$STATE_DIR"
|
|
|
|
# One agent at a time. Two overlapping runs of update.sh fight over the
|
|
# checkout, and the loser leaves it half-updated.
|
|
exec 9>"$LOCK"
|
|
flock -n 9 || exit 0
|
|
|
|
sync_console_allowlist() {
|
|
[[ -w "$(dirname "$ALLOWFILE")" || -w "$ALLOWFILE" ]] || return 0
|
|
|
|
local generated
|
|
generated="$(docker compose exec -T app php artisan clupilot:console-access caddy 2>/dev/null)" || return 0
|
|
# Never write an empty matcher: in Caddy that matches nothing, and the
|
|
# console would be unreachable from everywhere including the shell.
|
|
grep -q '@allowed remote_ip .' <<<"$generated" || return 0
|
|
|
|
local pending="$STATE_DIR/.caddy-reload-pending"
|
|
|
|
if [[ ! -f "$ALLOWFILE" ]] || ! diff -q <(printf '%s\n' "$generated") "$ALLOWFILE" >/dev/null 2>&1; then
|
|
printf '%s\n' "$generated" > "$ALLOWFILE"
|
|
: > "$pending"
|
|
fi
|
|
|
|
# A reload that failed once must be retried. Without the marker the next
|
|
# run sees an unchanged file, does nothing, and the proxy keeps the old
|
|
# list indefinitely — the console and the proxy quietly disagreeing is the
|
|
# exact failure this whole mechanism exists to prevent.
|
|
if [[ -f "$pending" ]]; then
|
|
# Reload, never restart: a restart drops every connection in flight,
|
|
# including the one belonging to whoever just changed the list.
|
|
if systemctl reload caddy >/dev/null 2>&1 || sudo -n systemctl reload caddy >/dev/null 2>&1; then
|
|
rm -f "$pending"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
# The tunnel gateway serves a certificate the PUBLIC Caddy owns and renews. Its
|
|
# tls directive loads the file once at startup, so after a renewal it would go
|
|
# on presenting the old one until something restarted it — and eventually serve
|
|
# an expired certificate to the only people who can reach the console.
|
|
sync_vpn_certificate() {
|
|
local stamp seen="$STATE_DIR/.vpn-cert-stamp" path
|
|
|
|
path="$(sed -n 's/^VPN_CERT_PATH=//p' "$ROOT/.env" 2>/dev/null | tail -1)"
|
|
[[ -n "$path" ]] || return 0
|
|
|
|
# Read from INSIDE the gateway, not from the host. Caddy's storage belongs
|
|
# to its own service account and is deliberately left that way — this agent
|
|
# runs unprivileged and cannot even traverse it, so a host-side check would
|
|
# silently never fire and the gateway would serve an expired certificate.
|
|
stamp="$(docker compose exec -T vpn-gateway stat -c %Y "$path" 2>/dev/null | tr -d '\r\n')"
|
|
[[ -n "$stamp" ]] || return 0
|
|
|
|
if [[ ! -f "$seen" ]] || [[ "$(cat "$seen" 2>/dev/null)" != "$stamp" ]]; then
|
|
# Stamped only on success. Recording a restart that did not happen means
|
|
# the next run considers this certificate handled and never retries —
|
|
# leaving the gateway serving the old one until the NEXT renewal.
|
|
if docker compose --profile vpn restart vpn-gateway >/dev/null 2>&1; then
|
|
printf '%s' "$stamp" > "$seen"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
sync_console_allowlist
|
|
sync_vpn_certificate
|
|
|
|
MODE="$(release_mode)"
|
|
BRANCH="$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo main)"
|
|
TARGET_RELEASE=''
|
|
|
|
json_escape() {
|
|
printf '%s' "${1-}" | sed 's/\\/\\\\/g; s/"/\\"/g; s/\t/\\t/g' | tr -d '\n\r'
|
|
}
|
|
|
|
# The step update.sh last announced, if any. Two tab-separated fields: key, time.
|
|
#
|
|
# Not copied into the status document below: this agent BLOCKS for the whole
|
|
# run, so anything it wrote there would be frozen at the first step. The panel
|
|
# reads the phase file itself, which update.sh keeps current. Here it is only
|
|
# needed once, to record which step a failed run died at.
|
|
read_phase() { cut -f1 "$PHASE_FILE" 2>/dev/null | tr -d '\r\n' || true; }
|
|
|
|
write_status() {
|
|
local state="$1" error="${2-}"
|
|
cat > "$STATUS.tmp" <<EOF
|
|
{
|
|
"state": "$(json_escape "$state")",
|
|
"mode": "$(json_escape "$MODE")",
|
|
"checked_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
|
"check_interval_minutes": ${CHECK_INTERVAL_MINUTES},
|
|
"started_at": "$(json_escape "${STARTED_AT:-}")",
|
|
"finished_at": "$(json_escape "${FINISHED_AT:-}")",
|
|
"local_commit": "$(json_escape "${LOCAL_COMMIT:-}")",
|
|
"remote_commit": "$(json_escape "${REMOTE_COMMIT:-}")",
|
|
"target_release": "$(json_escape "${TARGET_RELEASE:-}")",
|
|
"behind": ${BEHIND:-null},
|
|
"branch": "$(json_escape "$BRANCH")",
|
|
"error": "$(json_escape "$error")"
|
|
}
|
|
EOF
|
|
# Replaced atomically: the panel reads this on every settings render, and
|
|
# half a JSON document is exactly the kind of thing that breaks the page at
|
|
# the moment someone needs it.
|
|
mv -f "$STATUS.tmp" "$STATUS"
|
|
}
|
|
|
|
# The outcome of an actual run. Survives every later idle check.
|
|
write_run() {
|
|
local state="$1" error="${2-}"
|
|
cat > "$LASTRUN.tmp" <<EOF
|
|
{
|
|
"state": "$(json_escape "$state")",
|
|
"started_at": "$(json_escape "${STARTED_AT:-}")",
|
|
"finished_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
|
"phase": "$(json_escape "$(read_phase)")",
|
|
"commit": "$(json_escape "$(git rev-parse HEAD 2>/dev/null || echo '')")",
|
|
"error": "$(json_escape "$error")",
|
|
"exit_code": ${EXIT_CODE:-null}
|
|
}
|
|
EOF
|
|
mv -f "$LASTRUN.tmp" "$LASTRUN"
|
|
}
|
|
|
|
LOCAL_COMMIT="$(git rev-parse HEAD 2>/dev/null || echo '')"
|
|
REMOTE_COMMIT=''
|
|
BEHIND=null
|
|
FETCH_ERROR=''
|
|
|
|
# ── Is there anything newer? ──────────────────────────────────────────────
|
|
# Two different questions depending on what this installation follows. A pinned
|
|
# server does not care what landed on main — it moves between TAGS, and asking
|
|
# the branch would advertise updates that pressing the button cannot apply.
|
|
if [[ "$MODE" == "release" ]]; then
|
|
CURRENT_TAG="$(release_source)"; CURRENT_TAG="${CURRENT_TAG#refs/tags/}"
|
|
|
|
if git fetch --quiet --tags --force origin 2>/dev/null; then
|
|
# Newest by version order, not by tag date: v1.10.0 is newer than v1.9.0
|
|
# and sorts earlier alphabetically.
|
|
NEWEST_TAG="$(git tag --list 'v*' --sort=-v:refname 2>/dev/null | head -1)"
|
|
|
|
if [[ -n "$NEWEST_TAG" && "$NEWEST_TAG" != "$CURRENT_TAG" ]]; then
|
|
# Count only tags actually ahead of the deployed one.
|
|
BEHIND="$(git tag --list 'v*' --sort=-v:refname 2>/dev/null \
|
|
| awk -v cur="$CURRENT_TAG" '$0 == cur {exit} {n++} END {print n+0}')"
|
|
[[ "${BEHIND:-0}" -gt 0 ]] && TARGET_RELEASE="$NEWEST_TAG"
|
|
REMOTE_COMMIT="$(git rev-parse "refs/tags/${NEWEST_TAG}^{commit}" 2>/dev/null || echo '')"
|
|
else
|
|
BEHIND=0
|
|
fi
|
|
else
|
|
# A code, not a sentence: the panel is translated and this script is
|
|
# not, so a German string here would surface in the English interface.
|
|
FETCH_ERROR='repo_unreachable'
|
|
fi
|
|
else
|
|
if [[ "$BRANCH" == "HEAD" ]]; then
|
|
# Detached, but not pinned to a release either — nothing sane to follow.
|
|
FETCH_ERROR='detached_no_release'
|
|
elif git fetch --quiet origin "$BRANCH" 2>/dev/null; then
|
|
REMOTE_COMMIT="$(git rev-parse "origin/$BRANCH" 2>/dev/null || echo '')"
|
|
if [[ -n "$LOCAL_COMMIT" && -n "$REMOTE_COMMIT" ]]; then
|
|
BEHIND="$(git rev-list --count "HEAD..origin/$BRANCH" 2>/dev/null || echo 0)"
|
|
fi
|
|
else
|
|
# Reported, not swallowed: "cannot reach the repository" and "already up
|
|
# to date" look identical from the panel otherwise, and only one is fine.
|
|
# A code, not a sentence: the panel is translated and this script is
|
|
# not, so a German string here would surface in the English interface.
|
|
FETCH_ERROR='repo_unreachable'
|
|
fi
|
|
fi
|
|
|
|
# ── Is an update asked for, and is it still current? ──────────────────────
|
|
if [[ ! -f "$REQUEST" ]]; then
|
|
write_status idle "$FETCH_ERROR"
|
|
exit 0
|
|
fi
|
|
|
|
REQUEST_AGE_MINUTES="$(( ( $(date +%s) - $(stat -c %Y "$REQUEST" 2>/dev/null || date +%s) ) / 60 ))"
|
|
|
|
if (( REQUEST_AGE_MINUTES > REQUEST_EXPIRES_MINUTES )); then
|
|
# Written while this agent was stopped. Running it now is not what was
|
|
# asked for — it was asked for then.
|
|
rm -f "$REQUEST"
|
|
write_status idle "$FETCH_ERROR"
|
|
exit 0
|
|
fi
|
|
|
|
# Consumed BEFORE the run, not after: update.sh restarts the container stack and
|
|
# may well kill this shell with it. A request left in place would be picked up
|
|
# again on the next tick and update in a loop.
|
|
rm -f "$REQUEST"
|
|
|
|
# The previous run's last step is not this run's first one. Left in place, the
|
|
# console would show a stale phase for the seconds before update.sh writes its
|
|
# own — and on a run that dies before writing any, for the whole run.
|
|
rm -f "$PHASE_FILE"
|
|
STARTED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
|
|
write_status running "$FETCH_ERROR"
|
|
|
|
set +e
|
|
if [[ "$MODE" == "release" && -n "$TARGET_RELEASE" ]]; then
|
|
RELEASE="$TARGET_RELEASE" "$ROOT/deploy/update.sh" > "$RUNLOG" 2>&1
|
|
else
|
|
# BRANCH passed explicitly: update.sh defaults to main, so an installation
|
|
# following anything else would be told about ITS branch and then handed
|
|
# main's commits.
|
|
BRANCH="$BRANCH" "$ROOT/deploy/update.sh" > "$RUNLOG" 2>&1
|
|
fi
|
|
RESULT=$?
|
|
set -e
|
|
|
|
FINISHED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
LOCAL_COMMIT="$(git rev-parse HEAD 2>/dev/null || echo '')"
|
|
|
|
EXIT_CODE="$RESULT"
|
|
|
|
if [[ $RESULT -eq 0 ]]; then
|
|
BEHIND=0
|
|
TARGET_RELEASE=''
|
|
# Dropped before anything records it: a run that finished has no step it is
|
|
# on, and the last one it passed would read as "still at Rebuilding assets".
|
|
rm -f "$PHASE_FILE"
|
|
write_run succeeded ''
|
|
write_status idle ''
|
|
else
|
|
# The phase file stays: it names the step that failed, and the console shows
|
|
# it next to the log so the operator does not have to read the log to find
|
|
# out where it stopped.
|
|
write_run failed 'update_failed'
|
|
write_status idle "$FETCH_ERROR"
|
|
fi
|
|
|
|
exit 0
|