CluPilotCloud/deploy/update.sh

224 lines
9.0 KiB
Bash
Executable File

#!/usr/bin/env bash
#
# CluPilot — pull the latest code and apply it.
#
# The checkout is bind-mounted into every container, so a `git merge` swaps the
# running code instantly. There is no atomic release directory to switch to;
# what there is, is maintenance mode. So the order is:
#
# down → merge → dependencies → migrate → assets → restart → up
#
# That trades a short, deliberate outage for never serving code whose schema
# does not exist yet. If any step fails the site STAYS down: coming back up with
# new code on an old schema is worse than staying dark until someone looks.
set -euo pipefail
cd "$(dirname "$0")/.."
# shellcheck source=deploy/lib/release.sh
. deploy/lib/release.sh
BRANCH="${BRANCH:-main}"
# RELEASE=v1.2.0 moves a pinned server to that tag. Without it, the mode the
# server was installed in is kept — a production box pinned to a release does
# not quietly start following main because someone re-ran the updater.
RELEASE="${RELEASE:-}"
# Never as root: the checkout belongs to the service account, and running this
# as root would leave root-owned files behind that the app cannot write.
if [[ $EUID -eq 0 ]]; then
echo "Run this as the service account, not as root:" >&2
echo " sudo -u clupilot bash $0" >&2
exit 1
fi
STATE_FILE="storage/app/deployed-commit"
log() { printf '\n\033[1;34m==>\033[0m %s\n' "$*"; }
warn() { printf '\033[1;33m !\033[0m %s\n' "$*"; }
in_app() { docker compose exec -T app "$@"; }
down=0
finish() {
local code=$?
if [[ $code -ne 0 && $down -eq 1 ]]; then
warn "Update failed. The site is STILL in maintenance mode on purpose."
warn "Fix the problem and re-run this script, or force it back up with:"
warn " docker compose exec app php artisan up"
fi
exit $code
}
trap finish EXIT
# Which line this server follows, and what it is being moved to.
mode="$(release_mode)"
[[ -n "$RELEASE" ]] && mode="release"
if [[ "$mode" == "release" ]]; then
# A pinned server without an explicit RELEASE has nothing to do: staying on
# the tag is the whole point of being pinned.
if [[ -z "$RELEASE" ]]; then
RELEASE="$(release_source)"
RELEASE="${RELEASE#refs/tags/}"
[[ -n "$RELEASE" ]] || { echo "Pinned to a release, but no tag recorded. Pass RELEASE=vX.Y.Z." >&2; exit 1; }
fi
log "Fetching release $RELEASE"
git fetch --quiet --tags --force origin
git rev-parse -q --verify "refs/tags/${RELEASE}^{commit}" >/dev/null \
|| { echo "No such release tag: ${RELEASE}" >&2; exit 1; }
target="$(git rev-parse "refs/tags/${RELEASE}^{commit}")"
source_ref="refs/tags/${RELEASE}"
else
log "Fetching $BRANCH"
git fetch --quiet origin "$BRANCH"
target="$(git rev-parse "origin/$BRANCH")"
source_ref="$BRANCH"
fi
before="$(git rev-parse HEAD)"
# Backwards is not an update. The database has already been migrated forward,
# and older code against a newer schema is the one failure this script exists to
# prevent — with the added cruelty that the migrations needed to roll back are
# not in the older checkout at all. A genuine downgrade means restoring the
# pre-upgrade database snapshot first, deliberately, by hand.
if release_would_go_backwards "$before" "$target"; then
cat >&2 <<BACKWARDS
Refusing to move backwards: ${source_ref} is behind what is deployed.
deployed $(git rev-parse --short "$before")
requested $(git rev-parse --short "$target")
The schema has already moved forward, and older code against a newer schema is
what maintenance mode exists to avoid. To go back: restore the database
snapshot taken before the upgrade, then install the older release.
BACKWARDS
exit 1
fi
# What was last deployed successfully — not what is merely checked out. A run
# that died halfway leaves these different, and the next run finishes the job
# instead of declaring "already up to date".
deployed="$(cat "$STATE_FILE" 2>/dev/null || echo '')"
if [[ "$before" == "$target" && "$deployed" == "$target" ]]; then
# Nothing to deploy, but possibly something to decide: pinning a server to
# a release that happens to be the commit it is already on is a real and
# sensible thing to do. Taking this exit without recording it would leave
# the machine looking pinned while the next plain update quietly walks it
# back onto the branch.
# The manifest is checked too, not just the mode: if writing it failed at
# the end of the last run — a full disk is enough — nothing else would ever
# repair it, and the console would report the previous release forever
# while every later update declared itself already done.
if [[ "$(release_mode)" != "$mode" || "$(release_source)" != "$source_ref" \
|| "$(release_manifest_commit)" != "$before" ]]; then
log "Already on this commit — recording it as $mode ($source_ref)"
# And actually pin it. A checkout still attached to the branch is not
# pinned, whatever the manifest claims: the next thing that touches git
# moves it, and the machine drifts off the release it is supposed to be
# nailed to.
if [[ "$mode" == "release" ]]; then
git checkout --quiet --detach "$target"
fi
release_remember "$mode" "$source_ref"
release_write_manifest "$before" "$source_ref" "$mode"
else
log "Already up to date ($(git rev-parse --short HEAD), ${source_ref})"
fi
exit 0
fi
# What the change checks below compare against. After a failed run the checkout
# is already at the target, so diffing against it would compare a commit with
# itself and skip the very steps that did not finish — the base has to be the
# last commit we actually deployed.
base="$before"
if [[ -n "$deployed" ]] && git cat-file -e "${deployed}^{commit}" 2>/dev/null; then
base="$deployed"
fi
if [[ "$before" != "$target" ]]; then
log "Updating $(git rev-parse --short "$before")$(git rev-parse --short "$target")"
git --no-pager log --oneline "$before..$target" | sed 's/^/ /'
else
warn "Code is current but the last update did not finish — repeating the steps."
fi
log "Enabling maintenance mode"
in_app php artisan down --retry=60 >/dev/null || warn "Could not enable maintenance mode (continuing)"
down=1
if [[ "$mode" == "release" ]]; then
# Detached on purpose: a release is a fixed point, not a line to follow.
# `git merge --ff-only` would be meaningless here, and on a detached HEAD it
# is how a pinned server silently rejoins main.
log "Checking out $source_ref"
git checkout --quiet --detach "$target"
else
log "Checking out $BRANCH"
git merge --quiet --ff-only "origin/$BRANCH"
fi
after="$(git rev-parse HEAD)"
# The image is only rebuilt when its definition changed — minutes versus seconds.
if ! git diff --quiet "$base" "$after" -- docker/ 2>/dev/null; then
log "Rebuilding the image"
docker compose build --quiet app
# Recreate now, not at the end: everything below runs INSIDE this container,
# and an update that changes the PHP runtime would otherwise install and
# migrate under the old one.
docker compose up -d --force-recreate app
for _ in $(seq 1 30); do
in_app php -v >/dev/null 2>&1 && break
sleep 2
done
fi
# vendor/ and node_modules/ live in the bind mount, so they shadow whatever the
# image contains: rebuilding the image does NOT update them. Install explicitly
# whenever a lockfile moved, or the migration below runs against stale packages.
if ! git diff --quiet "$base" "$after" -- composer.json composer.lock 2>/dev/null || [[ ! -d vendor ]]; then
log "Installing PHP dependencies"
in_app composer install --no-interaction --no-dev --prefer-dist --no-progress --optimize-autoloader
fi
if ! git diff --quiet "$base" "$after" -- package.json package-lock.json 2>/dev/null || [[ ! -d node_modules ]]; then
log "Installing JS dependencies"
in_app npm ci --no-fund --no-audit
fi
log "Applying migrations"
in_app php artisan migrate --force
log "Rebuilding assets"
in_app npm run build
# Before the restarts, not after: a service that starts while the old cache is
# still on disk loads it and keeps those values for the life of its process.
log "Rebuilding caches"
# optimize:clear first, then optimize: a half-warm cache from before the update
# is what produces a page styled with assets that no longer exist.
in_app php artisan optimize:clear >/dev/null
in_app php artisan optimize >/dev/null
log "Restarting services"
docker compose up -d
# Workers hold their PHP classes for the life of the process; without this they
# keep running the code from before the update.
docker compose restart queue queue-provisioning scheduler reverb
log "Leaving maintenance mode"
in_app php artisan up >/dev/null
down=0
printf '%s' "$after" > "$STATE_FILE"
# Only now: the manifest is what the console reports, and it must mean "this
# came up". Written after `artisan up`, atomically, so a reader never catches it
# half-finished.
release_remember "$mode" "$source_ref"
release_write_manifest "$after" "$source_ref" "$mode"
log "Done — $(release_version) on $(git rev-parse --short HEAD) (${source_ref})"