670 lines
30 KiB
Bash
Executable File
670 lines
30 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# The host side of the panel's update button.
|
|
#
|
|
# The panel cannot run an update: it is www-data inside a container, and the
|
|
# update restarts that container. So it writes a request into the checkout and
|
|
# this agent — running on the host as the service account, on a timer — picks it
|
|
# up, runs deploy/update.sh, and writes back what happened.
|
|
#
|
|
# It also answers the question the panel cannot answer on its own: is there
|
|
# anything to update? That needs a `git fetch`, which needs credentials the
|
|
# application deliberately does not have.
|
|
#
|
|
# Installed by deploy/install-agent.sh (as root, once). Runs every few minutes.
|
|
#
|
|
set -Eeuo pipefail
|
|
|
|
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
cd "$ROOT"
|
|
|
|
# release_mode / release_source, so this agent and update.sh agree on what this
|
|
# installation is following rather than each deciding for itself.
|
|
# shellcheck source=lib/release.sh
|
|
. "$ROOT/deploy/lib/release.sh"
|
|
|
|
STATE_DIR="$ROOT/storage/app/deploy"
|
|
REQUEST="$STATE_DIR/update-request.json"
|
|
STATUS="$STATE_DIR/update-status.json"
|
|
RUNLOG="$STATE_DIR/update-last-run.log"
|
|
# The step update.sh is on, written by update.sh itself. Read back here so a run
|
|
# that failed reports WHERE it failed instead of only that it did.
|
|
PHASE_FILE="$STATE_DIR/update-phase"
|
|
# The outcome of the last RUN, kept apart from the periodic check. Written into
|
|
# one file, the next idle tick a minute later would overwrite a failure with
|
|
# "idle" — and an operator would usually never see that the update failed.
|
|
LASTRUN="$STATE_DIR/update-last-run.json"
|
|
# The outcome of the last worker RESTART — its own file, not folded into
|
|
# LASTRUN: a restart is not a deployment (no fetch, no maintenance mode, no
|
|
# update.sh), and the two must be able to fail independently.
|
|
RESTARTLAST="$STATE_DIR/restart-last-run.json"
|
|
# Where a freshly generated collection key is handed to the panel, once. Written
|
|
# 0600 by this agent, read and deleted by the panel — the same shape as the
|
|
# initial admin password an instance holds until somebody notes it down.
|
|
ARCHIVE_KEY="$STATE_DIR/archive-key.json"
|
|
LOCK="$STATE_DIR/.agent.lock"
|
|
# Ein Lebenszeichen, geschrieben als ERSTES bei jedem Lauf — vor dem Abruf der
|
|
# Gegenstelle, vor allem, was blockieren kann.
|
|
#
|
|
# Bis hierher war "wann hat der Agent zuletzt geschrieben" dasselbe wie "wann
|
|
# hat er zuletzt ERFOLGREICH nachgesehen": beides stand nur in der Statusdatei,
|
|
# und die wird erst nach rund 190 Zeilen geschrieben. Steigt ein Lauf davor aus
|
|
# — weil ein anderer die Sperre haelt —, sieht die Konsole zwanzig Minuten lang
|
|
# gar nichts und schliesst daraus, der Dienst sei tot. Er lief die ganze Zeit;
|
|
# er kam nur nicht an die Arbeit. Ein Betreiber wurde dadurch zweiundachtzig
|
|
# Minuten lang zu install-agent.sh geschickt, das an dieser Lage nichts aendert.
|
|
#
|
|
# Zwei Zustaende, eine Datei: "running" heisst, dieser Lauf hat die Sperre und
|
|
# arbeitet; "blocked" heisst, er ist ausgestiegen, weil jemand anderes sie
|
|
# haelt — mit `since` (seit wann ununterbrochen) und `held_by` (wer).
|
|
ALIVE="$STATE_DIR/agent-alive.json"
|
|
# Die Bitte, eine haengende Sperre zu loesen — und was dabei herauskam.
|
|
#
|
|
# EIGENE Datei, nicht der Postkasten oben, und zwar zwangslaeufig: die Sperre
|
|
# wird ein paar Zeilen weiter unten genommen, LANGE bevor dieses Skript
|
|
# `update-request.json` ueberhaupt ansieht. Ein blockierter Lauf steigt vorher
|
|
# aus. Eine Entsperr-Bitte im Postkasten erreichte den Agenten also genau dann
|
|
# nie, wenn sie gebraucht wird — und zusaetzlich nimmt der Postkasten eine
|
|
# Bitte zur Zeit an, sodass eine dreissig Minuten lang wartende Update-Anfrage
|
|
# ausgerechnet das Entsperren aussperrte.
|
|
UNBLOCK="$STATE_DIR/unblock-request.json"
|
|
UNBLOCK_LAST="$STATE_DIR/unblock-last-run.json"
|
|
# The reverse proxy's console allowlist, generated from the one the owner keeps
|
|
# in the console. Without this the proxy has its own hard-coded list that runs
|
|
# FIRST, so everything added in the console has no effect at all — and when the
|
|
# owner's address changes they are turned away before reaching the page that
|
|
# would have let them fix it.
|
|
ALLOWFILE="${CONSOLE_ALLOW_FILE:-/etc/caddy/clupilot-console-allow.conf}"
|
|
|
|
# A request older than this is abandoned. The panel already refuses to show one
|
|
# as pending; the agent has to agree, or a request written while the agent was
|
|
# stopped fires whenever the agent next starts — which could be days later.
|
|
REQUEST_EXPIRES_MINUTES=30
|
|
|
|
# How often systemd starts this agent — see OnUnitActiveSec in install-agent.sh.
|
|
# Reported so the console can say WHEN a queued update will start rather than
|
|
# "in a few minutes", which is the whole reason an operator sits there wondering
|
|
# whether anything is happening at all.
|
|
CHECK_INTERVAL_MINUTES=1
|
|
|
|
# Can this agent be woken the moment a request lands, or only on the next tick?
|
|
#
|
|
# Installed by install-agent.sh as clupilot-update-agent.path. It is reported
|
|
# rather than assumed because it lives on the host: a server that has not run
|
|
# install-agent.sh since this landed still waits for the timer, and the console
|
|
# has to say so instead of promising an answer it will not get. Reported by the
|
|
# agent for the same reason CHECK_INTERVAL_MINUTES is — the panel cannot see
|
|
# systemd from inside its container.
|
|
REQUEST_WATCH=false
|
|
if systemctl is-active --quiet clupilot-update-agent.path 2>/dev/null; then
|
|
REQUEST_WATCH=true
|
|
fi
|
|
|
|
mkdir -p "$STATE_DIR"
|
|
|
|
# Schreibt das Lebenszeichen. Bewusst ohne json_escape (das steht weiter unten,
|
|
# und hier oben darf noch nichts von unten gebraucht werden): der einzige freie
|
|
# Text ist die Ausgabe von ps, aus der Anfuehrungszeichen, Backslashes und
|
|
# Zeilenumbrueche entfernt werden.
|
|
write_alive() {
|
|
local state="$1" since="${2-}" held_by="${3-}" skips="${4-0}"
|
|
held_by="$(printf '%s' "$held_by" | tr -d '"\\' | tr '\n\r\t' ' ')"
|
|
|
|
cat > "$ALIVE.tmp" 2>/dev/null <<EOF || return 0
|
|
{
|
|
"at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
|
"state": "$state",
|
|
"since": "$since",
|
|
"held_by": "$held_by",
|
|
"skips": ${skips}
|
|
}
|
|
EOF
|
|
mv -f "$ALIVE.tmp" "$ALIVE" 2>/dev/null || true
|
|
}
|
|
|
|
# Was beim letzten Entsperren herauskam. Ohne diese Zeile waere der Knopf auf
|
|
# einem Wirt, dessen root-eigener Helfer den Schritt noch nicht kennt, ein
|
|
# Knopf, der nichts tut und nichts sagt.
|
|
write_unblock() {
|
|
cat > "$UNBLOCK_LAST.tmp" 2>/dev/null <<EOF || return 0
|
|
{
|
|
"state": "$1",
|
|
"finished_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
|
"error": "${2-}"
|
|
}
|
|
EOF
|
|
mv -f "$UNBLOCK_LAST.tmp" "$UNBLOCK_LAST" 2>/dev/null || true
|
|
}
|
|
|
|
# Den root-eigenen Helfer die Sperre loesen lassen.
|
|
#
|
|
# Aus diesem Skript heraus geht es nicht: der Sperrenhalter ist ein Prozess auf
|
|
# dem Wirt, und dieser Agent laeuft unprivilegiert als Dienstbenutzer. Was er
|
|
# darf, steht woertlich in sudoers und ist genau diese eine Befehlszeile.
|
|
release_stuck_lock() {
|
|
local step=/usr/local/sbin/clupilot-host-step have
|
|
|
|
if [[ ! -x "$step" ]]; then
|
|
write_unblock failed unblock_helper_missing
|
|
return 0
|
|
fi
|
|
|
|
# Kennt der Helfer den Schritt schon? Ein Wirt, auf dem install-agent.sh
|
|
# seither nicht mehr lief, hat die aeltere Fassung — und die lehnt
|
|
# `release-update-lock` als unbekannt ab. Das ist kein Fehler des
|
|
# Betreibers, aber er muss es erfahren, statt einen Knopf zu druecken, der
|
|
# still nichts tut.
|
|
have="$("$step" contract 2>/dev/null || true)"
|
|
[[ "$have" =~ ^[0-9]+$ ]] || have=0
|
|
|
|
if (( have < 3 )); then
|
|
write_unblock failed unblock_helper_old
|
|
return 0
|
|
fi
|
|
|
|
if sudo -n "$step" release-update-lock >/dev/null 2>&1; then
|
|
write_unblock ok
|
|
else
|
|
write_unblock failed unblock_failed
|
|
fi
|
|
}
|
|
|
|
# One agent at a time. Two overlapping runs of update.sh fight over the
|
|
# checkout, and the loser leaves it half-updated.
|
|
exec 9>"$LOCK"
|
|
|
|
if ! flock -n 9; then
|
|
# Der uebersprungene Lauf war bis hierher voellig stumm — kein Eintrag im
|
|
# Journal (systemd sieht einen sauberen Lauf), keine Zeile in der
|
|
# Statusdatei, nichts in der Konsole. Genau diese Stille hat einen Ausfall
|
|
# ueber achtzig Minuten unsichtbar gemacht. Jetzt hinterlaesst er, dass er
|
|
# uebersprungen hat, seit wann ununterbrochen, und wer die Sperre haelt.
|
|
#
|
|
# `since` wird aus dem vorigen Lebenszeichen uebernommen, solange die Serie
|
|
# laeuft — sonst stuende dort immer "seit einer Minute", und ein Zustand,
|
|
# der seit einer Stunde klemmt, laese sich von einem gesunden Ueberholen
|
|
# zweier Laeufe nicht unterscheiden.
|
|
#
|
|
# Und mit `skips` wird die Serie GEZAEHLT, nicht nur datiert. Bis hierher
|
|
# meldete die Konsole schon beim ERSTEN uebersprungenen Lauf eine Blockade;
|
|
# gemessen wurde das bei einem Ueberholen von neun Sekunden — Zeitgeber und
|
|
# Waechter laufen beide minuetlich, einer nimmt die Sperre, der andere geht
|
|
# weg. Das ist Betrieb. Der Besitzer hat daraufhin eine Stunde lang eine
|
|
# gesunde Anlage auseinandergenommen.
|
|
#
|
|
# Gezaehlt wird HIER und nicht in der Konsole: aus `since` eine Zahl von
|
|
# Laeufen zu machen hiesse, das Taktintervall zu raten, und das steht in
|
|
# der systemd-Unit auf dem Wirt. Ab wann die Zahl eine Meldung wert ist,
|
|
# entscheidet die Konsole (UpdateChannel::BLOCKED_AFTER_SKIPS).
|
|
HOLDER="$( { fuser "$LOCK" 2>/dev/null || true; } | tr -s ' ' | sed 's/^ *//;s/ *$//' )"
|
|
HOLDER_CMD=''
|
|
if [[ -n "$HOLDER" ]]; then
|
|
HOLDER_CMD="$(ps -o pid=,etime=,args= -p $HOLDER 2>/dev/null | head -2 | tr '\n' ' ' || true)"
|
|
fi
|
|
|
|
PREVIOUS_SINCE="$(sed -n 's/.*"since"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$ALIVE" 2>/dev/null | head -1 || true)"
|
|
PREVIOUS_STATE="$(sed -n 's/.*"state"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$ALIVE" 2>/dev/null | head -1 || true)"
|
|
PREVIOUS_SKIPS="$(sed -n 's/.*"skips"[[:space:]]*:[[:space:]]*\([0-9]*\).*/\1/p' "$ALIVE" 2>/dev/null | head -1 || true)"
|
|
|
|
if [[ "$PREVIOUS_STATE" != "blocked" || -z "$PREVIOUS_SINCE" ]]; then
|
|
# Kein Anschluss an eine laufende Serie: dieser Lauf ist der erste.
|
|
PREVIOUS_SINCE="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
SKIPS=1
|
|
else
|
|
# Ein Lebenszeichen von vor dieser Zaehlung hat kein `skips`. Es bleibt
|
|
# nach einem Update eine Minute lang liegen; solange zaehlt die Serie
|
|
# ab eins weiter, statt eine Zahl zu erfinden.
|
|
[[ "$PREVIOUS_SKIPS" =~ ^[0-9]+$ ]] || PREVIOUS_SKIPS=0
|
|
SKIPS=$(( PREVIOUS_SKIPS + 1 ))
|
|
fi
|
|
|
|
write_alive blocked "$PREVIOUS_SINCE" "${HOLDER_CMD:-$HOLDER}" "$SKIPS"
|
|
|
|
# Und hier — und NUR hier — wird eine Bitte gelesen, die haengende Sperre
|
|
# zu loesen. Dies ist der einzige Zweig, den ein blockierter Lauf erreicht,
|
|
# und die Blockade ist der einzige Zustand, in dem die Bitte einen Sinn
|
|
# hat.
|
|
if [[ -f "$UNBLOCK" ]]; then
|
|
UNBLOCK_AGE_MINUTES="$(( ( $(date +%s) - $(stat -c %Y "$UNBLOCK" 2>/dev/null || date +%s) ) / 60 ))"
|
|
# Verbraucht, bevor irgendetwas passiert. Bliebe sie liegen, zuendete
|
|
# sie bei der naechsten Blockade ungefragt — und das Beenden fremder
|
|
# Prozesse ist nichts, was zweimal aus einem Klick folgen darf.
|
|
rm -f "$UNBLOCK"
|
|
|
|
# Dieselbe Frist wie im Postkasten: was abgelegt wurde, als niemand es
|
|
# abholen konnte, ist keine Anweisung fuer irgendwann.
|
|
if (( UNBLOCK_AGE_MINUTES <= REQUEST_EXPIRES_MINUTES )); then
|
|
release_stuck_lock
|
|
fi
|
|
fi
|
|
|
|
exit 0
|
|
fi
|
|
|
|
write_alive running
|
|
|
|
# Die Sperre ist frei — eine noch liegende Entsperr-Bitte hat sich damit
|
|
# erledigt. Sie liegen zu lassen hiesse, sie bei der naechsten Blockade
|
|
# auszufuehren, die mit ihr nichts zu tun hat.
|
|
rm -f "$UNBLOCK"
|
|
|
|
sync_console_allowlist() {
|
|
[[ -w "$(dirname "$ALLOWFILE")" || -w "$ALLOWFILE" ]] || return 0
|
|
|
|
local generated
|
|
# -u www-data: exec defaults to root, and this one runs every minute. An
|
|
# artisan command that logs anything as root leaves storage/logs owned by
|
|
# root, after which the application cannot append to its own log — and every
|
|
# page that logs answers 500 with nothing written to say why.
|
|
# `timeout -k`, weil dieser Aufruf die Sperre haelt: haengt der Behaelter —
|
|
#
|
|
# Das `-k 10` ist am 4. August 2026 dazugekommen, nachdem ein Lauf mit
|
|
# `timeout 45` nach VIER MINUTEN noch stand. `timeout` schickt nach der
|
|
# Frist ein SIGTERM, und `docker compose exec` stirbt daran nicht, wenn
|
|
# es auf den Docker-Daemon wartet — danach wartet `timeout` selbst
|
|
# unbegrenzt. Eine Frist ohne Nachdruck ist keine Frist, sondern eine
|
|
# Bitte. Mit `-k 10` folgt zehn Sekunden spaeter ein SIGKILL, den
|
|
# niemand ignorieren kann.
|
|
#
|
|
# Das war der Ausfall, den die Konsole als "kommt seit 08:02 nicht an die
|
|
# Arbeit" gemeldet hat: ein einziger haengender Aufruf, der die Sperre
|
|
# stundenlang hielt, waehrend jeder folgende Lauf uebersprungen wurde.
|
|
# gerade neu gestartet, ueberlastet, halb tot —, haengt der Agent mit, und
|
|
# jeder folgende Takt steigt still aus. Lieber diese Runde ohne Allowlist
|
|
# als eine Konsole, die minutenlang nichts mehr von sich hoert.
|
|
generated="$(timeout -k 10 45 docker compose exec -T -u www-data app php artisan clupilot:console-access caddy 2>/dev/null)" || return 0
|
|
# Never write an empty matcher: in Caddy that matches nothing, and the
|
|
# console would be unreachable from everywhere including the shell.
|
|
grep -q '@allowed remote_ip .' <<<"$generated" || return 0
|
|
|
|
local pending="$STATE_DIR/.caddy-reload-pending"
|
|
|
|
if [[ ! -f "$ALLOWFILE" ]] || ! diff -q <(printf '%s\n' "$generated") "$ALLOWFILE" >/dev/null 2>&1; then
|
|
printf '%s\n' "$generated" > "$ALLOWFILE"
|
|
: > "$pending"
|
|
fi
|
|
|
|
# A reload that failed once must be retried. Without the marker the next
|
|
# run sees an unchanged file, does nothing, and the proxy keeps the old
|
|
# list indefinitely — the console and the proxy quietly disagreeing is the
|
|
# exact failure this whole mechanism exists to prevent.
|
|
if [[ -f "$pending" ]]; then
|
|
# Reload, never restart: a restart drops every connection in flight,
|
|
# including the one belonging to whoever just changed the list.
|
|
if systemctl reload caddy >/dev/null 2>&1 || sudo -n systemctl reload caddy >/dev/null 2>&1; then
|
|
rm -f "$pending"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
# The tunnel gateway serves a certificate the PUBLIC Caddy owns and renews. Its
|
|
# tls directive loads the file once at startup, so after a renewal it would go
|
|
# on presenting the old one until something restarted it — and eventually serve
|
|
# an expired certificate to the only people who can reach the console.
|
|
sync_vpn_certificate() {
|
|
local stamp seen="$STATE_DIR/.vpn-cert-stamp" path
|
|
|
|
path="$(sed -n 's/^VPN_CERT_PATH=//p' "$ROOT/.env" 2>/dev/null | tail -1)"
|
|
[[ -n "$path" ]] || return 0
|
|
|
|
# Read from INSIDE the gateway, not from the host. Caddy's storage belongs
|
|
# to its own service account and is deliberately left that way — this agent
|
|
# runs unprivileged and cannot even traverse it, so a host-side check would
|
|
# silently never fire and the gateway would serve an expired certificate.
|
|
stamp="$(docker compose exec -T vpn-gateway stat -c %Y "$path" 2>/dev/null | tr -d '\r\n')"
|
|
[[ -n "$stamp" ]] || return 0
|
|
|
|
if [[ ! -f "$seen" ]] || [[ "$(cat "$seen" 2>/dev/null)" != "$stamp" ]]; then
|
|
# Stamped only on success. Recording a restart that did not happen means
|
|
# the next run considers this certificate handled and never retries —
|
|
# leaving the gateway serving the old one until the NEXT renewal.
|
|
if docker compose --profile vpn restart vpn-gateway >/dev/null 2>&1; then
|
|
printf '%s' "$stamp" > "$seen"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
sync_console_allowlist
|
|
sync_vpn_certificate
|
|
|
|
MODE="$(release_mode)"
|
|
BRANCH="$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo main)"
|
|
TARGET_RELEASE=''
|
|
|
|
json_escape() {
|
|
printf '%s' "${1-}" | sed 's/\\/\\\\/g; s/"/\\"/g; s/\t/\\t/g' | tr -d '\n\r'
|
|
}
|
|
|
|
# The step update.sh last announced, if any. Two tab-separated fields: key, time.
|
|
#
|
|
# Not copied into the status document below: this agent BLOCKS for the whole
|
|
# run, so anything it wrote there would be frozen at the first step. The panel
|
|
# reads the phase file itself, which update.sh keeps current. Here it is only
|
|
# needed once, to record which step a failed run died at.
|
|
read_phase() { cut -f1 "$PHASE_FILE" 2>/dev/null | tr -d '\r\n' || true; }
|
|
|
|
write_status() {
|
|
local state="$1" error="${2-}"
|
|
cat > "$STATUS.tmp" <<EOF
|
|
{
|
|
"state": "$(json_escape "$state")",
|
|
"mode": "$(json_escape "$MODE")",
|
|
"checked_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
|
"check_interval_minutes": ${CHECK_INTERVAL_MINUTES},
|
|
"request_watch": ${REQUEST_WATCH},
|
|
"started_at": "$(json_escape "${STARTED_AT:-}")",
|
|
"finished_at": "$(json_escape "${FINISHED_AT:-}")",
|
|
"local_commit": "$(json_escape "${LOCAL_COMMIT:-}")",
|
|
"remote_commit": "$(json_escape "${REMOTE_COMMIT:-}")",
|
|
"target_release": "$(json_escape "${TARGET_RELEASE:-}")",
|
|
"behind": ${BEHIND:-null},
|
|
"branch": "$(json_escape "$BRANCH")",
|
|
"error": "$(json_escape "$error")"
|
|
}
|
|
EOF
|
|
# Replaced atomically: the panel reads this on every settings render, and
|
|
# half a JSON document is exactly the kind of thing that breaks the page at
|
|
# the moment someone needs it.
|
|
mv -f "$STATUS.tmp" "$STATUS"
|
|
}
|
|
|
|
# The outcome of an actual run. Survives every later idle check.
|
|
write_run() {
|
|
local state="$1" error="${2-}"
|
|
cat > "$LASTRUN.tmp" <<EOF
|
|
{
|
|
"state": "$(json_escape "$state")",
|
|
"started_at": "$(json_escape "${STARTED_AT:-}")",
|
|
"finished_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
|
"phase": "$(json_escape "$(read_phase)")",
|
|
"commit": "$(json_escape "$(git rev-parse HEAD 2>/dev/null || echo '')")",
|
|
"error": "$(json_escape "$error")",
|
|
"exit_code": ${EXIT_CODE:-null}
|
|
}
|
|
EOF
|
|
mv -f "$LASTRUN.tmp" "$LASTRUN"
|
|
}
|
|
|
|
# The outcome of the last worker restart. Survives until the next one.
|
|
write_restart() {
|
|
local state="$1" error="${2-}"
|
|
cat > "$RESTARTLAST.tmp" <<EOF
|
|
{
|
|
"state": "$(json_escape "$state")",
|
|
"finished_at": "$(date -u +%Y-%m-%dT%H:%M:%SZ)",
|
|
"error": "$(json_escape "$error")"
|
|
}
|
|
EOF
|
|
mv -f "$RESTARTLAST.tmp" "$RESTARTLAST"
|
|
}
|
|
|
|
LOCAL_COMMIT="$(git rev-parse HEAD 2>/dev/null || echo '')"
|
|
REMOTE_COMMIT=''
|
|
BEHIND=null
|
|
FETCH_ERROR=''
|
|
|
|
# ── Is there anything newer? ──────────────────────────────────────────────
|
|
#
|
|
# One question, whatever this installation is checked out as: is there a TAG
|
|
# whose version is higher than the one currently deployed?
|
|
#
|
|
# It used to be two questions. A pinned server compared tags; everything else
|
|
# counted commits on its branch — so a console following main offered an update
|
|
# for every commit that landed, verified or not, and the button meant "take
|
|
# whatever is on the branch right now". Landing on main and being released are
|
|
# different events, and only the second one is a decision somebody made.
|
|
#
|
|
# Compared by VERSION, not by ancestry or by tag date: a tag can be cut from
|
|
# anywhere, and what an operator is being offered is a version number. `sort -V`
|
|
# does the comparing, so v1.10.0 correctly beats v1.9.0.
|
|
DEPLOYED_VERSION="$(release_manifest_version)"
|
|
[[ -n "$DEPLOYED_VERSION" ]] || DEPLOYED_VERSION="$(release_version)"
|
|
|
|
# `timeout`, aus demselben Grund wie beim Aufruf in sync_console_allowlist: ein
|
|
# Abruf gegen eine tote Verbindung wartet, bis das Betriebssystem ihn nach
|
|
# vielen Minuten aufgibt — und haelt dabei die Sperre. Zwei Minuten sind
|
|
# grosszuegig fuer einen fetch gegen EINE Gegenstelle; laenger ist kein
|
|
# langsames Netz mehr, sondern eines, das nicht antwortet.
|
|
if timeout -k 10 120 git fetch --quiet --tags --force origin 2>/dev/null; then
|
|
# Newest by version order, not by tag date. Through the helper rather than
|
|
# `| head -1`: head exits after one line, git takes SIGPIPE, and pipefail
|
|
# ends the agent — see release_newest_tag.
|
|
NEWEST_TAG="$(release_newest_tag)"
|
|
NEWEST_VERSION="${NEWEST_TAG#v}"
|
|
|
|
if [[ -n "$NEWEST_TAG" ]] && release_version_gt "$NEWEST_VERSION" "$DEPLOYED_VERSION"; then
|
|
# How many releases ahead, so the console can say "2 Aktualisierungen"
|
|
# rather than only that something exists. Both helpers live in
|
|
# deploy/lib/release.sh so a test can run them — the version that lived
|
|
# here ended the agent on every tick under `set -e`, and no test in a
|
|
# PHP suite can reach a bash pipeline.
|
|
BEHIND="$(release_tags_ahead "$DEPLOYED_VERSION")"
|
|
TARGET_RELEASE="$NEWEST_TAG"
|
|
REMOTE_COMMIT="$(git rev-parse "refs/tags/${NEWEST_TAG}^{commit}" 2>/dev/null || echo '')"
|
|
else
|
|
# Includes "no tags at all yet": nothing has been released, so there is
|
|
# nothing to offer, and saying "up to date" is the truth.
|
|
BEHIND=0
|
|
fi
|
|
else
|
|
# Reported, not swallowed: "cannot reach the repository" and "already up to
|
|
# date" look identical from the panel otherwise, and only one is fine. A
|
|
# code, not a sentence: the panel is translated and this script is not, so a
|
|
# German string here would surface in the English interface.
|
|
FETCH_ERROR='repo_unreachable'
|
|
fi
|
|
|
|
# ── Is an update asked for, and is it still current? ──────────────────────
|
|
if [[ ! -f "$REQUEST" ]]; then
|
|
write_status idle "$FETCH_ERROR"
|
|
exit 0
|
|
fi
|
|
|
|
REQUEST_AGE_MINUTES="$(( ( $(date +%s) - $(stat -c %Y "$REQUEST" 2>/dev/null || date +%s) ) / 60 ))"
|
|
|
|
if (( REQUEST_AGE_MINUTES > REQUEST_EXPIRES_MINUTES )); then
|
|
# Written while this agent was stopped. Running it now is not what was
|
|
# asked for — it was asked for then.
|
|
rm -f "$REQUEST"
|
|
write_status idle "$FETCH_ERROR"
|
|
exit 0
|
|
fi
|
|
|
|
# What was actually asked for: a real run, only the check the block above
|
|
# already performed, or a worker restart. A distinct field, not a flag folded
|
|
# into the existing request — "check", "run" and "restart" are three
|
|
# different things the panel can ask for, not one request with a modifier.
|
|
# Read before the file is consumed; a request written by an older panel has
|
|
# no such field and is treated as a run, which is everything a request has
|
|
# ever meant until now.
|
|
REQUEST_KIND="$(sed -n 's/.*"kind"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$REQUEST" 2>/dev/null | head -1)"
|
|
|
|
# The whole document, kept for the kinds that carry more than a kind. Read here
|
|
# for the same reason the kind is: the file is consumed on the next line, and a
|
|
# field fetched afterwards would be fetched from a file that no longer exists.
|
|
REQUEST_BODY="$(cat "$REQUEST" 2>/dev/null || true)"
|
|
|
|
# Consumed BEFORE anything below runs, not after: update.sh restarts the
|
|
# container stack and may well kill this shell with it. A request left in
|
|
# place would be picked up again on the next tick and update in a loop.
|
|
rm -f "$REQUEST"
|
|
|
|
# ── A collection key for the invoice archive ──────────────────────────────
|
|
# Everything an operator otherwise does by hand across three machines: generate
|
|
# a keypair, install the public half restricted to one directory, hand over the
|
|
# private half. Done here because it is all on the HOST — the panel is www-data
|
|
# in a container and owns none of it.
|
|
#
|
|
# The restriction is rrsync, which ships with rsync: it parses what the client
|
|
# asked for, forces read-only and forces the directory. The obvious alternative,
|
|
# pinning the exact rsync option string in authorized_keys, breaks the moment
|
|
# either side is a different rsync version — and breaks silently, as a refusal
|
|
# with no reason given.
|
|
if [[ "$REQUEST_KIND" == "archive-key" ]]; then
|
|
ARCHIVE_PATH="$(sed -n 's/.*"archive_path"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' <<<"$REQUEST_BODY" | head -1)"
|
|
KEY_NAME="$(sed -n 's/.*"label"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' <<<"$REQUEST_BODY" | head -1)"
|
|
KEY_NAME="${KEY_NAME:-clupilot-archiv}"
|
|
|
|
ARCHIVE_ERROR=''
|
|
|
|
if [[ -z "$ARCHIVE_PATH" || "$ARCHIVE_PATH" != /* ]]; then
|
|
ARCHIVE_ERROR='archive_path_invalid'
|
|
elif ! command -v rrsync >/dev/null 2>&1; then
|
|
# Named, not guessed at. Without rrsync the only way to restrict the key
|
|
# is the brittle option string, and issuing a key that is NOT restricted
|
|
# when the panel says it is would be worse than issuing none.
|
|
ARCHIVE_ERROR='rrsync_missing'
|
|
else
|
|
HOME_DIR="$(getent passwd "$(id -un)" | cut -d: -f6)"
|
|
HOME_DIR="${HOME_DIR:-$HOME}"
|
|
KEY_FILE="$(mktemp "$STATE_DIR/.newkey.XXXXXX")"
|
|
rm -f "$KEY_FILE"
|
|
|
|
if ssh-keygen -q -t ed25519 -N '' -C "$KEY_NAME" -f "$KEY_FILE" 2>/dev/null; then
|
|
mkdir -p "$HOME_DIR/.ssh"
|
|
chmod 700 "$HOME_DIR/.ssh"
|
|
touch "$HOME_DIR/.ssh/authorized_keys"
|
|
chmod 600 "$HOME_DIR/.ssh/authorized_keys"
|
|
|
|
# restrict switches off every forwarding and the pty; the forced
|
|
# command means this key can do one thing and read one directory.
|
|
printf 'command="/usr/bin/rrsync -ro %s",restrict %s\n' \
|
|
"$ARCHIVE_PATH" "$(cat "$KEY_FILE.pub")" >> "$HOME_DIR/.ssh/authorized_keys"
|
|
|
|
umask 077
|
|
printf '{\n "created_at": "%s",\n "label": "%s",\n "path": "%s",\n "private_key": "%s"\n}\n' \
|
|
"$(date -u +%Y-%m-%dT%H:%M:%SZ)" \
|
|
"$(json_escape "$KEY_NAME")" \
|
|
"$(json_escape "$ARCHIVE_PATH")" \
|
|
"$(sed ':a;N;$!ba;s/\n/\\n/g' "$KEY_FILE" | sed 's/"/\\"/g')" \
|
|
> "$ARCHIVE_KEY"
|
|
chmod 600 "$ARCHIVE_KEY"
|
|
|
|
rm -f "$KEY_FILE" "$KEY_FILE.pub"
|
|
else
|
|
ARCHIVE_ERROR='keygen_failed'
|
|
rm -f "$KEY_FILE" "$KEY_FILE.pub"
|
|
fi
|
|
fi
|
|
|
|
write_status idle "$ARCHIVE_ERROR"
|
|
exit 0
|
|
fi
|
|
|
|
if [[ "$REQUEST_KIND" == "check" ]]; then
|
|
# Nothing further to do: the fetch and the BEHIND calculation above
|
|
# already answered it. The operator asked to look, not to touch
|
|
# anything, so this tick ends exactly like one that found no request —
|
|
# no maintenance mode, no restart, no phase file.
|
|
write_status idle "$FETCH_ERROR"
|
|
exit 0
|
|
fi
|
|
|
|
if [[ "$REQUEST_KIND" == "proxy-hosts" ]]; then
|
|
# Die Hostnamen aus der Konsole in die Proxy-Konfiguration übernehmen.
|
|
#
|
|
# Nichts wird aktualisiert: kein Wartungsmodus, kein Neustart, kein
|
|
# Anfassen des Auscheckens. Der Aufruf geht an EINE feste Befehlszeile des
|
|
# root-eigenen Helfers, die in sudoers wörtlich so steht.
|
|
#
|
|
# Der Helfer holt sich die Liste selbst — er bekommt sie nicht von hier
|
|
# gereicht. Wer die Liste liefern dürfte, könnte auch liefern, was sonst
|
|
# noch in die Konfiguration soll, und die Freigabe wäre nichts mehr wert.
|
|
if [[ ! -x /usr/local/sbin/clupilot-host-step ]]; then
|
|
write_status idle "proxy-hosts: clupilot-host-step fehlt — bitte 'sudo bash deploy/install-agent.sh' ausführen"
|
|
exit 0
|
|
fi
|
|
|
|
if PROXY_OUT="$(sudo -n /usr/local/sbin/clupilot-host-step apply-proxy-hosts 2>&1)"; then
|
|
write_status idle ""
|
|
else
|
|
# Die erste Zeile trägt die Aussage; `caddy validate` hängt einen
|
|
# Absatz an, und der passt in keine Statuszeile.
|
|
write_status idle "proxy-hosts: $(printf '%s' "$PROXY_OUT" | head -1)"
|
|
fi
|
|
exit 0
|
|
fi
|
|
|
|
if [[ "$REQUEST_KIND" == "restart" ]]; then
|
|
# queue, queue-provisioning, scheduler, reverb — and ONLY those four.
|
|
# `app` is deliberately never in this list: it is the container serving
|
|
# THIS very request (the panel wrote the request from inside it), so
|
|
# restarting it would kill the request that asked for the restart before
|
|
# the agent could even answer it. It is also unnecessary — PHP-FPM
|
|
# bootstraps a fresh Laravel application on every request it serves, so
|
|
# `app` already picks up whatever the panel just wrote to .env (and
|
|
# cleared from the config cache, which the panel also does itself, see
|
|
# App\Livewire\Admin\Integrations::saveEnv()) without being touched here.
|
|
# Not a deployment: no fetch was needed for this, no maintenance mode, no
|
|
# phase file, no update.sh.
|
|
#
|
|
# queue-provisioning steht mit Absicht NICHT dabei, und das ist hier noch
|
|
# wichtiger als in update.sh: diese Stelle läuft unbeaufsichtigt, ausgelöst
|
|
# von einem Knopf in der Konsole. Sein Container teilt sich den
|
|
# Netz-Namensraum mit dem Tunnel, ein `restart` baut ihn neu auf, und jede
|
|
# WireGuard-Sitzung reißt ab — die des Betreibers am Telefon inbegriffen.
|
|
# Jemand, der „Dienste neu starten" drückt, um eine geänderte Einstellung zu
|
|
# übernehmen, rechnet nicht damit, sich dabei selbst auszusperren.
|
|
#
|
|
# Stattdessen dasselbe Signal wie im Deployment: der Arbeiter beendet sich
|
|
# nach dem laufenden Auftrag, seine Schleife startet ihn mit dem neuen Stand
|
|
# neu (docker/provisioning-worker.sh), der Container bleibt stehen.
|
|
restart_ok=true
|
|
docker compose restart queue scheduler reverb >/dev/null 2>&1 || restart_ok=false
|
|
docker compose exec -T queue-provisioning php artisan queue:restart >/dev/null 2>&1 || true
|
|
|
|
if [[ "$restart_ok" == true ]]; then
|
|
write_restart succeeded
|
|
else
|
|
write_restart failed restart_failed
|
|
fi
|
|
write_status idle "$FETCH_ERROR"
|
|
exit 0
|
|
fi
|
|
|
|
# Only ever to a tag, and only to one that is genuinely newer. Checked BEFORE
|
|
# anything announces a run: a request with nothing to install is answered, not
|
|
# obeyed. The console does not offer the button in that state, so this is a
|
|
# stale tab or a forged request — and obeying it would take the site into
|
|
# maintenance mode to install what is already installed.
|
|
if [[ -z "$TARGET_RELEASE" ]]; then
|
|
write_status idle "$FETCH_ERROR"
|
|
exit 0
|
|
fi
|
|
|
|
# The previous run's last step is not this run's first one. Left in place, the
|
|
# console would show a stale phase for the seconds before update.sh writes its
|
|
# own — and on a run that dies before writing any, for the whole run.
|
|
rm -f "$PHASE_FILE"
|
|
STARTED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
|
|
write_status running "$FETCH_ERROR"
|
|
|
|
set +e
|
|
RELEASE="$TARGET_RELEASE" "$ROOT/deploy/update.sh" > "$RUNLOG" 2>&1
|
|
RESULT=$?
|
|
set -e
|
|
|
|
FINISHED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
LOCAL_COMMIT="$(git rev-parse HEAD 2>/dev/null || echo '')"
|
|
|
|
EXIT_CODE="$RESULT"
|
|
|
|
if [[ $RESULT -eq 0 ]]; then
|
|
BEHIND=0
|
|
TARGET_RELEASE=''
|
|
# Dropped before anything records it: a run that finished has no step it is
|
|
# on, and the last one it passed would read as "still at Rebuilding assets".
|
|
rm -f "$PHASE_FILE"
|
|
write_run succeeded ''
|
|
write_status idle ''
|
|
else
|
|
# The phase file stays: it names the step that failed, and the console shows
|
|
# it next to the log so the operator does not have to read the log to find
|
|
# out where it stopped.
|
|
write_run failed 'update_failed'
|
|
write_status idle "$FETCH_ERROR"
|
|
fi
|
|
|
|
exit 0
|