deploy scripts

deploy/systemd/deploy.sh

@@ -1,9 +1,11 @@
 #!/usr/bin/env bash
 # Zero-downtime prod deploy: pull, sync deps, build front, migrate, then
-# gracefully reload gunicorn (see polylan-submitter.service).
+# gracefully reload gunicorn (see polylan-submitter.service, installed and
+# restarted instead when it changed) and restart the RQ worker (see
+# polylan-submitter-worker.conf).
 #
-# Run from anywhere as the user owning the checkout (caddy), needs sudo for
-# systemctl:
+# Run from anywhere as the user owning the checkout, needs sudo for
+# systemctl/supervisorctl (no-op when already root):
 #   ./deploy/systemd/deploy.sh
 #
 # Migrations run while the old code still serves requests, so keep them
@@ -12,13 +14,17 @@
 set -euo pipefail
 
 SERVICE="${SERVICE:-polylan-submitter}"
-# Extra units to restart after the web reload, e.g. RQ workers:
-#   RESTART_SERVICES="polylan-submitter-worker" ./deploy/systemd/deploy.sh
-RESTART_SERVICES="${RESTART_SERVICES:-}"
+WORKER="${WORKER:-polylan-submitter-worker}"
+SUPERVISOR_CONF_DIR="${SUPERVISOR_CONF_DIR:-/etc/supervisor/conf.d}"
+WEB_URL="${WEB_URL:-https://polylan-submitter.legrems.xyz/}"
 
 REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+DEPLOY_DIR="$REPO_DIR/deploy/systemd"
 APP_DIR="$REPO_DIR/polylan_submitter"
 PYTHON="$REPO_DIR/.venv/bin/python"
+# Same as the unit's Environment=, so migrate/collectstatic see the prod settings.
+# Relative to $APP_DIR, where manage.py runs.
+export SETTINGS_ENV_FILE_LOCATION="${SETTINGS_ENV_FILE_LOCATION:-polylan_submitter/.env}"
 
 # gunicorn's master doesn't reload itself on HUP, so a server upgrade needs a
 # full restart instead.
@@ -33,6 +39,33 @@ for pkg in ("gunicorn", "uvicorn", "uvicorn-worker"):
 EOF
 }
 
+# wait_for <description> <command...>: retry for ~30s, fail the deploy if it
+# never succeeds.
+wait_for() {
+    local what="$1"
+    shift
+    for _ in $(seq 1 30); do
+        if "$@" >/dev/null 2>&1; then
+            echo "    $what: ok"
+            return 0
+        fi
+        sleep 1
+    done
+    echo "    $what: FAILED" >&2
+    return 1
+}
+
+web_responds() {
+    # 000 = nothing listening, 5xx = Caddy up but gunicorn down (502) or broken.
+    local code
+    code="$(curl -s -o /dev/null -w '%{http_code}' "$WEB_URL")"
+    [ "$code" != "000" ] && [ "$code" -lt 500 ]
+}
+
+worker_running() {
+    sudo supervisorctl status "$WORKER" | grep -q RUNNING
+}
+
 cd "$REPO_DIR"
 
 echo "==> git pull"
@@ -56,17 +89,37 @@ echo "==> migrate"
 echo "==> collectstatic"
 "$PYTHON" manage.py collectstatic --noinput
 
-if [ "$before" != "$after" ]; then
+echo "==> $SERVICE (systemd)"
+unit_changed=false
+if ! cmp -s "$DEPLOY_DIR/$SERVICE.service" "/etc/systemd/system/$SERVICE.service"; then
+    sudo install -m 644 "$DEPLOY_DIR/$SERVICE.service" "/etc/systemd/system/$SERVICE.service"
+    sudo systemctl daemon-reload
+    sudo systemctl enable "$SERVICE"
+    unit_changed=true
+fi
+
+# HUP re-forks workers with the master's original args, so a new ExecStart
+# (unit change) or a new gunicorn/uvicorn needs a full restart.
+if [ "$unit_changed" = true ]; then
+    echo "==> unit file changed, full restart of $SERVICE"
+    sudo systemctl restart "$SERVICE"
+elif [ "$before" != "$after" ]; then
     echo "==> server packages changed, full restart of $SERVICE"
     sudo systemctl restart "$SERVICE"
 else
     echo "==> graceful reload of $SERVICE"
     sudo systemctl reload "$SERVICE"
 fi
+wait_for "$SERVICE active" systemctl is-active --quiet "$SERVICE"
+wait_for "$SERVICE responds on $WEB_URL" web_responds
 
-for unit in $RESTART_SERVICES; do
-    echo "==> restart $unit"
-    sudo systemctl restart "$unit"
-done
+echo "==> $WORKER (supervisor)"
+sudo install -m 644 "$DEPLOY_DIR/$WORKER.conf" "$SUPERVISOR_CONF_DIR/$WORKER.conf"
+# reread+update picks up config changes (and starts it on first install),
+# restart loads the new code - TERM is a warm shutdown, the running job ends first.
+sudo supervisorctl reread
+sudo supervisorctl update "$WORKER"
+sudo supervisorctl restart "$WORKER"
+wait_for "$WORKER running" worker_running
 
 echo "==> done"