deploy scripts

deploy/systemd/deploy.sh

@@ -1,9 +1,11 @@
 #!/usr/bin/env bash
 # Zero-downtime prod deploy: pull, sync deps, build front, migrate, then
-# gracefully reload gunicorn (see polylan-submitter.service).
+# gracefully reload gunicorn (see polylan-submitter.service, installed and
+# restarted instead when it changed) and restart the RQ worker (see
+# polylan-submitter-worker.conf).
 #
-# Run from anywhere as the user owning the checkout (caddy), needs sudo for
+# Run from anywhere as the user owning the checkout, needs sudo for
-# systemctl:
+# systemctl/supervisorctl (no-op when already root):
 #   ./deploy/systemd/deploy.sh
 #
 # Migrations run while the old code still serves requests, so keep them
@@ -12,13 +14,17 @@
 set -euo pipefail
 
 SERVICE="${SERVICE:-polylan-submitter}"
-# Extra units to restart after the web reload, e.g. RQ workers:
+WORKER="${WORKER:-polylan-submitter-worker}"
-#   RESTART_SERVICES="polylan-submitter-worker" ./deploy/systemd/deploy.sh
+SUPERVISOR_CONF_DIR="${SUPERVISOR_CONF_DIR:-/etc/supervisor/conf.d}"
-RESTART_SERVICES="${RESTART_SERVICES:-}"
+WEB_URL="${WEB_URL:-https://polylan-submitter.legrems.xyz/}"
 
 REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+DEPLOY_DIR="$REPO_DIR/deploy/systemd"
 APP_DIR="$REPO_DIR/polylan_submitter"
 PYTHON="$REPO_DIR/.venv/bin/python"
+# Same as the unit's Environment=, so migrate/collectstatic see the prod settings.
+# Relative to $APP_DIR, where manage.py runs.
+export SETTINGS_ENV_FILE_LOCATION="${SETTINGS_ENV_FILE_LOCATION:-polylan_submitter/.env}"
 
 # gunicorn's master doesn't reload itself on HUP, so a server upgrade needs a
 # full restart instead.
@@ -33,6 +39,33 @@ for pkg in ("gunicorn", "uvicorn", "uvicorn-worker"):
 EOF
 }
 
+# wait_for <description> <command...>: retry for ~30s, fail the deploy if it
+# never succeeds.
+wait_for() {
+    local what="$1"
+    shift
+    for _ in $(seq 1 30); do
+        if "$@" >/dev/null 2>&1; then
+            echo "    $what: ok"
+            return 0
+        fi
+        sleep 1
+    done
+    echo "    $what: FAILED" >&2
+    return 1
+}
+
+web_responds() {
+    # 000 = nothing listening, 5xx = Caddy up but gunicorn down (502) or broken.
+    local code
+    code="$(curl -s -o /dev/null -w '%{http_code}' "$WEB_URL")"
+    [ "$code" != "000" ] && [ "$code" -lt 500 ]
+}
+
+worker_running() {
+    sudo supervisorctl status "$WORKER" | grep -q RUNNING
+}
+
 cd "$REPO_DIR"
 
 echo "==> git pull"
@@ -56,17 +89,37 @@ echo "==> migrate"
 echo "==> collectstatic"
 "$PYTHON" manage.py collectstatic --noinput
 
-if [ "$before" != "$after" ]; then
+echo "==> $SERVICE (systemd)"
+unit_changed=false
+if ! cmp -s "$DEPLOY_DIR/$SERVICE.service" "/etc/systemd/system/$SERVICE.service"; then
+    sudo install -m 644 "$DEPLOY_DIR/$SERVICE.service" "/etc/systemd/system/$SERVICE.service"
+    sudo systemctl daemon-reload
+    sudo systemctl enable "$SERVICE"
+    unit_changed=true
+fi
+
+# HUP re-forks workers with the master's original args, so a new ExecStart
+# (unit change) or a new gunicorn/uvicorn needs a full restart.
+if [ "$unit_changed" = true ]; then
+    echo "==> unit file changed, full restart of $SERVICE"
+    sudo systemctl restart "$SERVICE"
+elif [ "$before" != "$after" ]; then
     echo "==> server packages changed, full restart of $SERVICE"
     sudo systemctl restart "$SERVICE"
 else
     echo "==> graceful reload of $SERVICE"
     sudo systemctl reload "$SERVICE"
 fi
+wait_for "$SERVICE active" systemctl is-active --quiet "$SERVICE"
+wait_for "$SERVICE responds on $WEB_URL" web_responds
 
-for unit in $RESTART_SERVICES; do
+echo "==> $WORKER (supervisor)"
-    echo "==> restart $unit"
+sudo install -m 644 "$DEPLOY_DIR/$WORKER.conf" "$SUPERVISOR_CONF_DIR/$WORKER.conf"
-    sudo systemctl restart "$unit"
+# reread+update picks up config changes (and starts it on first install),
-done
+# restart loads the new code - TERM is a warm shutdown, the running job ends first.
+sudo supervisorctl reread
+sudo supervisorctl update "$WORKER"
+sudo supervisorctl restart "$WORKER"
+wait_for "$WORKER running" worker_running
 
 echo "==> done"

deploy/systemd/polylan-submitter-worker.conf

@@ -0,0 +1,24 @@
+; Prod RQ worker (submission OCR, AAndT sync, Steam refresh...), run by the
+; host's supervisord next to the polylan-submitter systemd unit (web).
+;
+; Installed to /etc/supervisor/conf.d/ by deploy/systemd/deploy.sh, which also
+; restarts it on every deploy so it picks up the new code.
+;
+; Logs: supervisorctl tail -f polylan-submitter-worker
+
+[program:polylan-submitter-worker]
+command=/home/caddy/polylan-submitter/.venv/bin/python manage.py rqworker default --job-class django_tasks_rq.Job --with-scheduler
+directory=/home/caddy/polylan-submitter/polylan_submitter
+; Relative to directory, same as the systemd unit.
+environment=SETTINGS_ENV_FILE_LOCATION="polylan_submitter/.env",PYTHONUNBUFFERED="1"
+user=root
+autostart=true
+autorestart=true
+startsecs=5
+; rqworker does a warm shutdown on TERM: it finishes the current job (OCR can
+; take a while) before exiting, only SIGKILLed after stopwaitsecs.
+stopsignal=TERM
+stopwaitsecs=120
+stopasgroup=true
+killasgroup=true
+redirect_stderr=true

deploy/systemd/polylan-submitter.service

@@ -4,10 +4,8 @@
 # keeps the :7777 socket open, boots new workers on the new code, then
 # gracefully stops the old ones - no downtime. See deploy/systemd/deploy.sh.
 #
-# Install:
+# Installed to /etc/systemd/system/ by deploy.sh: when this file changed it
-#   sudo cp deploy/systemd/polylan-submitter.service /etc/systemd/system/
+# does daemon-reload + a full restart (HUP keeps the old ExecStart args).
-#   sudo systemctl daemon-reload
-#   sudo systemctl restart polylan-submitter   # first switch from daphne needs a full restart
 
 [Unit]
 Description=PolyLAN submitter (gunicorn + uvicorn workers)
@@ -15,10 +13,12 @@ After=network.target
 
 [Service]
 Type=simple
-User=caddy
+Restart=always
-Group=caddy
+RestartSec=1
+User=root
 WorkingDirectory=/home/caddy/polylan-submitter/polylan_submitter
-# EnvironmentFile=/home/caddy/polylan-submitter/.env
+# Relative to WorkingDirectory, loaded by settings.py via python-dotenv.
+Environment="SETTINGS_ENV_FILE_LOCATION=polylan_submitter/.env"
 
 # No --preload: with it, HUP re-forks from the already imported (old) code.
 # --forwarded-allow-ips replaces daphne's --proxy-headers: trust
@@ -34,8 +34,6 @@ ExecReload=/bin/kill -s HUP $MAINPID
 # SIGTERM to the master only, it then drains its workers (graceful-timeout).
 KillMode=mixed
 TimeoutStopSec=30
-Restart=on-failure
-RestartSec=2
 
 [Install]
 WantedBy=multi-user.target