From d180eb3c90581913af722afb1fa4bbf58b5c23ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lo=C3=AFc=20Gremaud?= Date: Sat, 24 Oct 2026 04:15:31 +0200 Subject: [PATCH] deploy scripts --- deploy/systemd/deploy.sh | 75 +++++++++++++++++--- deploy/systemd/polylan-submitter-worker.conf | 24 +++++++ deploy/systemd/polylan-submitter.service | 16 ++--- 3 files changed, 95 insertions(+), 20 deletions(-) create mode 100644 deploy/systemd/polylan-submitter-worker.conf diff --git a/deploy/systemd/deploy.sh b/deploy/systemd/deploy.sh index 99a50b2..80b0f15 100755 --- a/deploy/systemd/deploy.sh +++ b/deploy/systemd/deploy.sh @@ -1,9 +1,11 @@ #!/usr/bin/env bash # Zero-downtime prod deploy: pull, sync deps, build front, migrate, then -# gracefully reload gunicorn (see polylan-submitter.service). +# gracefully reload gunicorn (see polylan-submitter.service, installed and +# restarted instead when it changed) and restart the RQ worker (see +# polylan-submitter-worker.conf). # -# Run from anywhere as the user owning the checkout (caddy), needs sudo for -# systemctl: +# Run from anywhere as the user owning the checkout, needs sudo for +# systemctl/supervisorctl (no-op when already root): # ./deploy/systemd/deploy.sh # # Migrations run while the old code still serves requests, so keep them @@ -12,13 +14,17 @@ set -euo pipefail SERVICE="${SERVICE:-polylan-submitter}" -# Extra units to restart after the web reload, e.g. RQ workers: -# RESTART_SERVICES="polylan-submitter-worker" ./deploy/systemd/deploy.sh -RESTART_SERVICES="${RESTART_SERVICES:-}" +WORKER="${WORKER:-polylan-submitter-worker}" +SUPERVISOR_CONF_DIR="${SUPERVISOR_CONF_DIR:-/etc/supervisor/conf.d}" +WEB_URL="${WEB_URL:-https://polylan-submitter.legrems.xyz/}" REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +DEPLOY_DIR="$REPO_DIR/deploy/systemd" APP_DIR="$REPO_DIR/polylan_submitter" PYTHON="$REPO_DIR/.venv/bin/python" +# Same as the unit's Environment=, so migrate/collectstatic see the prod settings. +# Relative to $APP_DIR, where manage.py runs. +export SETTINGS_ENV_FILE_LOCATION="${SETTINGS_ENV_FILE_LOCATION:-polylan_submitter/.env}" # gunicorn's master doesn't reload itself on HUP, so a server upgrade needs a # full restart instead. @@ -33,6 +39,33 @@ for pkg in ("gunicorn", "uvicorn", "uvicorn-worker"): EOF } +# wait_for : retry for ~30s, fail the deploy if it +# never succeeds. +wait_for() { + local what="$1" + shift + for _ in $(seq 1 30); do + if "$@" >/dev/null 2>&1; then + echo " $what: ok" + return 0 + fi + sleep 1 + done + echo " $what: FAILED" >&2 + return 1 +} + +web_responds() { + # 000 = nothing listening, 5xx = Caddy up but gunicorn down (502) or broken. + local code + code="$(curl -s -o /dev/null -w '%{http_code}' "$WEB_URL")" + [ "$code" != "000" ] && [ "$code" -lt 500 ] +} + +worker_running() { + sudo supervisorctl status "$WORKER" | grep -q RUNNING +} + cd "$REPO_DIR" echo "==> git pull" @@ -56,17 +89,37 @@ echo "==> migrate" echo "==> collectstatic" "$PYTHON" manage.py collectstatic --noinput -if [ "$before" != "$after" ]; then +echo "==> $SERVICE (systemd)" +unit_changed=false +if ! cmp -s "$DEPLOY_DIR/$SERVICE.service" "/etc/systemd/system/$SERVICE.service"; then + sudo install -m 644 "$DEPLOY_DIR/$SERVICE.service" "/etc/systemd/system/$SERVICE.service" + sudo systemctl daemon-reload + sudo systemctl enable "$SERVICE" + unit_changed=true +fi + +# HUP re-forks workers with the master's original args, so a new ExecStart +# (unit change) or a new gunicorn/uvicorn needs a full restart. +if [ "$unit_changed" = true ]; then + echo "==> unit file changed, full restart of $SERVICE" + sudo systemctl restart "$SERVICE" +elif [ "$before" != "$after" ]; then echo "==> server packages changed, full restart of $SERVICE" sudo systemctl restart "$SERVICE" else echo "==> graceful reload of $SERVICE" sudo systemctl reload "$SERVICE" fi +wait_for "$SERVICE active" systemctl is-active --quiet "$SERVICE" +wait_for "$SERVICE responds on $WEB_URL" web_responds -for unit in $RESTART_SERVICES; do - echo "==> restart $unit" - sudo systemctl restart "$unit" -done +echo "==> $WORKER (supervisor)" +sudo install -m 644 "$DEPLOY_DIR/$WORKER.conf" "$SUPERVISOR_CONF_DIR/$WORKER.conf" +# reread+update picks up config changes (and starts it on first install), +# restart loads the new code - TERM is a warm shutdown, the running job ends first. +sudo supervisorctl reread +sudo supervisorctl update "$WORKER" +sudo supervisorctl restart "$WORKER" +wait_for "$WORKER running" worker_running echo "==> done" diff --git a/deploy/systemd/polylan-submitter-worker.conf b/deploy/systemd/polylan-submitter-worker.conf new file mode 100644 index 0000000..4eb090e --- /dev/null +++ b/deploy/systemd/polylan-submitter-worker.conf @@ -0,0 +1,24 @@ +; Prod RQ worker (submission OCR, AAndT sync, Steam refresh...), run by the +; host's supervisord next to the polylan-submitter systemd unit (web). +; +; Installed to /etc/supervisor/conf.d/ by deploy/systemd/deploy.sh, which also +; restarts it on every deploy so it picks up the new code. +; +; Logs: supervisorctl tail -f polylan-submitter-worker + +[program:polylan-submitter-worker] +command=/home/caddy/polylan-submitter/.venv/bin/python manage.py rqworker default --job-class django_tasks_rq.Job --with-scheduler +directory=/home/caddy/polylan-submitter/polylan_submitter +; Relative to directory, same as the systemd unit. +environment=SETTINGS_ENV_FILE_LOCATION="polylan_submitter/.env",PYTHONUNBUFFERED="1" +user=root +autostart=true +autorestart=true +startsecs=5 +; rqworker does a warm shutdown on TERM: it finishes the current job (OCR can +; take a while) before exiting, only SIGKILLed after stopwaitsecs. +stopsignal=TERM +stopwaitsecs=120 +stopasgroup=true +killasgroup=true +redirect_stderr=true diff --git a/deploy/systemd/polylan-submitter.service b/deploy/systemd/polylan-submitter.service index f6d1e3d..b3b1c6c 100644 --- a/deploy/systemd/polylan-submitter.service +++ b/deploy/systemd/polylan-submitter.service @@ -4,10 +4,8 @@ # keeps the :7777 socket open, boots new workers on the new code, then # gracefully stops the old ones - no downtime. See deploy/systemd/deploy.sh. # -# Install: -# sudo cp deploy/systemd/polylan-submitter.service /etc/systemd/system/ -# sudo systemctl daemon-reload -# sudo systemctl restart polylan-submitter # first switch from daphne needs a full restart +# Installed to /etc/systemd/system/ by deploy.sh: when this file changed it +# does daemon-reload + a full restart (HUP keeps the old ExecStart args). [Unit] Description=PolyLAN submitter (gunicorn + uvicorn workers) @@ -15,10 +13,12 @@ After=network.target [Service] Type=simple -User=caddy -Group=caddy +Restart=always +RestartSec=1 +User=root WorkingDirectory=/home/caddy/polylan-submitter/polylan_submitter -# EnvironmentFile=/home/caddy/polylan-submitter/.env +# Relative to WorkingDirectory, loaded by settings.py via python-dotenv. +Environment="SETTINGS_ENV_FILE_LOCATION=polylan_submitter/.env" # No --preload: with it, HUP re-forks from the already imported (old) code. # --forwarded-allow-ips replaces daphne's --proxy-headers: trust @@ -34,8 +34,6 @@ ExecReload=/bin/kill -s HUP $MAINPID # SIGTERM to the master only, it then drains its workers (graceful-timeout). KillMode=mixed TimeoutStopSec=30 -Restart=on-failure -RestartSec=2 [Install] WantedBy=multi-user.target