deploy scripts
This commit is contained in:
+64
-11
@@ -1,9 +1,11 @@
|
||||
#!/usr/bin/env bash
|
||||
# Zero-downtime prod deploy: pull, sync deps, build front, migrate, then
|
||||
# gracefully reload gunicorn (see polylan-submitter.service).
|
||||
# gracefully reload gunicorn (see polylan-submitter.service, installed and
|
||||
# restarted instead when it changed) and restart the RQ worker (see
|
||||
# polylan-submitter-worker.conf).
|
||||
#
|
||||
# Run from anywhere as the user owning the checkout (caddy), needs sudo for
|
||||
# systemctl:
|
||||
# Run from anywhere as the user owning the checkout, needs sudo for
|
||||
# systemctl/supervisorctl (no-op when already root):
|
||||
# ./deploy/systemd/deploy.sh
|
||||
#
|
||||
# Migrations run while the old code still serves requests, so keep them
|
||||
@@ -12,13 +14,17 @@
|
||||
set -euo pipefail
|
||||
|
||||
SERVICE="${SERVICE:-polylan-submitter}"
|
||||
# Extra units to restart after the web reload, e.g. RQ workers:
|
||||
# RESTART_SERVICES="polylan-submitter-worker" ./deploy/systemd/deploy.sh
|
||||
RESTART_SERVICES="${RESTART_SERVICES:-}"
|
||||
WORKER="${WORKER:-polylan-submitter-worker}"
|
||||
SUPERVISOR_CONF_DIR="${SUPERVISOR_CONF_DIR:-/etc/supervisor/conf.d}"
|
||||
WEB_URL="${WEB_URL:-https://polylan-submitter.legrems.xyz/}"
|
||||
|
||||
REPO_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
DEPLOY_DIR="$REPO_DIR/deploy/systemd"
|
||||
APP_DIR="$REPO_DIR/polylan_submitter"
|
||||
PYTHON="$REPO_DIR/.venv/bin/python"
|
||||
# Same as the unit's Environment=, so migrate/collectstatic see the prod settings.
|
||||
# Relative to $APP_DIR, where manage.py runs.
|
||||
export SETTINGS_ENV_FILE_LOCATION="${SETTINGS_ENV_FILE_LOCATION:-polylan_submitter/.env}"
|
||||
|
||||
# gunicorn's master doesn't reload itself on HUP, so a server upgrade needs a
|
||||
# full restart instead.
|
||||
@@ -33,6 +39,33 @@ for pkg in ("gunicorn", "uvicorn", "uvicorn-worker"):
|
||||
EOF
|
||||
}
|
||||
|
||||
# wait_for <description> <command...>: retry for ~30s, fail the deploy if it
|
||||
# never succeeds.
|
||||
wait_for() {
|
||||
local what="$1"
|
||||
shift
|
||||
for _ in $(seq 1 30); do
|
||||
if "$@" >/dev/null 2>&1; then
|
||||
echo " $what: ok"
|
||||
return 0
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
echo " $what: FAILED" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
web_responds() {
|
||||
# 000 = nothing listening, 5xx = Caddy up but gunicorn down (502) or broken.
|
||||
local code
|
||||
code="$(curl -s -o /dev/null -w '%{http_code}' "$WEB_URL")"
|
||||
[ "$code" != "000" ] && [ "$code" -lt 500 ]
|
||||
}
|
||||
|
||||
worker_running() {
|
||||
sudo supervisorctl status "$WORKER" | grep -q RUNNING
|
||||
}
|
||||
|
||||
cd "$REPO_DIR"
|
||||
|
||||
echo "==> git pull"
|
||||
@@ -56,17 +89,37 @@ echo "==> migrate"
|
||||
echo "==> collectstatic"
|
||||
"$PYTHON" manage.py collectstatic --noinput
|
||||
|
||||
if [ "$before" != "$after" ]; then
|
||||
echo "==> $SERVICE (systemd)"
|
||||
unit_changed=false
|
||||
if ! cmp -s "$DEPLOY_DIR/$SERVICE.service" "/etc/systemd/system/$SERVICE.service"; then
|
||||
sudo install -m 644 "$DEPLOY_DIR/$SERVICE.service" "/etc/systemd/system/$SERVICE.service"
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable "$SERVICE"
|
||||
unit_changed=true
|
||||
fi
|
||||
|
||||
# HUP re-forks workers with the master's original args, so a new ExecStart
|
||||
# (unit change) or a new gunicorn/uvicorn needs a full restart.
|
||||
if [ "$unit_changed" = true ]; then
|
||||
echo "==> unit file changed, full restart of $SERVICE"
|
||||
sudo systemctl restart "$SERVICE"
|
||||
elif [ "$before" != "$after" ]; then
|
||||
echo "==> server packages changed, full restart of $SERVICE"
|
||||
sudo systemctl restart "$SERVICE"
|
||||
else
|
||||
echo "==> graceful reload of $SERVICE"
|
||||
sudo systemctl reload "$SERVICE"
|
||||
fi
|
||||
wait_for "$SERVICE active" systemctl is-active --quiet "$SERVICE"
|
||||
wait_for "$SERVICE responds on $WEB_URL" web_responds
|
||||
|
||||
for unit in $RESTART_SERVICES; do
|
||||
echo "==> restart $unit"
|
||||
sudo systemctl restart "$unit"
|
||||
done
|
||||
echo "==> $WORKER (supervisor)"
|
||||
sudo install -m 644 "$DEPLOY_DIR/$WORKER.conf" "$SUPERVISOR_CONF_DIR/$WORKER.conf"
|
||||
# reread+update picks up config changes (and starts it on first install),
|
||||
# restart loads the new code - TERM is a warm shutdown, the running job ends first.
|
||||
sudo supervisorctl reread
|
||||
sudo supervisorctl update "$WORKER"
|
||||
sudo supervisorctl restart "$WORKER"
|
||||
wait_for "$WORKER running" worker_running
|
||||
|
||||
echo "==> done"
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
; Prod RQ worker (submission OCR, AAndT sync, Steam refresh...), run by the
|
||||
; host's supervisord next to the polylan-submitter systemd unit (web).
|
||||
;
|
||||
; Installed to /etc/supervisor/conf.d/ by deploy/systemd/deploy.sh, which also
|
||||
; restarts it on every deploy so it picks up the new code.
|
||||
;
|
||||
; Logs: supervisorctl tail -f polylan-submitter-worker
|
||||
|
||||
[program:polylan-submitter-worker]
|
||||
command=/home/caddy/polylan-submitter/.venv/bin/python manage.py rqworker default --job-class django_tasks_rq.Job --with-scheduler
|
||||
directory=/home/caddy/polylan-submitter/polylan_submitter
|
||||
; Relative to directory, same as the systemd unit.
|
||||
environment=SETTINGS_ENV_FILE_LOCATION="polylan_submitter/.env",PYTHONUNBUFFERED="1"
|
||||
user=root
|
||||
autostart=true
|
||||
autorestart=true
|
||||
startsecs=5
|
||||
; rqworker does a warm shutdown on TERM: it finishes the current job (OCR can
|
||||
; take a while) before exiting, only SIGKILLed after stopwaitsecs.
|
||||
stopsignal=TERM
|
||||
stopwaitsecs=120
|
||||
stopasgroup=true
|
||||
killasgroup=true
|
||||
redirect_stderr=true
|
||||
@@ -4,10 +4,8 @@
|
||||
# keeps the :7777 socket open, boots new workers on the new code, then
|
||||
# gracefully stops the old ones - no downtime. See deploy/systemd/deploy.sh.
|
||||
#
|
||||
# Install:
|
||||
# sudo cp deploy/systemd/polylan-submitter.service /etc/systemd/system/
|
||||
# sudo systemctl daemon-reload
|
||||
# sudo systemctl restart polylan-submitter # first switch from daphne needs a full restart
|
||||
# Installed to /etc/systemd/system/ by deploy.sh: when this file changed it
|
||||
# does daemon-reload + a full restart (HUP keeps the old ExecStart args).
|
||||
|
||||
[Unit]
|
||||
Description=PolyLAN submitter (gunicorn + uvicorn workers)
|
||||
@@ -15,10 +13,12 @@ After=network.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=caddy
|
||||
Group=caddy
|
||||
Restart=always
|
||||
RestartSec=1
|
||||
User=root
|
||||
WorkingDirectory=/home/caddy/polylan-submitter/polylan_submitter
|
||||
# EnvironmentFile=/home/caddy/polylan-submitter/.env
|
||||
# Relative to WorkingDirectory, loaded by settings.py via python-dotenv.
|
||||
Environment="SETTINGS_ENV_FILE_LOCATION=polylan_submitter/.env"
|
||||
|
||||
# No --preload: with it, HUP re-forks from the already imported (old) code.
|
||||
# --forwarded-allow-ips replaces daphne's --proxy-headers: trust
|
||||
@@ -34,8 +34,6 @@ ExecReload=/bin/kill -s HUP $MAINPID
|
||||
# SIGTERM to the master only, it then drains its workers (graceful-timeout).
|
||||
KillMode=mixed
|
||||
TimeoutStopSec=30
|
||||
Restart=on-failure
|
||||
RestartSec=2
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
||||
Reference in New Issue
Block a user