chore: deploy scripts + env

This commit is contained in:
2026-09-17 17:28:06 +02:00
parent 28d2a7ad24
commit 979be91417
8 changed files with 344 additions and 23 deletions
+1
View File
@@ -178,3 +178,4 @@ pyrightconfig.json
tags
media/
settingsLocal.py
deploy/supervisor/polylan-submitter.env
+58
View File
@@ -0,0 +1,58 @@
#!/usr/bin/env bash
# Deploys polylan-submitter from /opt/polylan-submitter: pulls latest master,
# syncs the venv, renders and installs the supervisor config, restarts the
# supervisor programs, installs the (static) HAProxy config, reloads
# HAProxy, then re-adds the backend servers over HAProxy's admin socket -
# a reload always resets HAProxy back to just the one static server in
# deploy/haproxy/haproxy.cfg, so the rest get added fresh every time.
#
# Run on the target host (needs sudo for cp/supervisorctl/systemctl/socat):
# deploy/deploy.sh
set -euo pipefail
APP_DIR=/opt/polylan-submitter
RUN_DIR="$APP_DIR/run"
SUPERVISOR_CONF_DIR=/etc/supervisor/conf.d
HAPROXY_CONF=/etc/haproxy/haproxy.cfg
HAPROXY_ADMIN_SOCK=/run/haproxy/admin.sock
HAPROXY_BACKEND=polylan_submitter_web
# Single source of truth for daphne process count: drives supervisor's
# numprocs, and how many backend servers get re-added to HAProxy below
# (server 00 is already static in haproxy.cfg, so only 01..N-1 are added).
NUM_WEB_WORKERS=5
cd "$APP_DIR"
git pull --ff-only
uv sync --frozen
sudo mkdir -p "$RUN_DIR"
sudo chown polylan:polylan "$RUN_DIR"
render_dir=$(mktemp -d)
trap 'rm -rf "$render_dir"' EXIT
sed "s/__NUM_WEB_WORKERS__/$NUM_WEB_WORKERS/" \
"$APP_DIR/deploy/supervisor/polylan-submitter.conf" > "$render_dir/polylan-submitter.conf"
sudo cp "$render_dir/polylan-submitter.conf" "$SUPERVISOR_CONF_DIR/polylan-submitter.conf"
sudo supervisorctl reread
sudo supervisorctl update
sudo supervisorctl restart polylan-submitter:
sudo mkdir -p "$(dirname "$HAPROXY_ADMIN_SOCK")"
sudo cp "$APP_DIR/deploy/haproxy/haproxy.cfg" "$HAPROXY_CONF"
sudo haproxy -c -f "$HAPROXY_CONF"
sudo systemctl reload haproxy
# The admin socket only processes one command per connection - sending
# several commands into one socat session silently drops everything after
# the first, so each command below gets its own connection.
for ((i = 1; i < NUM_WEB_WORKERS; i++)); do
printf -v socket_num "%02d" "$i"
server="${HAPROXY_BACKEND}/web-${socket_num}"
socket="unix@${RUN_DIR}/web-${socket_num}.sock"
echo "add server ${server} ${socket} check" | sudo socat - "UNIX-CONNECT:${HAPROXY_ADMIN_SOCK}"
echo "enable server ${server}" | sudo socat - "UNIX-CONNECT:${HAPROXY_ADMIN_SOCK}"
done
+47
View File
@@ -0,0 +1,47 @@
# HAProxy config for polylan-submitter. Plain HTTP only - TLS is terminated
# upstream of this box, so nothing here handles certs or :443.
#
# Unlike nginx/Caddy's per-site snippet directories, HAProxy's default
# packaging expects one single /etc/haproxy/haproxy.cfg for the whole
# process, so this file IS the full config - deploy.sh copies it wholesale
# rather than dropping a fragment into an "enabled" dir. If this box also
# runs other things through HAProxy, fold this frontend/backend into your
# existing haproxy.cfg by hand instead of letting deploy.sh clobber it.
#
# This is no longer a template: it's a plain, ready-to-use config with one
# statically defined backend server (web-00, the first daphne socket), so
# the backend is never empty right after a reload - traffic has somewhere to
# go immediately. deploy.sh adds the remaining NUM_WEB_WORKERS-1 servers over
# the admin socket below (`add server` + `enable server`) every time it
# reloads HAProxy, since a reload always resets the running process back to
# exactly what's in this file. Deploy sequence is always: cp this file ->
# reload -> re-add the rest of the servers.
#
# WebSocket (submissions/clock/place) needs no special handling here: modern
# HAProxy in `mode http` detects the 101 Switching Protocols response and then
# just relays bytes both ways for the rest of that connection - that only
# works because keep-alive is the default connection mode below (no
# `option http-server-close` / `httpclose`, either of which would break it).
# Static files are served in-process by WhiteNoise (see MIDDLEWARE in
# settings.py), so nothing else needs to be carved out here either.
global
log /dev/log local0
maxconn 2000
stats socket /run/haproxy/admin.sock mode 660 level admin
defaults
mode http
log global
option httplog
timeout connect 5s
timeout client 60s
timeout server 60s
frontend polylan_submitter_http
bind *:7777
default_backend polylan_submitter_web
backend polylan_submitter_web
balance roundrobin
server web-00 unix@/opt/polylan-submitter/run/web-00.sock check
+66
View File
@@ -0,0 +1,66 @@
# Supervisor config template for polylan-submitter.
#
# This is a template, not a ready-to-use file: __NUM_WEB_WORKERS__ is filled
# in by deploy.sh (the same number it also uses to re-add HAProxy backend
# servers after each reload - see deploy/haproxy/haproxy.cfg), which then
# installs the rendered result into /etc/supervisor/conf.d/ and
# reread/updates supervisor. Run deploy.sh rather than copying this file by
# hand.
#
# Assumes the app is checked out at /opt/polylan-submitter with a venv at
# /opt/polylan-submitter/.venv, and a "polylan" system user owning it
# (adduser --system --group polylan). Adjust to taste.
#
# App secrets/config (DJANGO_SECRET_KEY, POSTGRES_*, AWS_*, etc. - see
# settings.py for the full list, polylan-submitter.env.dist for the file
# template) are NOT listed in the `environment=` lines below - only
# SETTINGS_ENV_FILE_LOCATION is, pointing settings.py at the file so it can
# load_dotenv() it itself. That's the only var that needs to reach the
# process from here at all: it's just a path, not a secret, so unlike the
# ~19 real secrets it's safe to write directly into this tracked file, and
# there's no %(ENV_x)s crash risk since it's a literal string, not a
# pass-through. Rotating a secret just means editing the env file and
# running `supervisorctl restart polylan-submitter:` (which deploy.sh
# already does on every deploy) - dotenv is read fresh on every process
# start, no systemd/supervisord-wide restart involved.
[group:polylan-submitter]
programs=polylan-submitter-web,polylan-submitter-worker
; Daphne has no built-in multi-worker/--workers flag (single Twisted reactor
; per process), so "more workers" means more daphne processes. Each one binds
; its own unix socket under run/; HAProxy load-balances across all of them
; (see deploy/haproxy/haproxy.cfg). They share state fine since channel
; layers, sessions, cache, etc. all already go through Redis/Postgres rather
; than in-process memory. __NUM_WEB_WORKERS__ comes from deploy.sh's
; NUM_WEB_WORKERS - change it there, not here.
[program:polylan-submitter-web]
command=/opt/polylan-submitter/.venv/bin/daphne -u /opt/polylan-submitter/run/web-%(process_num)02d.sock polylan_submitter.asgi:application
directory=/opt/polylan-submitter/polylan_submitter
process_name=%(program_name)s-%(process_num)02d
user=polylan
numprocs=__NUM_WEB_WORKERS__
autostart=true
autorestart=true
stopasgroup=true
killasgroup=true
stopsignal=TERM
stdout_logfile=/var/log/supervisor/polylan-submitter-web-%(process_num)02d.log
stderr_logfile=/var/log/supervisor/polylan-submitter-web-%(process_num)02d.log
environment=DJANGO_SETTINGS_MODULE="polylan_submitter.settings",SETTINGS_ENV_FILE_LOCATION="/opt/polylan-submitter/deploy/supervisor/polylan-submitter.env"
; RQ workers can have multiple `--with-scheduler`, there is a redis-lock
[program:polylan-submitter-worker]
command=/opt/polylan-submitter/.venv/bin/python manage.py rqworker default --job-class django_tasks_rq.Job --with-scheduler
directory=/opt/polylan-submitter/polylan_submitter
process_name=%(program_name)s-%(process_num)02d
user=polylan
numprocs=2
autostart=true
autorestart=true
stopasgroup=true
killasgroup=true
stopsignal=TERM
stdout_logfile=/var/log/supervisor/polylan-submitter-worker-%(process_num)02d.log
stderr_logfile=/var/log/supervisor/polylan-submitter-worker-%(process_num)02d.log
environment=DJANGO_SETTINGS_MODULE="polylan_submitter.settings",SETTINGS_ENV_FILE_LOCATION="/opt/polylan-submitter/deploy/supervisor/polylan-submitter.env"
@@ -0,0 +1,58 @@
# Template for deploy/supervisor/polylan-submitter.env inside the app
# checkout (/opt/polylan-submitter/deploy/supervisor/polylan-submitter.env on
# the target host) - that path is in .gitignore, same treatment as
# settingsLocal.py, so `git pull` never touches or overwrites it once it's
# there.
#
# Copy this file to that path, fill in real values, then:
# sudo chown polylan:polylan /opt/polylan-submitter/deploy/supervisor/polylan-submitter.env
# sudo chmod 600 /opt/polylan-submitter/deploy/supervisor/polylan-submitter.env
# (owned by "polylan", not root: this gets read by settings.py itself, which
# runs as the "polylan" user - see `user=` in polylan-submitter.conf. 600
# still keeps every other user out.)
#
# settings.py loads this via python-dotenv (see SETTINGS_ENV_FILE_LOCATION
# in polylan-submitter.conf's `environment=`, and the load_dotenv() call
# near the top of settings.py), not a shell - there's no $VAR expansion or
# shell-special-character risk here like a bash `source` would have. The one
# thing dotenv DOES expand is ${VAR_NAME} (braced, referencing another var
# already defined above it in this same file) - if a real secret value ever
# happens to literally contain that exact pattern, quote it (single or
# double quotes both suppress dotenv's own expansion and get stripped).
# Plain values like the ones below need no quoting at all.
#
# load_dotenv() is called with override=False, so an already-exported env
# var always wins over what's in this file - it only fills gaps. It's also
# read fresh on every process start (not cached in some long-lived daemon),
# so rotating a secret just means editing this file and running
# `sudo supervisorctl restart polylan-submitter:` - which deploy.sh already
# does on every routine deploy anyway.
#
# See settings.py for what each of these does and its dev-mode default.
DJANGO_SECRET_KEY=
DJANGO_DEBUG=false
DJANGO_ALLOWED_HOSTS=submitter.polylan.ch
POSTGRES_DB=
POSTGRES_USER=
POSTGRES_PASSWORD=
POSTGRES_HOST=
POSTGRES_PORT=
REDIS_HOST=
REDIS_PORT=
PLACE_REDIS_URL=
AWS_ACCESS_KEY_ID=
AWS_SECRET_ACCESS_KEY=
AWS_S3_ENDPOINT_URL=
AWS_STORAGE_BUCKET_NAME=
SENTRY_ENVIRONMENT=production
SENTRY_DSN=
MARKET_ENABLED=true
STEAM_API_KEY=
AANDT_API_BASE_URL=
SUBMITTER_BASE_URL=
+99 -20
View File
@@ -13,23 +13,54 @@ https://docs.djangoproject.com/en/5.2/ref/settings/
from pathlib import Path
import os
from dotenv import load_dotenv
# Build paths inside the project like this: BASE_DIR / 'subdir'.
BASE_DIR = Path(__file__).resolve().parent.parent
# Optional: point this at a KEY=value file (see
# deploy/supervisor/polylan-submitter.env.dist for the template, and
# deploy/supervisor/polylan-submitter.conf for how it's wired up in
# production) to fill in the env vars read throughout this file. override
# =False so an already-exported env var always wins over the file - this
# only fills gaps, same as every os.environ.get() below already treating a
# merely-blank value as "use my own default" rather than "".
if os.environ.get("SETTINGS_ENV_FILE_LOCATION"):
load_dotenv(os.environ["SETTINGS_ENV_FILE_LOCATION"], override=False)
os.environ.setdefault("AWS_REQUEST_CHECKSUM_CALCULATION", "when_required")
os.environ.setdefault("AWS_RESPONSE_CHECKSUM_VALIDATION", "when_required")
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/5.2/howto/deployment/checklist/
#
# All of the settings below that read from os.environ fall back to
# quick-start/dev-friendly defaults when the env var is unset, EXCEPT
# DJANGO_DEBUG, which defaults to False (secure by default per Django's own
# deployment checklist) - export DJANGO_DEBUG=true for local development.
#
# These all use `os.environ.get(X) or default` rather than
# `os.environ.get(X, default)`: an env var that's *defined but blank* (e.g.
# systemd's EnvironmentFile represents "not configured" as `KEY=`, not by
# omitting the line) must fall through to the default too, not become "".
# That distinction matters a lot here - `int(os.environ.get("REDIS_PORT",
# "6379"))` would still crash on an empty string.
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = "django-insecure-m0ivj_3gpbf281jl1$&n5pdo!5le(bp4z31u(1&4s=n#!tpy=n"
SECRET_KEY = (
os.environ.get("DJANGO_SECRET_KEY")
or "django-insecure-m0ivj_3gpbf281jl1$&n5pdo!5le(bp4z31u(1&4s=n#!tpy=n"
)
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
DEBUG = os.environ.get("DJANGO_DEBUG", "false").lower() == "true"
ALLOWED_HOSTS = ["127.0.0.1", "localhost"]
ALLOWED_HOSTS = [
host
for host in (os.environ.get("DJANGO_ALLOWED_HOSTS") or "127.0.0.1,localhost").split(",")
if host
]
# Application definition
@@ -58,6 +89,12 @@ INSTALLED_APPS = [
"place",
]
if DEBUG:
# Was unconditionally added by settingsLocal.py before it was retired in
# favor of env-var-driven settings; kept dev-only since it pulls in
# shell_plus and similar dev tooling that has no place in production.
INSTALLED_APPS += ["django_extensions"]
MIDDLEWARE = [
"django.middleware.security.SecurityMiddleware",
"whitenoise.middleware.WhiteNoiseMiddleware",
@@ -95,13 +132,28 @@ ASGI_APPLICATION = "polylan_submitter.asgi.application"
# Database
# https://docs.djangoproject.com/en/5.2/ref/settings/#databases
#
# Falls back to sqlite (the quick-start default) when POSTGRES_DB isn't set,
# so a fresh checkout with no env configured still works out of the box.
DATABASES = {
if os.environ.get("POSTGRES_DB"):
DATABASES = {
"default": {
"ENGINE": "django.db.backends.postgresql",
"NAME": os.environ["POSTGRES_DB"],
"USER": os.environ.get("POSTGRES_USER") or "postgres",
"PASSWORD": os.environ.get("POSTGRES_PASSWORD", ""),
"HOST": os.environ.get("POSTGRES_HOST") or "localhost",
"PORT": os.environ.get("POSTGRES_PORT") or "5432",
}
}
else:
DATABASES = {
"default": {
"ENGINE": "django.db.backends.sqlite3",
"NAME": BASE_DIR / "db.sqlite3",
}
}
}
# Password validation
@@ -156,15 +208,15 @@ AUTH_USER_MODEL = "accounts.CustomUser"
CAS_SERVER_URL = "https://polylan.ch/cas/"
# Steam API Configuration
STEAM_API_KEY = os.environ.get("STEAM_API_KEY", None) # Set via environment variable
STEAM_API_KEY = os.environ.get("STEAM_API_KEY") or None # Set via environment variable
# Base URL of the legacy PolyLAN site, used to sync AAndT matches/members
# (see submissions.services.aandt_sync)
AANDT_API_BASE_URL = os.environ.get("AANDT_API_BASE_URL", "https://polylan.ch")
AANDT_API_BASE_URL = os.environ.get("AANDT_API_BASE_URL") or "https://polylan.ch"
# Base URL of this site, used to build submission links pushed back to the
# legacy AAndT match messages (see submissions.services.aandt_sync)
SUBMITTER_BASE_URL = os.environ.get("SUBMITTER_BASE_URL", "http://localhost:7777")
SUBMITTER_BASE_URL = os.environ.get("SUBMITTER_BASE_URL") or "http://localhost:7777"
# File Upload Settings
MEDIA_URL = "/media/"
@@ -174,11 +226,16 @@ MEDIA_ROOT = BASE_DIR / "media"
FILE_UPLOAD_MAX_MEMORY_SIZE = 256 * 1024 * 1024 # 256MB
DATA_UPLOAD_MAX_MEMORY_SIZE = 256 * 1024 * 1024 # 256MB
# Redis host/port shared by the cache, channel layer, RQ, and place's canvas
# below - each still uses its own DB number, just on a common host.
REDIS_HOST = os.environ.get("REDIS_HOST") or "127.0.0.1"
REDIS_PORT = int(os.environ.get("REDIS_PORT") or "6379")
# Caching Configuration
CACHES = {
"default": {
"BACKEND": "django_redis.cache.RedisCache",
"LOCATION": "redis://127.0.0.1:6379/11",
"LOCATION": f"redis://{REDIS_HOST}:{REDIS_PORT}/11",
"TIMEOUT": 300, # 5 minutes default
"OPTIONS": {
"CLIENT_CLASS": "django_redis.client.DefaultClient",
@@ -191,7 +248,7 @@ CHANNEL_LAYERS = {
"default": {
"BACKEND": "channels_redis.core.RedisChannelLayer",
"CONFIG": {
"hosts": [("127.0.0.1", 6379)],
"hosts": [(REDIS_HOST, REDIS_PORT)],
"serializer_format": "cjson",
},
},
@@ -199,7 +256,7 @@ CHANNEL_LAYERS = {
# Redis for place's live pixel canvas (SETRANGE on the hot path - not a fit
# for django-redis's cache API, so it gets its own raw redis-py client).
PLACE_REDIS_URL = os.environ.get("PLACE_REDIS_URL", "redis://127.0.0.1:6379/15")
PLACE_REDIS_URL = os.environ.get("PLACE_REDIS_URL") or f"redis://{REDIS_HOST}:{REDIS_PORT}/15"
# Django Tasks Configuration (with RQ backend)
TASKS = {
@@ -210,8 +267,8 @@ TASKS = {
RQ_QUEUES = {
"default": {
"HOST": "127.0.0.1",
"PORT": 6379,
"HOST": REDIS_HOST,
"PORT": REDIS_PORT,
"DB": 12,
"DEFAULT_RESULT_TTL": 500,
}
@@ -228,7 +285,7 @@ ALLOWED_SUBMISSION_TYPES = [
]
# Market app settings
MARKET_ENABLED = os.environ.get("MARKET_ENABLED", "true").lower() == "true"
MARKET_ENABLED = (os.environ.get("MARKET_ENABLED") or "true").lower() == "true"
# Authentication backends
AUTHENTICATION_BACKENDS = [
@@ -243,7 +300,7 @@ LOGIN_REDIRECT_URL = "/"
LOGOUT_REDIRECT_URL = "/"
DJANGO_VITE = {
"default": {
"dev_mode": False,
"dev_mode": DEBUG,
}
}
STATIC_URL = "/static/"
@@ -253,24 +310,46 @@ STATICFILES_DIRS = [
os.path.join(BASE_DIR, "static_source/vite"),
]
# Submission/media uploads always go to S3 - there's no local fallback for
# that, so these must be set (via env) wherever the app runs, dev included.
AWS_ACCESS_KEY_ID = os.environ.get("AWS_ACCESS_KEY_ID", "")
AWS_SECRET_ACCESS_KEY = os.environ.get("AWS_SECRET_ACCESS_KEY", "")
AWS_S3_ENDPOINT_URL = os.environ.get("AWS_S3_ENDPOINT_URL") or "https://storage.polylan.ch"
AWS_STORAGE_BUCKET_NAME = os.environ.get("AWS_STORAGE_BUCKET_NAME") or "dev-submissions"
_s3_storage_options = {
"bucket_name": AWS_STORAGE_BUCKET_NAME,
"endpoint_url": AWS_S3_ENDPOINT_URL,
"access_key": AWS_ACCESS_KEY_ID,
"secret_key": AWS_SECRET_ACCESS_KEY,
}
STORAGES = {
"default": {
"BACKEND": "storages.backends.s3.S3Storage",
"OPTIONS": _s3_storage_options,
},
# Static files are collected to local disk and served by WhiteNoise in
# DEBUG (no need to push every CSS/JS tweak to S3 while developing);
# production collects straight to S3 like everything else here.
"staticfiles": {
"BACKEND": "storages.backends.s3.S3Storage",
"BACKEND": "whitenoise.storage.CompressedManifestStaticFilesStorage"
if DEBUG
else "storages.backends.s3.S3Storage",
**({} if DEBUG else {"OPTIONS": _s3_storage_options}),
},
}
SENTRY_ENVIRONMENT = "production"
from polylan_submitter.settingsLocal import * # noqa
SENTRY_ENVIRONMENT = os.environ.get("SENTRY_ENVIRONMENT") or ("development" if DEBUG else "production")
SENTRY_DSN = (
os.environ.get("SENTRY_DSN") or "https://cc62a4ce3f3470890b43accf02cc6d8c@sentry2.polylan.ch/12"
)
import sentry_sdk # noqa
sentry_sdk.init(
dsn="https://cc62a4ce3f3470890b43accf02cc6d8c@sentry2.polylan.ch/12",
dsn=SENTRY_DSN,
# Set traces_sample_rate to 1.0 to capture 100%
# of transactions for tracing.
traces_sample_rate=1.0,
+1
View File
@@ -27,6 +27,7 @@ dependencies = [
"django-tasks-rq",
"httpx>=0.28.1",
"django-object-actions>=5.1.2",
"python-dotenv>=1.2.3",
]
[dependency-groups]
Generated
+11
View File
@@ -1651,6 +1651,7 @@ dependencies = [
{ name = "pillow" },
{ name = "psycopg" },
{ name = "pytesseract" },
{ name = "python-dotenv" },
{ name = "redis" },
{ name = "requests" },
{ name = "sentry-sdk", extra = ["django"] },
@@ -1692,6 +1693,7 @@ requires-dist = [
{ name = "pillow" },
{ name = "psycopg" },
{ name = "pytesseract" },
{ name = "python-dotenv", specifier = ">=1.2.3" },
{ name = "redis" },
{ name = "requests" },
{ name = "sentry-sdk", extras = ["django"] },
@@ -2052,6 +2054,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/b7/6f/a05a317a66fee0aad270011461f1a63a453ed12471249f172f7d2e2bc7b4/python_discovery-1.3.1-py3-none-any.whl", hash = "sha256:ed188687ebb3b82c01a17cd5ac62fc94d9f6487a7f1a0f9dfe89753fec91039c", size = 33185, upload-time = "2026-05-12T20:53:34.969Z" },
]
[[package]]
name = "python-dotenv"
version = "1.2.3"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/6a/53/ed9d74092561d4b01a2ef1349d52cdbc135e526c245f366b089cfca6de49/python_dotenv-1.2.3.tar.gz", hash = "sha256:a20a594dabeaa385725aa239d5244871c143ecb356add8a20fcf23773a6c3a35", size = 58945, upload-time = "2026-08-16T16:54:54.067Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/0d/17/c5c6b53ddc18f297992099b3d9ec16c855c0ccc83263a21fe4d1c625ec6c/python_dotenv-1.2.3-py3-none-any.whl", hash = "sha256:904552145e8bfed22162c09dab1c2b9b54fefa7b23ba780f4f26ca0316b0f0d9", size = 22780, upload-time = "2026-08-16T16:54:52.473Z" },
]
[[package]]
name = "pytz"
version = "2026.2"