fix(clock): dedup tasks lock bug removal

polylan_submitter/clock/tasks.py

@@ -12,6 +12,7 @@ acts.
 """
 
 from datetime import timedelta
+from uuid import uuid4
 
 from asgiref.sync import async_to_sync
 from channels.layers import get_channel_layer
@@ -30,13 +31,20 @@ HEARTBEAT_INTERVAL_SECONDS = 1
 # One chain ticks every event with the game enabled - this lock keeps a
 # second chain from starting underneath an already-running one (e.g. two
 # EventGame saves close together). Refreshed every tick, wide enough that a
-# single slow tick doesn't let a second chain slip in.
+# single slow tick doesn't let a second chain slip in. The lock holds the
+# chain id: if it expires anyway (worker down/restarted for a few seconds) and
+# a new chain starts, the old one stops at its next tick (instead of both
+# ticking forever).
 HEARTBEAT_LOCK_KEY = "clock:heartbeat:lock"
 HEARTBEAT_LOCK_TIMEOUT_SECONDS = HEARTBEAT_INTERVAL_SECONDS * 5
 
 
 @task
-def clock_heartbeat_task():
+def clock_heartbeat_task(chain_id: str):
+    if cache.get(HEARTBEAT_LOCK_KEY) != chain_id:
+        # Another chain owns the lock (or it expired), stop this one
+        return
+
     event_games = list(
         EventGame.objects.filter(game__path="clock", enabled=True).select_related(
             "event"
@@ -46,7 +54,8 @@ def clock_heartbeat_task():
     if not event_games:
         # Nothing left to tick - let the chain die. start_clock_heartbeat()
         # starts a fresh one next time an event enables the game.
-        cache.delete(HEARTBEAT_LOCK_KEY)
+        if cache.get(HEARTBEAT_LOCK_KEY) == chain_id:
+            cache.delete(HEARTBEAT_LOCK_KEY)
         return
 
     channel_layer = get_channel_layer()
@@ -67,13 +76,14 @@ def clock_heartbeat_task():
             },
         )
 
-    cache.set(HEARTBEAT_LOCK_KEY, 1, timeout=HEARTBEAT_LOCK_TIMEOUT_SECONDS)
+    cache.set(HEARTBEAT_LOCK_KEY, chain_id, timeout=HEARTBEAT_LOCK_TIMEOUT_SECONDS)
     clock_heartbeat_task.using(
         run_after=timezone.now() + timedelta(seconds=HEARTBEAT_INTERVAL_SECONDS)
-    ).enqueue()
+    ).enqueue(chain_id)
 
 
 def start_clock_heartbeat():
     """Enqueues the heartbeat chain, unless one is already running."""
-    if cache.add(HEARTBEAT_LOCK_KEY, 1, timeout=HEARTBEAT_LOCK_TIMEOUT_SECONDS):
-        clock_heartbeat_task.enqueue()
+    chain_id = uuid4().hex
+    if cache.add(HEARTBEAT_LOCK_KEY, chain_id, timeout=HEARTBEAT_LOCK_TIMEOUT_SECONDS):
+        clock_heartbeat_task.enqueue(chain_id)