fix(clock): dedup tasks lock bug removal

polylan_submitter/clock/tasks.py

@@ -12,6 +12,7 @@ acts.
 """
 
 from datetime import timedelta
+from uuid import uuid4
 
 from asgiref.sync import async_to_sync
 from channels.layers import get_channel_layer
@@ -30,13 +31,20 @@ HEARTBEAT_INTERVAL_SECONDS = 1
 # One chain ticks every event with the game enabled - this lock keeps a
 # second chain from starting underneath an already-running one (e.g. two
 # EventGame saves close together). Refreshed every tick, wide enough that a
-# single slow tick doesn't let a second chain slip in.
+# single slow tick doesn't let a second chain slip in. The lock holds the
+# chain id: if it expires anyway (worker down/restarted for a few seconds) and
+# a new chain starts, the old one stops at its next tick (instead of both
+# ticking forever).
 HEARTBEAT_LOCK_KEY = "clock:heartbeat:lock"
 HEARTBEAT_LOCK_TIMEOUT_SECONDS = HEARTBEAT_INTERVAL_SECONDS * 5
 
 
 @task
-def clock_heartbeat_task():
+def clock_heartbeat_task(chain_id: str):
+    if cache.get(HEARTBEAT_LOCK_KEY) != chain_id:
+        # Another chain owns the lock (or it expired), stop this one
+        return
+
     event_games = list(
         EventGame.objects.filter(game__path="clock", enabled=True).select_related(
             "event"
@@ -46,7 +54,8 @@ def clock_heartbeat_task():
     if not event_games:
         # Nothing left to tick - let the chain die. start_clock_heartbeat()
         # starts a fresh one next time an event enables the game.
-        cache.delete(HEARTBEAT_LOCK_KEY)
+        if cache.get(HEARTBEAT_LOCK_KEY) == chain_id:
+            cache.delete(HEARTBEAT_LOCK_KEY)
         return
 
     channel_layer = get_channel_layer()
@@ -67,13 +76,14 @@ def clock_heartbeat_task():
             },
         )
 
-    cache.set(HEARTBEAT_LOCK_KEY, 1, timeout=HEARTBEAT_LOCK_TIMEOUT_SECONDS)
+    cache.set(HEARTBEAT_LOCK_KEY, chain_id, timeout=HEARTBEAT_LOCK_TIMEOUT_SECONDS)
     clock_heartbeat_task.using(
         run_after=timezone.now() + timedelta(seconds=HEARTBEAT_INTERVAL_SECONDS)
-    ).enqueue()
+    ).enqueue(chain_id)
 
 
 def start_clock_heartbeat():
     """Enqueues the heartbeat chain, unless one is already running."""
-    if cache.add(HEARTBEAT_LOCK_KEY, 1, timeout=HEARTBEAT_LOCK_TIMEOUT_SECONDS):
-        clock_heartbeat_task.enqueue()
+    chain_id = uuid4().hex
+    if cache.add(HEARTBEAT_LOCK_KEY, chain_id, timeout=HEARTBEAT_LOCK_TIMEOUT_SECONDS):
+        clock_heartbeat_task.enqueue(chain_id)

polylan_submitter/clock/tests.py

@@ -0,0 +1,57 @@
+from unittest import mock
+
+from django.core.cache import cache
+from django.test import TestCase, override_settings
+
+from events.models import Event
+from games.models import EventGame, Game
+
+from .tasks import HEARTBEAT_LOCK_KEY, clock_heartbeat_task, start_clock_heartbeat
+
+LOCMEM_CACHE = {"default": {"BACKEND": "django.core.cache.backends.locmem.LocMemCache"}}
+
+
+@override_settings(CACHES=LOCMEM_CACHE)
+@mock.patch("clock.tasks.get_channel_layer")
+class ClockHeartbeatTests(TestCase):
+    def setUp(self):
+        cache.clear()
+        event = Event.objects.create(name="Event", path="event")
+        clock = Game.objects.create(name="Clock", path="clock", custom_path=True)
+        self.event_game = EventGame.objects.create(event=event, game=clock)
+        cache.clear()  # post_save signal can start a chain
+
+    @mock.patch("clock.tasks.clock_heartbeat_task")
+    def test_only_one_chain_is_started(self, task, _):
+        start_clock_heartbeat()
+        start_clock_heartbeat()
+
+        task.enqueue.assert_called_once_with(cache.get(HEARTBEAT_LOCK_KEY))
+
+    def test_owning_chain_reschedules_itself(self, channel_layer):
+        channel_layer.return_value.group_send = mock.AsyncMock()
+        cache.set(HEARTBEAT_LOCK_KEY, "chain")
+        with mock.patch("clock.tasks.clock_heartbeat_task") as patched:
+            clock_heartbeat_task.call("chain")
+
+        patched.using.return_value.enqueue.assert_called_once_with("chain")
+        channel_layer.return_value.group_send.assert_awaited_once()
+        self.assertEqual(cache.get(HEARTBEAT_LOCK_KEY), "chain")
+
+    def test_superseded_chain_stops(self, channel_layer):
+        cache.set(HEARTBEAT_LOCK_KEY, "new-chain")
+        with mock.patch("clock.tasks.clock_heartbeat_task") as patched:
+            clock_heartbeat_task.call("old-chain")
+
+        patched.using.assert_not_called()
+        channel_layer.assert_not_called()
+        self.assertEqual(cache.get(HEARTBEAT_LOCK_KEY), "new-chain")
+
+    def test_chain_dies_and_releases_lock_when_game_disabled(self, _):
+        EventGame.objects.filter(pk=self.event_game.pk).update(enabled=False)
+        cache.set(HEARTBEAT_LOCK_KEY, "chain")
+        with mock.patch("clock.tasks.clock_heartbeat_task") as patched:
+            clock_heartbeat_task.call("chain")
+
+        patched.using.assert_not_called()
+        self.assertIsNone(cache.get(HEARTBEAT_LOCK_KEY))