dev: exception handling

apiserver/plane/db/management/commands/safe_migrate.py

@@ -1,5 +1,7 @@
 # Python imports
+import time
 import uuid
+import atexit
 
 # Django imports
 from django.core.management.base import BaseCommand
@@ -7,42 +9,47 @@ from django.core.management import call_command
 from django.db.migrations.executor import MigrationExecutor
 from django.db import connections, DEFAULT_DB_ALIAS
 
-# Module imports
-from plane.settings.redis import redis_instance
-
 
 class Command(BaseCommand):
-    help = 'Run migrations with Redis distributed locking if there are pending migrations'
+    help = 'Wait for migrations to be completed and acquire lock before starting migrations'
 
     def handle(self, *args, **kwargs):
-        # Check for pending migrations
-        if not self._pending_migrations():
-            self.stdout.write("No pending migrations.")
-            return
+        self.lock_key = 'django_migration_lock'
+        self.lock_value = str(uuid.uuid4())  # Unique value for the lock
+        self.client = redis_instance()
 
-        # Proceed with acquiring the lock and running migrations
-        lock_key = 'django_migration_lock'
-        lock_value = str(uuid.uuid4())  # Unique value for the lock
-        client = redis_instance()
+        # Register the cleanup function
+        atexit.register(self.cleanup)
 
-        # Try acquiring the lock
-        if client.set(lock_key, lock_value, nx=True, ex=300):  # 5 minutes expiry
-            try:
-                self.stdout.write("Acquired migration lock, running migrations...")
-                call_command('migrate')
-            finally:
-                # Release the lock if it belongs to this process
-                stored_value = client.get(lock_key)
-                if stored_value and stored_value.decode() == lock_value:
-                    client.delete(lock_key)
-                    self.stdout.write("Released migration lock.")
-                else:
-                    self.stdout.write("Lock was not released, as it was held by another instance.")
-        else:
-            self.stdout.write("Migration lock is held by another instance.")
+        while self._pending_migrations():
+            # Try acquiring the lock
+            if self.client.set(self.lock_key, self.lock_value, nx=True, ex=300):  # 5 minutes expiry
+                try:
+                    self.stdout.write("Acquired migration lock, running migrations...")
+                    call_command('migrate')
+                except Exception as e:
+                    self.stdout.write(f"An error occurred during migrations: {e}")
+                finally:
+                    # Release the lock if it belongs to this process
+                    self.cleanup()
+                return  # Exit after attempting migration
+            else:
+                self.stdout.write("Migration lock is held by another instance. Waiting 10 seconds to retry...")
+                time.sleep(10)  # Wait for 10 seconds before retrying
+
+        self.stdout.write("No pending migrations.")
 
     def _pending_migrations(self):
         connection = connections[DEFAULT_DB_ALIAS]
         executor = MigrationExecutor(connection)
         targets = executor.loader.graph.leaf_nodes()
         return bool(executor.migration_plan(targets))
+
+    def cleanup(self):
+        """
+        Clean up function to release the lock.
+        """
+        stored_value = self.client.get(self.lock_key)
+        if stored_value and stored_value.decode() == self.lock_value:
+            self.client.delete(self.lock_key)
+            self.stdout.write("Released migration lock.")

docker-compose-local.yml

@@ -76,6 +76,8 @@ services:
       - web
 
   api:
+    deploy:
+      replicas: 3
     build:
       context: ./apiserver
       dockerfile: Dockerfile.dev
@@ -86,7 +88,7 @@ services:
       - dev_env
     volumes:
       - ./apiserver:/code
-    # command: /bin/sh -c "python manage.py migrate && python manage.py runserver 0.0.0.0:8000 --settings=plane.settings.local"
+    command: ./bin/takeoff.local
     env_file:
       - ./apiserver/.env
     depends_on:
@@ -104,7 +106,7 @@ services:
       - dev_env
     volumes:
       - ./apiserver:/code
-    command: /bin/sh -c "celery -A plane worker -l info"
+    command: ./bin/worker
     env_file:
       - ./apiserver/.env
     depends_on:
@@ -123,7 +125,7 @@ services:
       - dev_env
     volumes:
       - ./apiserver:/code
-    command: /bin/sh -c "celery -A plane beat -l info"
+    command: ./bin/beat
     env_file:
       - ./apiserver/.env
     depends_on: