diff --git a/apiserver/bin/beat b/apiserver/bin/beat old mode 100644 new mode 100755 diff --git a/apiserver/plane/db/management/commands/safe_migrate.py b/apiserver/plane/db/management/commands/safe_migrate.py index fbe026670..76f1c4278 100644 --- a/apiserver/plane/db/management/commands/safe_migrate.py +++ b/apiserver/plane/db/management/commands/safe_migrate.py @@ -1,5 +1,7 @@ # Python imports +import time import uuid +import atexit # Django imports from django.core.management.base import BaseCommand @@ -7,42 +9,47 @@ from django.core.management import call_command from django.db.migrations.executor import MigrationExecutor from django.db import connections, DEFAULT_DB_ALIAS -# Module imports -from plane.settings.redis import redis_instance - class Command(BaseCommand): - help = 'Run migrations with Redis distributed locking if there are pending migrations' + help = 'Wait for migrations to be completed and acquire lock before starting migrations' def handle(self, *args, **kwargs): - # Check for pending migrations - if not self._pending_migrations(): - self.stdout.write("No pending migrations.") - return + self.lock_key = 'django_migration_lock' + self.lock_value = str(uuid.uuid4()) # Unique value for the lock + self.client = redis_instance() - # Proceed with acquiring the lock and running migrations - lock_key = 'django_migration_lock' - lock_value = str(uuid.uuid4()) # Unique value for the lock - client = redis_instance() + # Register the cleanup function + atexit.register(self.cleanup) - # Try acquiring the lock - if client.set(lock_key, lock_value, nx=True, ex=300): # 5 minutes expiry - try: - self.stdout.write("Acquired migration lock, running migrations...") - call_command('migrate') - finally: - # Release the lock if it belongs to this process - stored_value = client.get(lock_key) - if stored_value and stored_value.decode() == lock_value: - client.delete(lock_key) - self.stdout.write("Released migration lock.") - else: - self.stdout.write("Lock was not released, as it was held by another instance.") - else: - self.stdout.write("Migration lock is held by another instance.") + while self._pending_migrations(): + # Try acquiring the lock + if self.client.set(self.lock_key, self.lock_value, nx=True, ex=300): # 5 minutes expiry + try: + self.stdout.write("Acquired migration lock, running migrations...") + call_command('migrate') + except Exception as e: + self.stdout.write(f"An error occurred during migrations: {e}") + finally: + # Release the lock if it belongs to this process + self.cleanup() + return # Exit after attempting migration + else: + self.stdout.write("Migration lock is held by another instance. Waiting 10 seconds to retry...") + time.sleep(10) # Wait for 10 seconds before retrying + + self.stdout.write("No pending migrations.") def _pending_migrations(self): connection = connections[DEFAULT_DB_ALIAS] executor = MigrationExecutor(connection) targets = executor.loader.graph.leaf_nodes() return bool(executor.migration_plan(targets)) + + def cleanup(self): + """ + Clean up function to release the lock. + """ + stored_value = self.client.get(self.lock_key) + if stored_value and stored_value.decode() == self.lock_value: + self.client.delete(self.lock_key) + self.stdout.write("Released migration lock.") \ No newline at end of file diff --git a/docker-compose-local.yml b/docker-compose-local.yml index b0fb9da24..d609eb761 100644 --- a/docker-compose-local.yml +++ b/docker-compose-local.yml @@ -76,6 +76,8 @@ services: - web api: + deploy: + replicas: 3 build: context: ./apiserver dockerfile: Dockerfile.dev @@ -86,7 +88,7 @@ services: - dev_env volumes: - ./apiserver:/code - # command: /bin/sh -c "python manage.py migrate && python manage.py runserver 0.0.0.0:8000 --settings=plane.settings.local" + command: ./bin/takeoff.local env_file: - ./apiserver/.env depends_on: @@ -104,7 +106,7 @@ services: - dev_env volumes: - ./apiserver:/code - command: /bin/sh -c "celery -A plane worker -l info" + command: ./bin/worker env_file: - ./apiserver/.env depends_on: @@ -123,7 +125,7 @@ services: - dev_env volumes: - ./apiserver:/code - command: /bin/sh -c "celery -A plane beat -l info" + command: ./bin/beat env_file: - ./apiserver/.env depends_on: