From 512ad83c085366e426512e806751c6a0ce1f8145 Mon Sep 17 00:00:00 2001 From: Nikhil <118773738+pablohashescobar@users.noreply.github.com> Date: Tue, 23 Jan 2024 15:07:45 +0530 Subject: [PATCH] dev: migration check (#3440) * dev: wait for migrations script for beat, worker, takeoff * dev: update docker compose and migrator commands * dev: migrator commands for self hosted setup * dev: add recursive flag for chown --- apiserver/Dockerfile.dev | 11 +++----- apiserver/bin/beat | 3 +++ apiserver/bin/takeoff | 3 ++- apiserver/bin/takeoff.local | 3 ++- apiserver/bin/worker | 3 +++ .../commands/wait_for_migrations.py | 21 +++++++++++++++ deploy/selfhost/docker-compose.yml | 12 +++++++++ docker-compose-local.yml | 26 ++++++++++++++++--- 8 files changed, 69 insertions(+), 13 deletions(-) mode change 100644 => 100755 apiserver/bin/beat create mode 100644 apiserver/plane/db/management/commands/wait_for_migrations.py diff --git a/apiserver/Dockerfile.dev b/apiserver/Dockerfile.dev index cb2d1ca28..bd6684fd5 100644 --- a/apiserver/Dockerfile.dev +++ b/apiserver/Dockerfile.dev @@ -33,15 +33,10 @@ RUN pip install -r requirements/local.txt --compile --no-cache-dir RUN addgroup -S plane && \ adduser -S captain -G plane -RUN chown captain.plane /code +COPY . . -USER captain - -# Add in Django deps and generate Django's static files - -USER root - -# RUN chmod +x ./bin/takeoff ./bin/worker ./bin/beat +RUN chown -R captain.plane /code +RUN chmod -R +x /code/bin RUN chmod -R 777 /code USER captain diff --git a/apiserver/bin/beat b/apiserver/bin/beat old mode 100644 new mode 100755 index 45d357442..3a9602a9e --- a/apiserver/bin/beat +++ b/apiserver/bin/beat @@ -2,4 +2,7 @@ set -e python manage.py wait_for_db +# Wait for migrations +python manage.py wait_for_migrations +# Run the processes celery -A plane beat -l info \ No newline at end of file diff --git a/apiserver/bin/takeoff b/apiserver/bin/takeoff index 0ec2e495c..efea53f87 100755 --- a/apiserver/bin/takeoff +++ b/apiserver/bin/takeoff @@ -1,7 +1,8 @@ #!/bin/bash set -e python manage.py wait_for_db -python manage.py migrate +# Wait for migrations +python manage.py wait_for_migrations # Create the default bucket #!/bin/bash diff --git a/apiserver/bin/takeoff.local b/apiserver/bin/takeoff.local index b89c20874..8f62370ec 100755 --- a/apiserver/bin/takeoff.local +++ b/apiserver/bin/takeoff.local @@ -1,7 +1,8 @@ #!/bin/bash set -e python manage.py wait_for_db -python manage.py migrate +# Wait for migrations +python manage.py wait_for_migrations # Create the default bucket #!/bin/bash diff --git a/apiserver/bin/worker b/apiserver/bin/worker index 9d2da1254..a70b5f77c 100755 --- a/apiserver/bin/worker +++ b/apiserver/bin/worker @@ -2,4 +2,7 @@ set -e python manage.py wait_for_db +# Wait for migrations +python manage.py wait_for_migrations +# Run the processes celery -A plane worker -l info \ No newline at end of file diff --git a/apiserver/plane/db/management/commands/wait_for_migrations.py b/apiserver/plane/db/management/commands/wait_for_migrations.py new file mode 100644 index 000000000..51f2cf339 --- /dev/null +++ b/apiserver/plane/db/management/commands/wait_for_migrations.py @@ -0,0 +1,21 @@ +# wait_for_migrations.py +import time +from django.core.management.base import BaseCommand +from django.db.migrations.executor import MigrationExecutor +from django.db import connections, DEFAULT_DB_ALIAS + +class Command(BaseCommand): + help = 'Wait for database migrations to complete before starting Celery worker/beat' + + def handle(self, *args, **kwargs): + while self._pending_migrations(): + self.stdout.write("Waiting for database migrations to complete...") + time.sleep(10) # wait for 10 seconds before checking again + + self.stdout.write(self.style.SUCCESS("No migrations Pending. Starting processes ...")) + + def _pending_migrations(self): + connection = connections[DEFAULT_DB_ALIAS] + executor = MigrationExecutor(connection) + targets = executor.loader.graph.leaf_nodes() + return bool(executor.migration_plan(targets)) diff --git a/deploy/selfhost/docker-compose.yml b/deploy/selfhost/docker-compose.yml index e42f53c7a..b223e722a 100644 --- a/deploy/selfhost/docker-compose.yml +++ b/deploy/selfhost/docker-compose.yml @@ -122,6 +122,18 @@ services: - plane-db - plane-redis + migrator: + <<: *app-env + image: ${DOCKERHUB_USER:-makeplane}/plane-backend:${APP_RELEASE:-latest} + pull_policy: ${PULL_POLICY:-always} + restart: no + command: > + sh -c "python manage.py wait_for_db && + python manage.py migrate" + depends_on: + - plane-db + - plane-redis + plane-db: <<: *app-env image: postgres:15.2-alpine diff --git a/docker-compose-local.yml b/docker-compose-local.yml index b0fb9da24..a2e518708 100644 --- a/docker-compose-local.yml +++ b/docker-compose-local.yml @@ -86,7 +86,7 @@ services: - dev_env volumes: - ./apiserver:/code - # command: /bin/sh -c "python manage.py migrate && python manage.py runserver 0.0.0.0:8000 --settings=plane.settings.local" + command: ./bin/takeoff.local env_file: - ./apiserver/.env depends_on: @@ -104,7 +104,7 @@ services: - dev_env volumes: - ./apiserver:/code - command: /bin/sh -c "celery -A plane worker -l info" + command: ./bin/worker env_file: - ./apiserver/.env depends_on: @@ -123,7 +123,7 @@ services: - dev_env volumes: - ./apiserver:/code - command: /bin/sh -c "celery -A plane beat -l info" + command: ./bin/beat env_file: - ./apiserver/.env depends_on: @@ -131,6 +131,26 @@ services: - plane-db - plane-redis + migrator: + build: + context: ./apiserver + dockerfile: Dockerfile.dev + args: + DOCKER_BUILDKIT: 1 + restart: no + networks: + - dev_env + volumes: + - ./apiserver:/code + command: > + sh -c "python manage.py wait_for_db --settings=plane.settings.local && + python manage.py migrate --settings=plane.settings.local" + env_file: + - ./apiserver/.env + depends_on: + - plane-db + - plane-redis + proxy: build: context: ./nginx