fix: deploy on fly, migrate to overmind

This commit is contained in:
orion 2024-04-15 22:37:21 -05:00
parent 37c51cfca5
commit 34d35a8da0
Signed by: orion
GPG Key ID: 6D4165AE4C928719
47 changed files with 192 additions and 163 deletions

View File

@ -2,8 +2,6 @@
*.pyc
.env
venv
node_modules/
**/node_modules/
npm-debug.log
.next/
**/.next/

View File

@ -1,50 +1,15 @@
FROM git.orionkindel.com/tpl/asdf:bookworm AS system
FROM git.orionkindel.com/tpl/asdf:bookworm-full AS base
ARG S6_OVERLAY_VERSION=3.1.6.2
RUN apt-get update && apt-get install -y wget nginx procps lsb-release tmux
RUN wget https://dl.min.io/server/minio/release/linux-amd64/minio -O /usr/bin/minio && chmod +x /usr/bin/minio
RUN go install github.com/DarthSim/overmind/v2@latest
RUN asdf global python 3.12.3 && asdf plugin remove postgres && pip install setuptools
RUN npm i -g yarn && yarn set version 1.22.19
ADD https://github.com/just-containers/s6-overlay/releases/download/v${S6_OVERLAY_VERSION}/s6-overlay-noarch.tar.xz /tmp
RUN tar -C / -Jxpf /tmp/s6-overlay-noarch.tar.xz
ADD https://github.com/just-containers/s6-overlay/releases/download/v${S6_OVERLAY_VERSION}/s6-overlay-x86_64.tar.xz /tmp
RUN tar -C / -Jxpf /tmp/s6-overlay-x86_64.tar.xz
RUN apt-get update
RUN apt-get install -y \
build-essential \
zlib1g-dev \
libncurses5-dev \
libgdbm-dev \
libnss3-dev \
libssl-dev \
libreadline-dev \
libffi-dev \
libsqlite3-dev \
wget \
libbz2-dev \
uuid-dev \
nginx \
procps
RUN asdf plugin add nodejs \
&& asdf plugin add python \
&& asdf plugin add postgres
RUN --mount=type=cache,target=/.asdf-build \
export ASDF_DOWNLOAD_PATH=/.asdf-build \
&& export TMPDIR=/.asdf-build \
&& export POSTGRES_SKIP_INITDB=y \
&& asdf install nodejs 20.9.0 \
&& asdf install python 3.11.1 \
&& asdf install postgres 15.3
RUN asdf global nodejs 20.9.0 \
&& asdf global postgres 15.3 \
&& asdf global python 3.11.1
RUN useradd -m postgres && passwd -d postgres
ADD https://dl.min.io/server/minio/release/linux-amd64/minio /usr/bin
RUN chmod +x /usr/bin/minio
RUN sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' \
&& wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
&& apt-get update \
&& apt-get -y install postgresql
RUN set -eo pipefail; \
curl -fsSL https://packages.redis.io/gpg | gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg; \
@ -52,59 +17,23 @@ RUN set -eo pipefail; \
apt-get update; \
apt-get install -y redis
FROM system AS next_prebuild
FROM base AS run
RUN npm i -g yarn
RUN --mount=type=cache,target=/.yarn-cache \
yarn config set cache-folder /.yarn-cache
COPY --from=git.orionkindel.com/thunderstrike/plane-base:latest /app /app
COPY --from=git.orionkindel.com/thunderstrike/plane-base:latest /etc /etc
COPY --from=git.orionkindel.com/thunderstrike/plane-base:latest /root/.asdf/installs/python /root/.asdf/installs/python
COPY package.json turbo.json yarn.lock app.json ./
COPY packages packages
COPY web web
COPY space space
WORKDIR /app
RUN --mount=type=cache,target=/.yarn-cache \
yarn install
FROM next_prebuild AS next_build
RUN --mount=type=cache,target=/.yarn-cache \
--mount=type=cache,target=/web/.next \
--mount=type=cache,target=/space/.next \
yarn build && \
cp -R /web/.next /web/_next && \
cp -R /space/.next /space/_next
RUN mv /web/_next /web/.next && \
mv /space/_next /space/.next && \
cp -R /web/.next/standalone/web/* /web/ && \
cp -R /space/.next/standalone/space/* /space/
FROM next_build AS api_build
ENV PYTHONDONTWRITEBYTECODE=1
ENV PYTHONUNBUFFERED=1
ENV PIP_DISABLE_PIP_VERSION_CHECK=1
COPY apiserver apiserver
RUN --mount=type=cache,target=/root/.cache/pip \
cd /apiserver \
&& pip install -r requirements.txt --compile
FROM api_build AS s6
COPY docker/etc/ /etc/
RUN chmod -R 777 /root \
&& chmod -R 777 /root/.asdf \
RUN chmod -R 755 /root \
&& chmod -R 755 /root/.asdf \
&& chmod -R 755 apiserver \
&& chmod -R 755 web \
&& chmod -R 755 space \
&& chmod -x /root/.asdf/lib/commands/* \
&& chmod -R 777 /apiserver \
&& chmod -R 777 /web \
&& chmod -R 777 /space \
&& ln $(asdf which postgres) /usr/bin/postgres \
&& ln $(asdf which initdb) /usr/bin/initdb \
&& chmod +x /etc/db-run.sh \
&& ln $(asdf which node) /usr/bin/node \
&& ln $(asdf which npm) /usr/bin/npm \
&& ln $(asdf which python) /usr/bin/python
ENV S6_KEEP_ENV=1
ENTRYPOINT ["/init"]
ENTRYPOINT ["bash", "-lc", "/etc/entrypoint.sh"]

58
Dockerfile.base Normal file
View File

@ -0,0 +1,58 @@
FROM git.orionkindel.com/tpl/asdf:bookworm-full AS system
RUN apt-get update && apt-get install -y wget nginx procps lsb-release tmux
RUN wget https://dl.min.io/server/minio/release/linux-amd64/minio -O /usr/bin/minio && chmod +x /usr/bin/minio
RUN go install github.com/DarthSim/overmind/v2@latest
RUN asdf global python 3.12.3 && asdf plugin remove postgres && pip install setuptools
RUN npm i -g yarn && yarn set version 1.22.19
RUN sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' \
&& wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \
&& apt-get update \
&& apt-get -y install postgresql
RUN set -eo pipefail; \
curl -fsSL https://packages.redis.io/gpg | gpg --dearmor -o /usr/share/keyrings/redis-archive-keyring.gpg; \
echo "deb [signed-by=/usr/share/keyrings/redis-archive-keyring.gpg] https://packages.redis.io/deb bookworm main" | tee /etc/apt/sources.list.d/redis.list; \
apt-get update; \
apt-get install -y redis
FROM system AS next_build
RUN mkdir /app
WORKDIR /app
COPY package.json turbo.json yarn.lock app.json .
COPY packages packages
COPY web web
COPY space space
RUN --mount=type=cache,target=/root/.cache/yarn \
yarn install --frozen-lockfile \
&& yarn build \
&& rm -rf node_modules
RUN mkdir -p web/.next/standalone/web/.next \
&& mv web/public web/.next/standalone/web/public \
&& mv web/.next/static web/.next/standalone/web/.next/static \
&& mkdir -p space/.next/standalone/_next \
&& mv space/public space/.next/standalone/space/public \
&& mv space/.next/static space/.next/standalone/space/.next/static
FROM next_build AS api_build
ENV PYTHONDONTWRITEBYTECODE=1
ENV PYTHONUNBUFFERED=1
ENV PIP_DISABLE_PIP_VERSION_CHECK=1
COPY apiserver apiserver
RUN --mount=type=cache,target=/root/.cache/pip \
cd apiserver \
&& pip install -r requirements.txt --compile \
&& pip install -r requirements/local.txt --compile
FROM api_build AS run
COPY docker/etc/ /etc/
ENTRYPOINT ["bash", "-l"]

10
docker/etc/Procfile Normal file
View File

@ -0,0 +1,10 @@
migrator: sh -c "cd /app/apiserver && python manage.py wait_for_db --settings=plane.settings.local && python manage.py migrate --settings=plane.settings.local"
api: /bin/bash -lc 'cd /app/apiserver && ./bin/takeoff'
api-beatworker: /bin/bash -lc 'cd /app/apiserver && ./bin/beat'
api-worker: /bin/bash -lc 'cd /app/apiserver && ./bin/worker'
db: /etc/db-run.sh
minio: /usr/bin/minio server /ext/minio_data --console-address ":9090"
nginx: nginx -g "daemon off;"
redis: /usr/bin/redis-server /etc/redis/redis.conf
spaces: cd /app/space/.next/standalone/space && HOSTNAME=127.0.0.1 PORT=3001 node server.js space
web: cd /app/web/.next/standalone/web && NEXT_PUBLIC_DEPLOY_URL="http://localhost/spaces" HOSTNAME=127.0.0.1 PORT=3000 node server.js web

34
docker/etc/db-run.sh Normal file
View File

@ -0,0 +1,34 @@
#!/bin/bash
script=$(mktemp)
chmod 777 "$script"
chmod -R 750 /ext/pg_data
chown -R postgres:postgres /ext/pg_data
cat << "EOSH" > "$script"
set -m
PGBIN=/usr/lib/postgresql/16/bin/
PGSHARE=/usr/share/postgresql/16/
export POSTGRES_USER=plane
export POSTGRES_PASSWORD=plane
export POSTGRES_DB=plane
export PGDATA=/ext/pg_data
export ROOT_DB_URL=postgresql://plane:plane@localhost:5432/postgres
export TZ=GMT
pwfile=$(mktemp)
echo -n $POSTGRES_PASSWORD > $pwfile
$PGBIN/initdb -U $POSTGRES_USER --pwfile=$pwfile --no-locale -L $PGSHARE || true
$PGBIN/postgres -c 'max_connections=1000' & true;
until pg_isready; do sleep 1; done;
psql -d postgres -c 'create database plane;'
fg %1
EOSH
su postgres -c "/bin/bash $script"

9
docker/etc/entrypoint.sh Executable file
View File

@ -0,0 +1,9 @@
#! /bin/bash
mkdir -p /ext/redis_data /ext/minio_data /ext/pg_data
asdf reshim
OVERMIND_CAN_DIE=migrator \
OVERMIND_AUTO_RESTART=api,web,spaces,api-worker,api-beatworker \
OVERMIND_NO_PORT=1 \
overmind start -f /etc/Procfile

View File

@ -507,7 +507,7 @@ rdb-del-sync-files no
# The Append Only File will also be created inside this directory.
#
# Note that you must specify a directory here, not a file name.
dir /redis_data
dir /ext/redis_data
################################# REPLICATION #################################

View File

@ -1,2 +0,0 @@
#!/bin/bash
/bin/bash -lc 'cd /apiserver && /apiserver/bin/beat'

View File

@ -1 +0,0 @@
longrun

View File

@ -1,2 +0,0 @@
#!/bin/bash
/bin/bash -lc 'cd /apiserver && /apiserver/bin/worker'

View File

@ -1 +0,0 @@
longrun

View File

@ -1,2 +0,0 @@
#!/bin/bash
/bin/bash -lc 'cd /apiserver && /apiserver/bin/takeoff'

View File

@ -1 +0,0 @@
longrun

View File

@ -1,2 +0,0 @@
#!/bin/bash
/run/s6/basedir/bin/halt

View File

@ -1,26 +0,0 @@
#!/bin/bash
script=$(mktemp)
chmod 777 "$script"
chmod -R 750 /pg_data
chown -R postgres:postgres /pg_data
cat << "EOSH" > "$script"
pgdir=/root/.asdf/installs/postgres/15.3
export POSTGRES_USER=plane
export POSTGRES_PASSWORD=plane
export POSTGRES_DB=plane
export PGDATA=/pg_data
export ROOT_DB_URL=postgresql://plane:plane@localhost:5432/postgres
export TZ=GMT
pwfile=$(mktemp)
echo -n $POSTGRES_PASSWORD > $pwfile
$pgdir/bin/initdb -U $POSTGRES_USER --pwfile=$pwfile --no-locale -L $pgdir/share/ || true
$pgdir/bin/postgres -c 'max_connections=1000'
EOSH
su postgres -c "/bin/bash $script"

View File

@ -1 +0,0 @@
longrun

View File

@ -1,2 +0,0 @@
#!/command/execlineb -P
/usr/bin/minio server /minio_data --console-address ":9090"

View File

@ -1 +0,0 @@
longrun

View File

@ -1,2 +0,0 @@
#!/command/execlineb -P
nginx -g "daemon off;"

View File

@ -1 +0,0 @@
longrun

View File

@ -1,2 +0,0 @@
#!/command/execlineb -P
/usr/bin/redis-server /etc/redis/redis.conf

View File

@ -1 +0,0 @@
longrun

View File

@ -1,4 +0,0 @@
#!/bin/bash
HOSTNAME=127.0.0.1 \
PORT=3001 \
node /space/server.js space

View File

@ -1 +0,0 @@
longrun

View File

@ -1,5 +0,0 @@
#!/bin/bash
NEXT_PUBLIC_DEPLOY_URL="http://localhost/spaces" \
HOSTNAME=127.0.0.1 \
PORT=3000 \
node /web/server.js web

View File

@ -1 +0,0 @@
longrun

57
fly.toml Normal file
View File

@ -0,0 +1,57 @@
# fly.toml app configuration file generated for lopiq-plane on 2024-04-13T19:01:18-05:00
#
# See https://fly.io/docs/reference/configuration/ for information about how to use this file.
#
app = 'lopiq-plane'
primary_region = 'ord'
[build]
image = 'git.orionkindel.com/thunderstrike/plane:stable-46b18dca279a49eeba91e4d99021fb95'
[env]
AWS_S3_BUCKET_NAME = 'uploads'
AWS_S3_ENDPOINT_URL = 'http://localhost:9000'
CORS_ALLOWED_ORIGINS = 'http://plane.lopiq.com'
DATABASE_URL = 'postgresql://plane:plane@localhost:5432/plane'
DEBUG = '0'
ENABLE_EMAIL_PASSWORD = '0'
ENABLE_MAGIC_LINK_LOGIN = '0'
ENABLE_SIGNUP = '0'
FILE_SIZE_LIMIT = '5242880'
GUNICORN_WORKERS = '2'
NEXT_PUBLIC_DEPLOY_WITH_NGINX = '1'
NEXT_PUBLIC_ENABLE_OAUTH = '0'
NEXT_PUBLIC_ENABLE_SENTRY = '0'
NEXT_PUBLIC_ENABLE_SESSION_RECORDER = '0'
NEXT_PUBLIC_TRACK_EVENTS = '0'
NGINX_PORT = '80'
PGDATABASE = 'plane'
PGHOST = 'localhost'
PGPASSWORD = 'plane'
PGUSER = 'plane'
POSTGRES_PASSWORD = 'plane'
POSTGRES_USER = 'plane'
REDIS_HOST = 'localhost'
REDIS_PORT = '6379'
REDIS_URL = 'redis://localhost:6379/'
SENTRY_ENVIRONMENT = 'development'
USE_MINIO = '1'
WEB_URL = 'http://plane.lopiq.com'
[[mounts]]
source = 'volume'
destination = '/ext'
[http_service]
internal_port = 80
force_https = true
auto_stop_machines = true
auto_start_machines = true
min_machines_running = 0
processes = ['app']
[[vm]]
memory = '1gb'
cpu_kind = 'shared'
cpus = 1