From 985660be398c3cd8300cde4522e384eb171d2d23 Mon Sep 17 00:00:00 2001 From: Manish Gupta Date: Mon, 27 May 2024 17:48:15 +0530 Subject: [PATCH] finalize the dockerfiles with volume exposure --- aio/Dockerfile | 87 +++++++++++++++++--------------- aio/Dockerfile.base | 18 +++---- aio/aio.sh | 30 +++++++++++ aio/nginx.conf.aio | 73 +++++++++++++++++++++++++++ aio/pg-setup.sh | 14 ++++++ aio/postgresql.conf | 12 +++++ aio/supervisord.conf | 117 +++++++++++++++++++++++++------------------ apiserver/bin/aio.sh | 18 ------- 8 files changed, 252 insertions(+), 117 deletions(-) create mode 100644 aio/aio.sh create mode 100644 aio/nginx.conf.aio create mode 100644 aio/pg-setup.sh create mode 100644 aio/postgresql.conf delete mode 100644 apiserver/bin/aio.sh diff --git a/aio/Dockerfile b/aio/Dockerfile index 1f513950f17..775af811556 100644 --- a/aio/Dockerfile +++ b/aio/Dockerfile @@ -63,23 +63,50 @@ WORKDIR /app SHELL [ "/bin/bash", "-c" ] +# PYTHON APPLICATION SETUP + +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 +ENV PIP_DISABLE_PIP_VERSION_CHECK=1 + +COPY apiserver/requirements.txt ./api/ +COPY apiserver/requirements ./api/requirements + +RUN python3.12 -m venv /app/venv && \ + source /app/venv/bin/activate && \ + /app/venv/bin/pip install --upgrade pip && \ + /app/venv/bin/pip install -r ./api/requirements.txt --compile --no-cache-dir + +# Add in Django deps and generate Django's static files +COPY apiserver/manage.py ./api/manage.py +COPY apiserver/plane ./api/plane/ +COPY apiserver/templates ./api/templates/ +COPY package.json ./api/package.json + +COPY apiserver/bin ./api/bin/ + +RUN chmod +x ./api/bin/* +RUN chmod -R 777 ./api/ + +# NEXTJS BUILDS + COPY --from=installer /app/web/next.config.js ./web/ COPY --from=installer /app/web/package.json ./web/ COPY --from=installer /app/web/.next/standalone ./web -COPY --from=installer /app/web/.next/static ./web/.next/static -COPY --from=installer /app/web/public ./web/public +COPY --from=installer /app/web/.next/static ./web/web/.next/static +COPY --from=installer /app/web/public ./web/web/public COPY --from=installer /app/space/next.config.js ./space/ COPY --from=installer /app/space/package.json ./space/ COPY --from=installer /app/space/.next/standalone ./space -COPY --from=installer /app/space/.next/static ./space/.next/static -COPY --from=installer /app/space/public ./space/public +COPY --from=installer /app/space/.next/static ./space/space/.next/static +COPY --from=installer /app/space/public ./space/space/public COPY --from=installer /app/admin/next.config.js ./admin/ COPY --from=installer /app/admin/package.json ./admin/ COPY --from=installer /app/admin/.next/standalone ./admin -COPY --from=installer /app/admin/.next/static ./admin/.next/static -COPY --from=installer /app/admin/public ./admin/public +COPY --from=installer /app/admin/.next/static ./admin/admin/.next/static +COPY --from=installer /app/admin/public ./admin/admin/public ARG NEXT_PUBLIC_API_BASE_URL="" ENV NEXT_PUBLIC_API_BASE_URL=$NEXT_PUBLIC_API_BASE_URL @@ -102,45 +129,23 @@ ENV NEXT_PUBLIC_WEB_BASE_URL=$NEXT_PUBLIC_WEB_BASE_URL ENV NEXT_TELEMETRY_DISABLED 1 ENV TURBO_TELEMETRY_DISABLED 1 +COPY aio/supervisord.conf /app/supervisord.conf -# set environment variables -ENV PYTHONDONTWRITEBYTECODE 1 -ENV PYTHONUNBUFFERED 1 -ENV PIP_DISABLE_PIP_VERSION_CHECK=1 - -COPY apiserver/requirements.txt ./api/ -COPY apiserver/requirements ./api/requirements - -# RUN python3.12 -m pip install --upgrade pip - -# RUN echo "alias python=/usr/local/bin/python3.12" >> ~/.bashrc && \ -# echo "alias pip=/usr/local/bin/pip3.12" >> ~/.bashrc - -RUN pip install -r ./api/requirements.txt --compile --no-cache-dir -# RUN apt-get install python3.12-venv -y - -# RUN python3 -m venv /app/venv && \ -# source /app/venv/bin/activate && \ -# /app/venv/bin/pip install --upgrade pip && \ -# /app/venv/bin/pip install -r ./api/requirements.txt --compile --no-cache-dir - -# Add in Django deps and generate Django's static files -COPY apiserver/manage.py ./api/manage.py -COPY apiserver/plane ./api/plane/ -COPY apiserver/templates ./api/templates/ - -COPY apiserver/bin ./api/bin/ +COPY aio/aio.sh /app/aio.sh +RUN chmod +x /app/aio.sh -RUN chmod +x ./api/bin/* -RUN chmod -R 777 ./api/ +COPY aio/pg-setup.sh /app/pg-setup.sh +RUN chmod +x /app/pg-setup.sh -COPY aio/supervisord.conf /app/supervisord.conf +COPY deploy/selfhost/variables.env /app/plane.env -RUN chown -R app-user:app-user /app +# NGINX Conf Copy +COPY ./aio/nginx.conf.aio /etc/nginx/nginx.conf.template +COPY ./nginx/env.sh /app/nginx-start.sh +RUN chmod +x /app/nginx-start.sh -USER app-user +RUN ./pg-setup.sh -# RUN echo "alias python=/usr/local/bin/python3.12" >> ~/.bashrc && \ -# echo "alias pip=/usr/local/bin/pip3.12" >> ~/.bashrc +VOLUME [ "/app/data/minio/uploads", "/var/lib/postgresql/data" ] -CMD ["sudo", "/usr/bin/supervisord", "-c", "/app/supervisord.conf"] \ No newline at end of file +CMD ["/usr/bin/supervisord", "-c", "/app/supervisord.conf"] diff --git a/aio/Dockerfile.base b/aio/Dockerfile.base index 58b9a7fdd62..092deb79723 100644 --- a/aio/Dockerfile.base +++ b/aio/Dockerfile.base @@ -23,10 +23,13 @@ RUN echo "deb http://deb.debian.org/debian $(lsb_release -cs)-backports main" > apt-get install -y redis-server # Install PostgreSQL 15 +ENV POSTGRES_VERSION 15 RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /usr/share/keyrings/pgdg-archive-keyring.gpg && \ echo "deb [signed-by=/usr/share/keyrings/pgdg-archive-keyring.gpg] http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list && \ apt-get update && \ - apt-get install -y postgresql-15 postgresql-client-15 + apt-get install -y postgresql-$POSTGRES_VERSION postgresql-client-$POSTGRES_VERSION && \ + mkdir -p /var/lib/postgresql/data && \ + chown -R postgres:postgres /var/lib/postgresql # Install MinIO ARG TARGETARCH @@ -73,16 +76,11 @@ RUN mkdir -p /app/{data,logs} && \ COPY supervisord.base /app/supervisord.conf RUN apt-get update && \ - apt-get install -y sudo lsof net-tools libpq-dev && \ + apt-get install -y sudo lsof net-tools libpq-dev procps gettext && \ apt-get clean -# Add a new user 'app-user' and give sudo access -RUN useradd -m -s /bin/bash app-user && \ - echo 'app-user ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers - -RUN chown -R app-user:app-user /app - -USER app-user +RUN sudo -u postgres /usr/lib/postgresql/$POSTGRES_VERSION/bin/initdb -D /var/lib/postgresql/data +COPY postgresql.conf /etc/postgresql/postgresql.conf RUN echo "alias python=/usr/local/bin/python3.12" >> ~/.bashrc && \ echo "alias pip=/usr/local/bin/pip3.12" >> ~/.bashrc @@ -91,4 +89,4 @@ RUN echo "alias python=/usr/local/bin/python3.12" >> ~/.bashrc && \ EXPOSE 6379 5432 9000 80 # Start Supervisor -CMD ["sudo", "/usr/bin/supervisord", "-c", "/app/supervisord.conf"] +CMD ["/usr/bin/supervisord", "-c", "/app/supervisord.conf"] diff --git a/aio/aio.sh b/aio/aio.sh new file mode 100644 index 00000000000..53adbf42b6d --- /dev/null +++ b/aio/aio.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -e + + +if [ "$1" = 'api' ]; then + source /app/venv/bin/activate + cd /app/api + exec ./bin/docker-entrypoint-api.sh +elif [ "$1" = 'worker' ]; then + source /app/venv/bin/activate + cd /app/api + exec ./bin/docker-entrypoint-worker.sh +elif [ "$1" = 'beat' ]; then + source /app/venv/bin/activate + cd /app/api + exec ./bin/docker-entrypoint-beat.sh +elif [ "$1" = 'migrator' ]; then + source /app/venv/bin/activate + cd /app/api + exec ./bin/docker-entrypoint-migrator.sh +elif [ "$1" = 'web' ]; then + node /app/web/web/server.js +elif [ "$1" = 'space' ]; then + node /app/space/space/server.js +elif [ "$1" = 'admin' ]; then + node /app/admin/admin/server.js +else + echo "Command not found" + exit 1 +fi \ No newline at end of file diff --git a/aio/nginx.conf.aio b/aio/nginx.conf.aio new file mode 100644 index 00000000000..1a1f3c0b829 --- /dev/null +++ b/aio/nginx.conf.aio @@ -0,0 +1,73 @@ +events { +} + +http { + sendfile on; + + server { + listen 80; + root /www/data/; + access_log /var/log/nginx/access.log; + + client_max_body_size ${FILE_SIZE_LIMIT}; + + add_header X-Content-Type-Options "nosniff" always; + add_header Referrer-Policy "no-referrer-when-downgrade" always; + add_header Permissions-Policy "interest-cohort=()" always; + add_header Strict-Transport-Security "max-age=31536000; includeSubDomains" always; + add_header X-Forwarded-Proto "${dollar}scheme"; + add_header X-Forwarded-Host "${dollar}host"; + add_header X-Forwarded-For "${dollar}proxy_add_x_forwarded_for"; + add_header X-Real-IP "${dollar}remote_addr"; + + location / { + proxy_http_version 1.1; + proxy_set_header Upgrade ${dollar}http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host ${dollar}http_host; + proxy_pass http://localhost:3001/; + } + + location /spaces/ { + rewrite ^/spaces/?$ /spaces/login break; + proxy_http_version 1.1; + proxy_set_header Upgrade ${dollar}http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host ${dollar}http_host; + proxy_pass http://localhost:3002/spaces/; + } + + + location /god-mode/ { + proxy_http_version 1.1; + proxy_set_header Upgrade ${dollar}http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host ${dollar}http_host; + proxy_pass http://localhost:3003/god-mode/; + } + + location /api/ { + proxy_http_version 1.1; + proxy_set_header Upgrade ${dollar}http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host ${dollar}http_host; + proxy_pass http://localhost:8000/api/; + } + + location /auth/ { + proxy_http_version 1.1; + proxy_set_header Upgrade ${dollar}http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host ${dollar}http_host; + proxy_pass http://localhost:8000/auth/; + } + + location /${BUCKET_NAME}/ { + proxy_http_version 1.1; + proxy_set_header Upgrade ${dollar}http_upgrade; + proxy_set_header Connection "upgrade"; + proxy_set_header Host ${dollar}http_host; + proxy_pass http://localhost:9000/uploads/; + } + } +} diff --git a/aio/pg-setup.sh b/aio/pg-setup.sh new file mode 100644 index 00000000000..2035f225d71 --- /dev/null +++ b/aio/pg-setup.sh @@ -0,0 +1,14 @@ +#!/bin/bash + + +# Variables +set -o allexport +source plane.env set +set +o allexport + +PGHOST=localhost + +sudo -u postgres /usr/lib/postgresql/${POSTGRES_VERSION}/bin/pg_ctl -D /var/lib/postgresql/data start +sudo -u postgres /usr/lib/postgresql/${POSTGRES_VERSION}/bin/psql --command "CREATE USER $POSTGRES_USER WITH SUPERUSER PASSWORD '$POSTGRES_PASSWORD';" && \ +sudo -u postgres /usr/lib/postgresql/${POSTGRES_VERSION}/bin/createdb -O $POSTGRES_USER $POSTGRES_DB && \ +sudo -u postgres /usr/lib/postgresql/${POSTGRES_VERSION}/bin/pg_ctl -D /var/lib/postgresql/data stop diff --git a/aio/postgresql.conf b/aio/postgresql.conf new file mode 100644 index 00000000000..8c6223fc464 --- /dev/null +++ b/aio/postgresql.conf @@ -0,0 +1,12 @@ +# PostgreSQL configuration file + +# Allow connections from any IP address +listen_addresses = '*' + +# Set the maximum number of connections +max_connections = 100 + +# Set the shared buffers size +shared_buffers = 128MB + +# Other custom configurations can be added here diff --git a/aio/supervisord.conf b/aio/supervisord.conf index a7456cad53d..46ef1b4faed 100644 --- a/aio/supervisord.conf +++ b/aio/supervisord.conf @@ -1,94 +1,115 @@ [supervisord] user=root nodaemon=true -stdout_logfile=/app/logs/access/supervisor.log -stderr_logfile=/app/logs/error/supervisor.err.log +priority=1 +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stdout +stderr_logfile_maxbytes=0 [program:redis] directory=/app/data/redis command=redis-server autostart=true autorestart=true -stdout_logfile=/app/logs/access/redis.log -stderr_logfile=/app/logs/error/redis.err.log +priority=1 +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stdout +stderr_logfile_maxbytes=0 [program:postgresql] user=postgres -command=/usr/lib/postgresql/15/bin/postgres --config-file=/etc/postgresql/15/main/postgresql.conf +command=/usr/lib/postgresql/15/bin/postgres -D /var/lib/postgresql/data --config-file=/etc/postgresql/postgresql.conf autostart=true autorestart=true -stdout_logfile=/app/logs/access/postgresql.log -stderr_logfile=/app/logs/error/postgresql.err.log +priority=1 +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stdout +stderr_logfile_maxbytes=0 [program:minio] directory=/app/data/minio command=minio server /app/data/minio autostart=true autorestart=true +priority=1 stdout_logfile=/app/logs/access/minio.log stderr_logfile=/app/logs/error/minio.err.log [program:nginx] -directory=/app/data/nginx -command=/usr/sbin/nginx -g 'daemon off;' +command=/app/nginx-start.sh autostart=true autorestart=true +priority=1 stdout_logfile=/app/logs/access/nginx.log stderr_logfile=/app/logs/error/nginx.err.log [program:web] -user=app-user -command=node /app/web/web/server.js +command=/app/aio.sh web autostart=true autorestart=true -stdout_logfile=/app/logs/access/web.log -stderr_logfile=/app/logs/error/web_err.log -environment=PORT=3001 +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stdout +stderr_logfile_maxbytes=0 +environment=PORT=3001,HOSTNAME=0.0.0.0 [program:space] -user=app-user -command=node /app/space/space/server.js +command=/app/aio.sh space autostart=true autorestart=true -stdout_logfile=/app/logs/access/space.log -stderr_logfile=/app/logs/error/space_err.log -environment=PORT=3002 +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stdout +stderr_logfile_maxbytes=0 +environment=PORT=3002,HOSTNAME=0.0.0.0 [program:admin] -user=app-user -command=node /app/admin/admin/server.js +command=/app/aio.sh admin autostart=true autorestart=true -stdout_logfile=/app/logs/access/admin.log -stderr_logfile=/app/logs/error/admin_err.log -environment=PORT=3003 +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stdout +stderr_logfile_maxbytes=0 +environment=PORT=3003,HOSTNAME=0.0.0.0 -# [program:migrator] -# command=/app/api/bin/aio.sh migrator -# autostart=true -# autorestart=false -# stdout_logfile=/app/logs/access/migrator.log -# stderr_logfile=/app/logs/error/migrator_err.log +[program:migrator] +command=/app/aio.sh migrator +autostart=true +autorestart=false +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stdout +stderr_logfile_maxbytes=0 -# [program:api] -# command=/app/api/bin/aio.sh api -# autostart=true -# autorestart=true -# stdout_logfile=/app/logs/access/api.log -# stderr_logfile=/app/logs/error/api_err.log +[program:api] +command=/app/aio.sh api +autostart=true +autorestart=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stdout +stderr_logfile_maxbytes=0 -# [program:worker] -# command=/app/api/bin/aio.sh worker -# autostart=true -# autorestart=true -# stdout_logfile=/app/logs/access/worker.log -# stderr_logfile=/app/logs/error/worker_err.log +[program:worker] +command=/app/aio.sh worker +autostart=true +autorestart=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stdout +stderr_logfile_maxbytes=0 -# [program:beat] -# command=/app/api/bin/aio.sh beat -# autostart=true -# autorestart=true -# stdout_logfile=/app/logs/access/beat.log -# stderr_logfile=/app/logs/error/beat_err.log +[program:beat] +command=/app/aio.sh beat +autostart=true +autorestart=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stdout +stderr_logfile_maxbytes=0 diff --git a/apiserver/bin/aio.sh b/apiserver/bin/aio.sh deleted file mode 100644 index 3aaaada4d89..00000000000 --- a/apiserver/bin/aio.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -set -e - -source /app/venv/bin/activate -cd /app/api - -if [ "$1" = 'api' ]; then - source /app/api/bin/docker-entrypoint-api.sh -elif [ "$1" = 'worker' ]; then - source /app/api/bin/docker-entrypoint-worker.sh -elif [ "$1" = 'beat' ]; then - source /app/api/bin/docker-entrypoint-beat.sh -elif [ "$1" = 'migrator' ]; then - source /app/api/bin/docker-entrypoint-migrator.sh -else - echo "Command not found" - exit 1 -fi \ No newline at end of file