Skip to content

Commit 509d96c

Browse files
DragoryDragory
committed
refactor: new dev/prod containers
- Use a single Dockerfile for all Zeppelin services - Add a Dockerfile in project root that can be used by app hosting services - Provide a standalone and lightweight prod setup - Standalone is the same as the old setup, with mysql+nginx - Lightweight only runs bot+backend+dash, no mysql/nginx - Remove mounted mysql data folders for dev and prod - This resolves permission issues caused by the mount - The mysql service uses a regular named volume now - Simplify .env options and clearly separate different prod setups - Remove update.sh - Different setups require different update procedures, so a common update.sh no longer works
1 parent 730b8c1 commit 509d96c

30 files changed

+947
-273
lines changed

‎.devcontainer/devcontainer.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,5 +5,5 @@
55

66
"service": "devenv",
77
"remoteUser": "ubuntu",
8-
"workspaceFolder": "/home/ubuntu/zeppelin"
8+
"workspaceFolder": "/workspace/zeppelin"
99
}

‎.dockerignore

+93
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,97 @@
88

99
node_modules
1010
/backend/dist
11+
/shared/dist
1112
/dashboard/dist
13+
14+
# FROM GITIGNORE:
15+
16+
# Created by .ignore support plugin (hsz.mobi)
17+
### Node template
18+
# Logs
19+
/logs
20+
*.log
21+
npm-debug.log*
22+
yarn-debug.log*
23+
yarn-error.log*
24+
.clinic
25+
.clinic-bot
26+
.clinic-api
27+
28+
# Runtime data
29+
pids
30+
*.pid
31+
*.seed
32+
*.pid.lock
33+
34+
# Directory for instrumented libs generated by jscoverage/JSCover
35+
lib-cov
36+
37+
# Coverage directory used by tools like istanbul
38+
coverage
39+
40+
# nyc test coverage
41+
.nyc_output
42+
43+
# Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files)
44+
.grunt
45+
46+
# Bower dependency directory (https://bower.io/)
47+
bower_components
48+
49+
# node-waf configuration
50+
.lock-wscript
51+
52+
# Compiled binary addons (https://nodejs.org/api/addons.html)
53+
build/Release
54+
55+
# Dependency directories
56+
node_modules/
57+
jspm_packages/
58+
59+
# Typescript v1 declaration files
60+
typings/
61+
62+
# Optional npm cache directory
63+
.npm
64+
65+
# Optional eslint cache
66+
.eslintcache
67+
68+
# Optional REPL history
69+
.node_repl_history
70+
71+
# Output of 'npm pack'
72+
*.tgz
73+
74+
# Yarn Integrity file
75+
.yarn-integrity
76+
77+
# dotenv environment variables file
78+
*.env
79+
.env
80+
81+
# windows folder options
82+
desktop.ini
83+
84+
# PHPStorm
85+
.idea/
86+
87+
# Misc
88+
/convert.js
89+
/startscript.js
90+
.cache
91+
npm-ls.txt
92+
npm-audit.txt
93+
.vscode/launch.json
94+
95+
# Debug files
96+
*.debug.ts
97+
*.debug.js
98+
99+
.vscode/
100+
101+
config-errors.txt
102+
/config-schema.json
103+
104+
*.tsbuildinfo

‎.env.example

+43-34
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
# ==========================
2+
# GENERAL OPTIONS
3+
# ==========================
4+
15
# 32 character encryption key
26
KEY=
37

@@ -17,58 +21,63 @@ STAFF=
1721
# A comma-separated list of server IDs that should be allowed by default
1822
DEFAULT_ALLOWED_SERVERS=
1923

20-
# When using the Docker-based development environment, this is only used internally. The API will be available at localhost:DOCKER_DEV_WEB_PORT/api.
21-
API_PORT=3000
22-
2324
# Only required if relevant feature is used
2425
#PHISHERMAN_API_KEY=
2526

26-
# The user ID and group ID that should be used within the Docker containers
27-
# This should match your own user ID and group ID. Run `id -u` and `id -g` to find them.
28-
DOCKER_USER_UID=
29-
DOCKER_USER_GID=
3027

31-
#
32-
# DOCKER (DEVELOPMENT)
33-
# NOTE: You only need to fill in these values for running the development environment. See production config further below.
34-
#
28+
# ==========================
29+
# DEVELOPMENT
30+
# NOTE: You only need to fill in these values for running the development environment
31+
# ==========================
3532

36-
DOCKER_DEV_WEB_PORT=3300
33+
DEVELOPMENT_WEB_PORT=3300
3734

3835
# The MySQL database running in the container is exposed to the host on this port,
3936
# allowing access with database tools such as DBeaver
40-
DOCKER_DEV_MYSQL_PORT=3001
37+
DEVELOPMENT_MYSQL_PORT=3356
4138
# Password for the Zeppelin database user
42-
DOCKER_DEV_MYSQL_PASSWORD=
39+
DEVELOPMENT_MYSQL_PASSWORD=password
4340
# Password for the MySQL root user
44-
DOCKER_DEV_MYSQL_ROOT_PASSWORD=
41+
DEVELOPMENT_MYSQL_ROOT_PASSWORD=password
4542

4643
# The development environment container has an SSH server that you can connect to.
4744
# This is the port that server is exposed to the host on.
48-
DOCKER_DEV_SSH_PORT=3002
49-
DOCKER_DEV_SSH_PASSWORD=password
45+
DEVELOPMENT_SSH_PORT=3022
46+
DEVELOPMENT_SSH_PASSWORD=password
5047

5148
# If your user has a different UID than 1000, you might have to fill that in here to avoid permission issues
52-
#DOCKER_DEV_UID=1000
49+
#DEVELOPMENT_UID=1000
50+
51+
52+
# ==========================
53+
# PRODUCTION - STANDALONE
54+
# NOTE: You only need to fill in these values for running the standalone production environment
55+
# ==========================
56+
57+
STANDALONE_DOMAIN=
5358

54-
#
55-
# DOCKER (PRODUCTION)
56-
# NOTE: You only need to fill in these values for running the production environment. See development config above.
57-
#
59+
STANDALONE_WEB_PORT=443
5860

59-
DOCKER_PROD_DOMAIN=
60-
DOCKER_PROD_WEB_PORT=443
6161
# The MySQL database running in the container is exposed to the host on this port,
6262
# allowing access with database tools such as DBeaver
63-
DOCKER_PROD_MYSQL_PORT=3001
63+
STANDALONE_MYSQL_PORT=3356
6464
# Password for the Zeppelin database user
65-
DOCKER_PROD_MYSQL_PASSWORD=
65+
STANDALONE_MYSQL_PASSWORD=
6666
# Password for the MySQL root user
67-
DOCKER_PROD_MYSQL_ROOT_PASSWORD=
68-
69-
# You only need to set these if you're running an external database.
70-
# In a standard setup, the database is run in a docker container.
71-
#DB_HOST=
72-
#DB_USER=
73-
#DB_PASSWORD=
74-
#DB_DATABASE=
67+
STANDALONE_MYSQL_ROOT_PASSWORD=
68+
69+
70+
# ==========================
71+
# PRODUCTION - LIGHTWEIGHT
72+
# NOTE: You only need to fill in these values for running the lightweight production environment
73+
# ==========================
74+
75+
# Ports where the API/dashboard are exposed on the host
76+
LIGHTWEIGHT_API_PORT=3001
77+
LIGHTWEIGHT_DASHBOARD_PORT=3002
78+
79+
LIGHTWEIGHT_DB_HOST=
80+
LIGHTWEIGHT_DB_PORT=
81+
LIGHTWEIGHT_DB_USER=
82+
LIGHTWEIGHT_DB_PASSWORD=
83+
LIGHTWEIGHT_DB_DATABASE=

‎.gitignore

+4
Original file line numberDiff line numberDiff line change
@@ -87,3 +87,7 @@ config-errors.txt
8787
/config-schema.json
8888

8989
*.tsbuildinfo
90+
91+
# Legacy data folders
92+
/docker/development/data
93+
/docker/production/data

‎Dockerfile

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
FROM node:20
2+
USER node
3+
4+
COPY --chown=node:node . /zeppelin
5+
6+
# Install dependencies for all packages
7+
WORKDIR /zeppelin
8+
RUN npm ci
9+
10+
# Build backend
11+
WORKDIR /zeppelin/backend
12+
RUN npm run build
13+
14+
# Build dashboard
15+
WORKDIR /zeppelin/dashboard
16+
RUN npm run build
17+
18+
# Prune dev dependencies
19+
WORKDIR /zeppelin
20+
RUN npm prune --production

‎PRODUCTION.md

+55-22
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,67 @@
11
# Zeppelin production environment
2-
Zeppelin's production environment - that is, the **bot, API, and dashboard** - uses Docker.
2+
Zeppelin's production environment uses Docker. There are a few different ways to run Zeppelin in production:
33

4-
## Starting the production environment
4+
1. **Standalone**
5+
* The easiest way to get Zeppelin up and running. This setup comes with a built-in database and web server.
6+
2. **Lightweight**
7+
* In case you don't want to use the built-in database and web server. This setup only runs the bot, API, and dashboard themselves. You'll have to provide your own database connection options and set up a proxy server for the API and dashboard.
8+
3. **Manual**
9+
* If you only want to run a specific service, you can use Zeppelin's Dockerfile directly.
10+
11+
## Standalone
12+
13+
### Setup
514
1. Install Docker on the machine running the bot
615
2. Make a copy of `.env.example` called `.env`
7-
3. Fill in the missing values in `.env`
8-
4. Run `docker compose -f docker-compose.production.yml build`
9-
5. Run `docker compose -f docker-compose.production.yml up -d`
16+
3. Fill in the missing values in `.env` (including the "PRODUCTION - STANDALONE" section)
17+
18+
**Note:** The dashboard and API are exposed with a self-signed certificate. It is recommended to set up a proxy with a proper certificate in front of them. A popular option for this is [Cloudflare Tunnel](https://www.cloudflare.com/products/tunnel/).
19+
20+
### Running the bot
21+
`docker compose -f docker-compose.standalone.yml up -d`
22+
23+
### Shutting the bot down
24+
`docker compose -f docker-compose.standalone.yml down`
25+
26+
### Updating the bot
27+
1. Shut the bot down
28+
2. Update the files (e.g. `git pull`)
29+
3. Rebuild: `docker compose -f docker-compose.standalone.yml build`
30+
4. Run the bot again
31+
32+
### Viewing logs
33+
`docker compose -f docker-compose.standalone.yml logs -t -f`
34+
35+
## Lightweight
1036

11-
**Note:** The dashboard and API are exposed with a self-signed certificate. It is recommended to set up a proxy with a proper certificate in front of them. Cloudflare is a popular choice here.
37+
### Setup
38+
1. Install Docker on the machine running the bot
39+
2. Make a copy of `.env.example` called `.env`
40+
3. Fill in the missing values in `.env` (including the "PRODUCTION - LIGHTWEIGHT" section)
1241

13-
## Updating the bot
42+
### Running the bot
43+
`docker compose -f docker-compose.lightweight.yml up -d`
1444

15-
### One-click script
16-
If you've downloaded the bot's files by cloning the git repository, you can use `update.sh` to update the bot.
45+
### Shutting the bot down
46+
`docker compose -f docker-compose.lightweight.yml down`
1747

18-
### Manual instructions
19-
1. Shut the bot down: `docker compose -f docker-compose.production.yml down`
48+
### Updating the bot
49+
1. Shut the bot down
2050
2. Update the files (e.g. `git pull`)
21-
3. Build new images: `docker compose -f docker-compose.production.yml build`
22-
3. Start the bot again: `docker compose -f docker-compose.production.yml up -d`
51+
3. Rebuild: `docker compose -f docker-compose.lightweight.yml build`
52+
4. Run the bot again
53+
54+
### Viewing logs
55+
`docker compose -f docker-compose.lightweight.yml logs -t -f`
2356

24-
### Ephemeral hotfixes
25-
If you need to make a hotfix to the bot's source files directly on the server:
26-
1. Shut the bot down: `docker compose -f docker-compose.production.yml down`
27-
2. Make your edits
28-
3. Build new images: `docker compose -f docker-compose.production.yml build`
29-
4. Start the bot again: `docker compose -f docker-compose.production.yml up -d`
57+
## Manual
58+
1. Build the Zeppelin image: `docker build --tag 'zeppelin' .`
59+
2. Run the service:
60+
* Bot: `docker run zeppelin npm run start-bot`
61+
* API: `docker run zeppelin npm run start-api`
62+
* Dashboard: `docker run zeppelin npm run start-dashboard`
3063

31-
Make sure to revert any hotfixes before updating the bot normally.
64+
If you're using an application platform such as Railway, you can simply point it to Zeppelin's repository and it should pick up the Dockerfile from there.
65+
For the start command, you can use the same commands as above: `npm run start-bot`, `npm run start-api`, `npm run start-dashboard`.
3266

33-
## View logs
34-
To view real-time logs, run `docker compose -f docker-compose.production.yml logs -t -f`
67+
**Note:** You'll need to provide the necessary env variables. For example, `docker run --env-file .env zeppelin`

‎backend/package.json

+10-10
Original file line numberDiff line numberDiff line change
@@ -4,28 +4,28 @@
44
"description": "",
55
"private": true,
66
"scripts": {
7-
"watch": "cross-env NODE_ENV=development tsc-watch --onSuccess \"node start-dev.js\"",
8-
"watch-yaml-parse-test": "cross-env NODE_ENV=development tsc-watch --onSuccess \"node dist/backend/src/yamlParseTest.js\"",
7+
"watch": "NODE_ENV=development HOST_MODE=development tsc-watch --onSuccess \"node start-dev.js\"",
8+
"watch-yaml-parse-test": "NODE_ENV=development HOST_MODE=development tsc-watch --onSuccess \"node dist/backend/src/yamlParseTest.js\"",
99
"build": "tsc --build",
10-
"start-bot-dev": "cross-env NODE_ENV=development node --enable-source-maps --stack-trace-limit=30 --inspect=0.0.0.0:9229 dist/backend/src/index.js",
10+
"start-bot-dev": "NODE_ENV=development HOST_MODE=development node --enable-source-maps --stack-trace-limit=30 --inspect=0.0.0.0:9229 dist/backend/src/index.js",
1111
"start-bot-dev-debug": "NODE_ENV=development DEBUG=true clinic heapprofiler --collect-only --dest .clinic-bot -- node --enable-source-maps --stack-trace-limit=30 --inspect=0.0.0.0:9229 dist/backend/src/index.js",
1212
"start-bot-prod": "cross-env NODE_ENV=production node --enable-source-maps --stack-trace-limit=30 dist/backend/src/index.js",
1313
"start-bot-prod-debug": "NODE_ENV=production DEBUG=true clinic heapprofiler --collect-only --dest .clinic-bot -- node --enable-source-maps --stack-trace-limit=30 dist/backend/src/index.js",
14-
"watch-bot": "cross-env NODE_ENV=development tsc-watch --onSuccess \"npm run start-bot-dev\"",
15-
"start-api-dev": "cross-env NODE_ENV=development node --enable-source-maps --stack-trace-limit=30 --inspect=0.0.0.0:9239 dist/backend/src/api/index.js",
14+
"watch-bot": "NODE_ENV=development HOST_MODE=development tsc-watch --onSuccess \"npm run start-bot-dev\"",
15+
"start-api-dev": "NODE_ENV=development HOST_MODE=development node --enable-source-maps --stack-trace-limit=30 --inspect=0.0.0.0:9239 dist/backend/src/api/index.js",
1616
"start-api-dev-debug": "NODE_ENV=development DEBUG=true clinic heapprofiler --collect-only --dest .clinic-api -- node --enable-source-maps --stack-trace-limit=30 --inspect=0.0.0.0:9239 dist/backend/src/api/index.js",
1717
"start-api-prod": "cross-env NODE_ENV=production node --enable-source-maps --stack-trace-limit=30 dist/backend/src/api/index.js",
1818
"start-api-prod-debug": "NODE_ENV=production DEBUG=true clinic heapprofiler --collect-only --dest .clinic-api -- node --enable-source-maps --stack-trace-limit=30 dist/backend/src/api/index.js",
19-
"watch-api": "cross-env NODE_ENV=development tsc-watch --onSuccess \"npm run start-api-dev\"",
19+
"watch-api": "NODE_ENV=development HOST_MODE=development tsc-watch --onSuccess \"npm run start-api-dev\"",
2020
"typeorm": "node ./node_modules/typeorm/cli.js",
2121
"migrate": "npm run typeorm -- migration:run -d dist/backend/src/data/dataSource.js",
2222
"migrate-prod": "cross-env NODE_ENV=production npm run migrate",
23-
"migrate-dev": "cross-env NODE_ENV=development npm run build && npm run migrate",
23+
"migrate-dev": "NODE_ENV=development HOST_MODE=development npm run build && npm run migrate",
2424
"migrate-rollback": "npm run typeorm -- migration:revert -d dist/backend/src/data/dataSource.js",
2525
"migrate-rollback-prod": "cross-env NODE_ENV=production npm run migrate",
26-
"migrate-rollback-dev": "cross-env NODE_ENV=development npm run build && npm run migrate",
27-
"validate-active-configs": "cross-env NODE_ENV=development node --enable-source-maps dist/backend/src/validateActiveConfigs.js > ../config-errors.txt",
28-
"export-config-json-schema": "cross-env NODE_ENV=development node --enable-source-maps dist/backend/src/exportSchemas.js > ../config-schema.json",
26+
"migrate-rollback-dev": "NODE_ENV=development HOST_MODE=development npm run build && npm run migrate",
27+
"validate-active-configs": "NODE_ENV=development HOST_MODE=development node --enable-source-maps dist/backend/src/validateActiveConfigs.js > ../config-errors.txt",
28+
"export-config-json-schema": "NODE_ENV=development HOST_MODE=development node --enable-source-maps dist/backend/src/exportSchemas.js > ../config-schema.json",
2929
"test": "npm run build && npm run run-tests",
3030
"run-tests": "ava",
3131
"test-watch": "tsc-watch --onSuccess \"npx ava\""

‎backend/src/api/start.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,7 @@ app.use((req, res, next) => {
5151
return notFound(res);
5252
});
5353

54-
const port = env.API_PORT;
54+
const port = 3001;
5555
app.listen(port, "0.0.0.0", () => console.log(`API server listening on port ${port}`)); // tslint:disable-line
5656

5757
startBackgroundTasks();

0 commit comments

Comments
 (0)