diff --git a/.github/actions/checkout/action.yml b/.github/actions/checkout/action.yml new file mode 100644 index 000000000..741ff786f --- /dev/null +++ b/.github/actions/checkout/action.yml @@ -0,0 +1,17 @@ +name: Checkout files action +inputs: + ref: + required: false + type: string + repository: + required: false + type: string + +runs: + using: "composite" + steps: + - name: Checkout files + uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + repository: ${{ inputs.repository }} diff --git a/.github/actions/docker_build_and_push/action.yml b/.github/actions/docker_build_and_push/action.yml new file mode 100644 index 000000000..2244e5138 --- /dev/null +++ b/.github/actions/docker_build_and_push/action.yml @@ -0,0 +1,47 @@ +name: Docker build and push action +inputs: + dockerhub_username: + required: true + type: string + dockerhub_token: + required: true + type: string + context: + required: true + type: string + file: + required: true + type: string + tags: + required: true + type: string + push: + required: true + type: boolean + +runs: + using: "composite" + steps: + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to DockerHub + uses: docker/login-action@v3 + with: + username: ${{ inputs.dockerhub_username }} + password: ${{ inputs.dockerhub_token }} + + - name: Build and push + id: docker_build + uses: docker/build-push-action@v5 + with: + context: ${{ inputs.context }}} + file: ${{ inputs.file }} + push: ${{ inputs.push }} + tags: ${{ inputs.tags }} + + - name: Image digest + run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/.github/actions/scp_action/action.yml b/.github/actions/scp_action/action.yml new file mode 100644 index 000000000..7ddfe9958 --- /dev/null +++ b/.github/actions/scp_action/action.yml @@ -0,0 +1,29 @@ +name: Copy files action +inputs: + host: + required: true + type: string + username: + required: true + type: string + key: + required: true + type: string + source: + required: true + type: string + target: + required: true + type: string + +runs: + using: "composite" + steps: + - name: Copy file via scp + uses: appleboy/scp-action@master + with: + host: ${{ inputs.host }} + username: ${{ inputs.username }} + key: ${{ inputs.key }} + source: ${{ inputs.source }} + target: ${{ inputs.target }} diff --git a/.github/actions/ssh_action/action.yml b/.github/actions/ssh_action/action.yml new file mode 100644 index 000000000..85ae8c647 --- /dev/null +++ b/.github/actions/ssh_action/action.yml @@ -0,0 +1,25 @@ +name: Execute remote command action +inputs: + host: + required: true + type: string + username: + required: true + type: string + key: + required: true + type: string + script: + required: true + type: string + +runs: + using: "composite" + steps: + - name: Executing remote command + uses: appleboy/ssh-action@master + with: + host: ${{ inputs.host }} + username: ${{ inputs.username }} + key: ${{ inputs.key }} + script: ${{ inputs.script }} diff --git a/.github/workflows/build_deploy_backend.yml b/.github/workflows/build_deploy_backend.yml index f7383c444..f426a76e9 100644 --- a/.github/workflows/build_deploy_backend.yml +++ b/.github/workflows/build_deploy_backend.yml @@ -13,64 +13,32 @@ on: workflow_dispatch: jobs: - build: + build_and_push: runs-on: ubuntu-latest + strategy: + matrix: + service: ['web', 'celery', 'media'] steps: - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - - name: Login to DockerHub - uses: docker/login-action@v1 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Checkout files - uses: actions/checkout@v2 - - - name: Build and push django api - id: docker_build_web - uses: docker/build-push-action@v2 - with: - context: ./zubhub_backend/ - file: ./zubhub_backend/compose/web/prod/Dockerfile - push: true - tags: unstructuredstudio/zubhub-services_web:latest - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} - - - name: Build and push celery worker - id: docker_build_celery - uses: docker/build-push-action@v2 - with: - context: ./zubhub_backend/ - file: ./zubhub_backend/compose/celery/prod/Dockerfile - push: true - tags: unstructuredstudio/zubhub-services_celery:latest - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} + - uses: unstructuredstudio/zubhub/.github/actions/checkout@master - - name: Build and push media service - id: docker_build_media - uses: docker/build-push-action@v2 + - name: Build and push ${{ matrix.service }} + uses: unstructuredstudio/zubhub/.github/actions/docker_build_and_push@master with: + dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }} + dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }} context: ./zubhub_backend/ - file: ./zubhub_backend/compose/media/prod/Dockerfile + file: ./zubhub_backend/compose/${{ matrix.service }}/prod/Dockerfile push: true - tags: unstructuredstudio/zubhub-services_media:latest - - name: Image digest - run: echo ${{ steps.docker_build.outputs.digest }} + tags: unstructuredstudio/zubhub-services_${{ matrix.service }}:latest deploy: - needs: build + needs: build_and_push runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: unstructuredstudio/zubhub/.github/actions/checkout@master - - name: Copy file via scp - uses: appleboy/scp-action@master + - uses: unstructuredstudio/zubhub/.github/actions/scp_action@master with: host: ${{ secrets.DO_BACKEND_HOST }} username: ${{ secrets.DO_BACKEND_USERNAME }} @@ -78,8 +46,7 @@ jobs: source: "." target: "/home/zubhub-services/zubhub" - - name: Executing remote command - uses: appleboy/ssh-action@master + - uses: unsctructuredstudio/zubhub/.github/actions/ssh_action@master with: host: ${{ secrets.DO_BACKEND_HOST }} username: ${{ secrets.DO_BACKEND_USERNAME }} diff --git a/.github/workflows/build_deploy_docs.yml b/.github/workflows/build_deploy_docs.yml index fdc5b9674..bdab90f65 100644 --- a/.github/workflows/build_deploy_docs.yml +++ b/.github/workflows/build_deploy_docs.yml @@ -11,8 +11,8 @@ jobs: deploy: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-python@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: python-version: "3.x" - run: | diff --git a/.github/workflows/build_deploy_frontend.yml b/.github/workflows/build_deploy_frontend.yml index ea07d4fc1..ee047492d 100644 --- a/.github/workflows/build_deploy_frontend.yml +++ b/.github/workflows/build_deploy_frontend.yml @@ -17,21 +17,21 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Checkout files - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Build and push id: docker_build - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: context: ./zubhub_frontend/zubhub/ file: ./zubhub_frontend/zubhub/Dockerfile.prod @@ -45,7 +45,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v4 - name: Copy file via scp uses: appleboy/scp-action@master diff --git a/.github/workflows/build_locust.yml b/.github/workflows/build_locust.yml index ea70921ed..41ff94655 100644 --- a/.github/workflows/build_locust.yml +++ b/.github/workflows/build_locust.yml @@ -23,21 +23,21 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Checkout files - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Build and push locust id: docker_build_locust - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: context: ./locust/ file: ./locust/Dockerfile diff --git a/.github/workflows/create_destroy_test_vm.yaml b/.github/workflows/create_destroy_test_vm.yaml index 9c63b1277..30c92dfa6 100644 --- a/.github/workflows/create_destroy_test_vm.yaml +++ b/.github/workflows/create_destroy_test_vm.yaml @@ -47,7 +47,7 @@ jobs: # See https://stackoverflow.com/questions/74957218/what-is-the-difference-between-pull-request-and-pull-request-target-event-in-git # See https://dev.to/suzukishunsuke/secure-github-actions-by-pullrequesttarget-641#:~:text=pull_request_target%20is%20one%20of%20the,the%20pull%20request's%20base%20branch. - name: Checkout source branch - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: ref: ${{ env.CHECKOUT_COMMIT_REF }} repository: ${{ env.CHECKOUT_REPO }} @@ -148,7 +148,7 @@ jobs: # See https://stackoverflow.com/questions/74957218/what-is-the-difference-between-pull-request-and-pull-request-target-event-in-git # See https://dev.to/suzukishunsuke/secure-github-actions-by-pullrequesttarget-641#:~:text=pull_request_target%20is%20one%20of%20the,the%20pull%20request's%20base%20branch. - name: Checkout source branch - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: ref: ${{ env.CHECKOUT_COMMIT_REF }} repository: ${{ env.CHECKOUT_REPO }} @@ -175,20 +175,20 @@ jobs: EOF - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v3 - name: Login to DockerHub - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Build and push ${{ matrix.service }} id: docker_build - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: # for frontend, context is ./zubhub_frontend/zubhub/ # for backend services, context is ./zubhub_backend/ diff --git a/zubhub_backend/docker-compose.prod.yml b/zubhub_backend/docker-compose.prod.yml index dbd5c4b3b..6b769d819 100644 --- a/zubhub_backend/docker-compose.prod.yml +++ b/zubhub_backend/docker-compose.prod.yml @@ -95,20 +95,20 @@ services: depends_on: - rabbitmq - prometheus: - image: prom/prometheus - command: - - --config.file=/etc/prometheus/prometheus.yml - deploy: - replicas: 1 - restart_policy: - condition: on-failure - placement: - max_replicas_per_node: 1 - constraints: - - "node.role==manager" - volumes: - - ./compose/prometheus.yml:/etc/prometheus/prometheus.yml:ro + # prometheus: + # image: prom/prometheus + # command: + # - --config.file=/etc/prometheus/prometheus.yml + # deploy: + # replicas: 1 + # restart_policy: + # condition: on-failure + # placement: + # max_replicas_per_node: 1 + # constraints: + # - "node.role==manager" + # volumes: + # - ./compose/prometheus.yml:/etc/prometheus/prometheus.yml:ro secrets: zubhub_services_secrets: