chore: moved core app & e2e tests as monorepo prep (#118)

electric-sql · Apr 19, 2023 · 632b5c0 · 632b5c0
1 parent 43741b7
commit 632b5c0
Show file tree

Hide file tree

Showing 159 changed files with 245 additions and 157 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -19,13 +19,15 @@ jobs:
   deps:
     name: Get all the necessary dependencies
     runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: components/electric
     steps:
       - uses: actions/checkout@v3
         with:
           ref: ${{ github.event.pull_request.head.sha }}
           # Tags are required for proper version inference, and this is the only way to pull the tags in
           fetch-depth: 0
-      - run: ls -lah
       - name: Inject slug/short variables
         uses: rlespinasse/github-slug-action@v4
       - uses: erlef/setup-beam@v1
@@ -40,24 +42,27 @@ jobs:
         id: cache-deps
         uses: actions/cache@v3
         with:
-          path: deps
-          key: ${{ runner.os }}-mixdeps-${{ hashFiles('**/mix.lock') }}
+          path: components/electric/deps
+          key: ${{ runner.os }}-mixdeps-${{ hashFiles('components/electric/**/mix.lock') }}
 
       - name: Cache compiled code
         id: cache-build
         uses: actions/cache@v3
         with:
           path: |
-            _build/*/lib
-            !_build/*/lib/electric
-          key: ${{ runner.os }}-mixbuild-${{ env.OTP_VERSION }}-${{ env.ELIXIR_VERSION }}-${{ hashFiles('**/mix.lock') }}
+            components/electric/_build/*/lib
+            !components/electric/_build/*/lib/electric
+          key: ${{ runner.os }}-mixbuild-${{ env.OTP_VERSION }}-${{ env.ELIXIR_VERSION }}-${{ hashFiles('components/electric/**/mix.lock') }}
 
       - run: make deps pretest_compile
 
   tests:
     name: elixir tests
     runs-on: ubuntu-latest
     needs: deps
+    defaults:
+      run:
+        working-directory: components/electric
     steps:
       - uses: actions/checkout@v3
         with:
@@ -78,17 +83,17 @@ jobs:
         id: cache-deps
         uses: actions/cache/restore@v3
         with:
-          path: deps
-          key: ${{ runner.os }}-mixdeps-${{ hashFiles('**/mix.lock') }}
+          path: components/electric/deps
+          key: ${{ runner.os }}-mixdeps-${{ hashFiles('components/electric/**/mix.lock') }}
 
       - name: Restore compiled code
         id: cache-build
         uses: actions/cache/restore@v3
         with:
           path: |
-            _build/*/lib
-            !_build/*/lib/electric
-          key: ${{ runner.os }}-mixbuild-${{ env.OTP_VERSION }}-${{ env.ELIXIR_VERSION }}-${{ hashFiles('**/mix.lock') }}
+            components/electric/_build/*/lib
+            !components/electric/_build/*/lib/electric
+          key: ${{ runner.os }}-mixbuild-${{ env.OTP_VERSION }}-${{ env.ELIXIR_VERSION }}-${{ hashFiles('components/electric/**/mix.lock') }}
 
       - uses: webfactory/[email protected]
         with:
@@ -105,15 +110,19 @@ jobs:
       - name: Report code coverage
         uses: zgosalvez/github-actions-report-lcov@v3
         with:
-          coverage-files: cover/lcov.info
+          coverage-files: components/electric/cover/lcov.info
           artifact-name: code-coverage-report
           github-token: ${{ secrets.GITHUB_TOKEN }}
           update-comment: true
+          working-directory: components/electric
 
   dialyzer:
     name: dialyzer and formatting
     runs-on: ubuntu-latest
     needs: deps
+    defaults:
+      run:
+        working-directory: components/electric
     steps:
       - uses: actions/checkout@v3
         with:
@@ -131,22 +140,22 @@ jobs:
         id: cache-deps
         uses: actions/cache/restore@v3
         with:
-          path: deps
-          key: ${{ runner.os }}-mixdeps-${{ hashFiles('**/mix.lock') }}
+          path: components/electric/deps
+          key: ${{ runner.os }}-mixdeps-${{ hashFiles('components/electric/**/mix.lock') }}
 
       - name: Restore compiled code
         id: cache-build
         uses: actions/cache/restore@v3
         with:
           path: |
-            _build/*/lib
-            !_build/*/lib/electric
-          key: ${{ runner.os }}-mixbuild-${{ env.OTP_VERSION }}-${{ env.ELIXIR_VERSION }}-${{ hashFiles('**/mix.lock') }}
+            components/electric/_build/*/lib
+            !components/electric/_build/*/lib/electric
+          key: ${{ runner.os }}-mixbuild-${{ env.OTP_VERSION }}-${{ env.ELIXIR_VERSION }}-${{ hashFiles('components/electric/**/mix.lock') }}
 
       - name: Cache PLT
         uses: actions/cache@v3
         with:
-          path: _build/*/*.plt
+          path: components/electric/_build/*/*.plt
           key: ${{ runner.os }}-plt-${{ github.ref_name }}
           restore-keys: ${{ runner.os }}-plt-
 
@@ -163,6 +172,9 @@ jobs:
     name: E2E tests
     runs-on: ubuntu-latest
     needs: deps
+    defaults:
+      run:
+        working-directory: e2e
     env:
       VAXINE_IMAGE: europe-docker.pkg.dev/vaxine/vaxine-io/vaxine:latest
     steps:
@@ -188,45 +200,45 @@ jobs:
         id: cache-deps
         uses: actions/cache/restore@v3
         with:
-          path: deps
-          key: ${{ runner.os }}-mixdeps-${{ hashFiles('**/mix.lock') }}
+          path: components/electric/deps
+          key: ${{ runner.os }}-mixdeps-${{ hashFiles('components/electric/**/mix.lock') }}
 
       - name: Restore compiled code
         id: cache-build
         uses: actions/cache/restore@v3
         with:
           path: |
-            _build/*/lib
-            !_build/*/lib/electric
-          key: ${{ runner.os }}-mixbuild-${{ env.OTP_VERSION }}-${{ env.ELIXIR_VERSION }}-${{ hashFiles('**/mix.lock') }}
+            components/electric/_build/*/lib
+            !components/electric/_build/*/lib/electric
+          key: ${{ runner.os }}-mixbuild-${{ env.OTP_VERSION }}-${{ env.ELIXIR_VERSION }}-${{ hashFiles('components/electric/**/mix.lock') }}
+
       - run: |
           echo "ELECTRIC_VERSION=$(make --silent print_version_from_git)" >> $GITHUB_ENV
+        working-directory: components/electric
       - run: make docker-build-ci
         env:
           ELECTRIC_IMAGE_NAME: ghcr.io/${{ github.repository }}/electric
+        working-directory: components/electric
       - run: make pretest_compile
+        working-directory: components/electric
 
       - name: Cache built lux
         uses: actions/cache@v3
         with:
           path: |
-            integration_tests/lux/bin
-            integration_tests/lux/ebin
-            integration_tests/lux/priv
+            e2e/lux/bin
+            e2e/lux/ebin
+            e2e/lux/priv
           key: ${{ runner.os }}-luxbuilt-${{ env.OTP_VERSION }}-${{ env.ELIXIR_VERSION }}
 
       - run: make lux
-        working-directory: integration_tests
 
       - run: make vaxine
-        working-directory: integration_tests
 
       - run: make postgres
-        working-directory: integration_tests
 
       - run: make test
         id: tests
-        working-directory: integration_tests
         env:
           ELECTRIC_IMAGE_NAME: ghcr.io/${{ github.repository }}/electric
           ELECTRIC_IMAGE_TAG: ${{ env.ELECTRIC_VERSION }}
@@ -236,4 +248,4 @@ jobs:
         if: ${{ always() }}
         with:
           name: lux_logs
-          path: integration_tests/lux_logs/run_*
+          path: e2e/lux_logs/run_*
diff --git a/Makefile b/Makefile
@@ -1,106 +1,6 @@
+build_all:
+	make -C components/electric docker-build
 
-.PHONY: build_tools deps compile tests start_dev_env stop_dev_env integration_tests rm_offset_storage print_version_from_git
-
-INFERRED_VERSION = $(shell git describe --abbrev=7 --tags --always --first-parent)
-
-print_version_from_git:
-	echo "${INFERRED_VERSION}"
-
-build_tools:
-	mix local.hex --force
-	mix local.rebar --force
-
-deps:
-	mix deps.get
-
-dialyzer:
-	mix dialyzer
-
-compile:
-	mix compile
-
-compile-%:
-	MIX_ENV="$*" mix compile
-
-release:
-	MIX_ENV="prod" mix release
-
-pretest_compile: deps
-	MIX_ENV="test" mix compile --force --warnings-as-error
-
-tests:
-	mix test --trace
-
-format:
-	mix format
-
-check-format:
-	mix format --check-formatted
-
-DC_CONFIG=compose.yaml
-
-start_dev_env:
-	docker compose -f ${DC_CONFIG} up -d
-
-export UID=$(shell id -u)
-export GID=$(shell id -g)
-start_dev_env_mounted:
-	mkdir -p _tmp_vaxine_data
-	docker compose -f compose-mounts.yaml up -d
-
-stop_dev_env:
-	docker compose -f ${DC_CONFIG} down
-	rm -rf _tmp_vaxine_data
-
-DOCKER_PREFIX:=$(shell basename $(CURDIR))
-docker-pgsql-%:
-	docker exec -it -e PGPASSWORD=password ${DOCKER_PREFIX}_$*_1 psql -h $* -U electric -d electric
-
-ELECTRIC_VERSION ?= ${INFERRED_VERSION}
-docker-build:
-	docker build --build-arg ELECTRIC_VERSION=${ELECTRIC_VERSION} -t electric:local-build .
-
-docker-build-ci:
-	mkdir -p deps
-	docker build --build-arg ELECTRIC_VERSION=${ELECTRIC_VERSION} \
-      -t ${ELECTRIC_IMAGE_NAME}:${ELECTRIC_VERSION} \
-      -t electric:local-build .
-	docker push ${ELECTRIC_IMAGE_NAME}:${ELECTRIC_VERSION}
-ifeq (${TAG_AS_LATEST}, true)
-	docker tag "${ELECTRIC_IMAGE_NAME}:${ELECTRIC_VERSION}" "${ELECTRIC_IMAGE_NAME}:latest"
-	docker push "${ELECTRIC_IMAGE_NAME}:latest"
-endif
-
-docker-build-ci-crossplatform:
-	mkdir -p deps
-	docker buildx build --platform linux/arm64/v8,linux/amd64 --push \
-			--build-arg ELECTRIC_VERSION=${ELECTRIC_VERSION} \
-			-t ${ELECTRIC_IMAGE_NAME}:${ELECTRIC_VERSION} \
-			-t ${ELECTRIC_IMAGE_NAME}:latest .
-
-docker-clean:
-ifneq ($(docker images -q electric:local-build 2> /dev/null), "")
-	docker image rm -f electric:local-build
-endif
-
-rm_offset_storage:
-	rm offset_storage_*
-
-update_protobuf: deps
-	mix electric.gen.proto.package \
-		--output-path=./lib/electric/satellite/protobuf_package.ex \
-		./deps/satellite_proto/proto/satellite.proto
-	mix protox.generate \
-		--output-path=./lib/electric/satellite/protobuf_messages.ex \
-		./deps/satellite_proto/proto/satellite.proto
-shell:
-	iex -S mix
-
-shell_clean:
-	iex -S mix run --no-start
-
-apply_migration:
-	./apply-local-migration.sh $(name)
-
-codespell:
-	codespell -L authentification --skip .git --skip deps
+test_all:
+	make -C components/electric deps tests
+	make -C e2e test
diff --git a/.dockerignore → components/electric/.dockerignore b/.dockerignore → components/electric/.dockerignore
diff --git a/.formatter.exs → components/electric/.formatter.exs b/.formatter.exs → components/electric/.formatter.exs
diff --git a/components/electric/.gitignore b/components/electric/.gitignore
@@ -0,0 +1,31 @@
+# The directory Mix will write compiled artifacts to.
+/_build/
+
+# If you run "mix test --cover", coverage assets end up here.
+/cover/
+
+# The directory Mix downloads your dependencies sources to.
+/deps/
+
+# Where third-party dependencies like ExDoc output generated docs.
+/doc/
+
+# Ignore .fetch files in case you like to edit your project deps locally.
+/.fetch
+
+# If the VM crashes, it generates a dump, let's ignore it too.
+erl_crash.dump
+
+# Also ignore archive artifacts (built via "mix archive.build").
+*.ez
+
+# Ignore package tarball (built via "mix hex.build").
+electric-*.tar
+
+# Temporary files, for example, from tests.
+/tmp/
+
+*offset_storage*.dat
+
+/archives
+.node_repl_history
diff --git a/Dockerfile → components/electric/Dockerfile b/Dockerfile → components/electric/Dockerfile