From a89bbedda0cbffbb63bb2c884b377c4ebdd9accd Mon Sep 17 00:00:00 2001 From: Akshay Mankar Date: Mon, 24 Oct 2022 11:29:18 +0200 Subject: [PATCH 1/3] Makefile: Allow db-migrate and db-reset to migrate/reset all keyspaces --- Makefile | 64 ++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 46 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index c14b958194..1b165316db 100644 --- a/Makefile +++ b/Makefile @@ -139,18 +139,6 @@ regen-local-nix-derivations: check-local-nix-derivations: regen-local-nix-derivations git diff --exit-code -# reset db using cabal -.PHONY: db-reset-package -db-reset-package: c - $(EXE_SCHEMA) --keyspace $(package)_test --replication-factor 1 --reset - -# migrate db using cabal -# For using stack see the Makefile of the package, e.g. services/brig/Makefile -# Usage: make db-migrate-package package=galley -.PHONY: db-migrate-package -db-migrate-package: c - $(EXE_SCHEMA) --keyspace $(package)_test --replication-factor 1 - # Build everything (Haskell services and nginz) .PHONY: services services: init install @@ -217,13 +205,53 @@ cqlsh: @echo "make sure you have ./deploy/dockerephemeral/run.sh running in another window!" docker exec -it $(CASSANDRA_CONTAINER) /usr/bin/cqlsh +.PHONY: db-reset-package +db-reset-package: + @echo "Deprecated! Please use 'db-reset' instead" + $(MAKE) db-reset package=$(package) + +.PHONY: db-migrate-package +db-migrate-package: + @echo "Deprecated! Please use 'db-migrate' instead" + $(MAKE) db-migrate package=$(package) + +# Usage: +# +# Reset all keyspaces +# make db-reset +# +# Reset keyspace for only one service, say galley: +# make db-reset package=galley .PHONY: db-reset -db-reset: - @echo "make sure you have ./deploy/dockerephemeral/run.sh running in another window!" - make db-reset-package package=brig - make db-reset-package package=galley - make db-reset-package package=gundeck - make db-reset-package package=spar +db-reset: c + @echo "Make sure you have ./deploy/dockerephemeral/run.sh running in another window!" +ifeq ($(package), all) + ./dist/brig-schema --keyspace brig_test --replication-factor 1 --reset + ./dist/galley-schema --keyspace galley_test --replication-factor 1 --reset + ./dist/gundeck-schema --keyspace gundeck_test --replication-factor 1 --reset + ./dist/spar-schema --keyspace spar_test --replication-factor 1 --reset +else + $(EXE_SCHEMA) --keyspace $(package)_test --replication-factor 1 --reset +endif + +# Usage: +# +# Migrate all keyspaces +# make db-migrate +# +# Migrate keyspace for only one service, say galley: +# make db-migrate package=galley +.PHONY: db-migrate +db-migrate: c +ifeq ($(package), all) + ./dist/brig-schema --keyspace brig_test --replication-factor 1 + ./dist/galley-schema --keyspace galley_test --replication-factor 1 + ./dist/gundeck-schema --keyspace gundeck_test --replication-factor 1 + ./dist/spar-schema --keyspace spar_test --replication-factor 1 +else + $(EXE_SCHEMA) --keyspace $(package)_test --replication-factor 1 +endif + ################################# ## dependencies From 849620e4b3d74c8c08182874fb9faf49b636720a Mon Sep 17 00:00:00 2001 From: Akshay Mankar Date: Mon, 24 Oct 2022 11:32:29 +0200 Subject: [PATCH 2/3] Use locally build schema binaries for db migrations Also make sure DB is up to date every time tests are run and not just when DBs are spun up. --- Makefile | 2 +- deploy/dockerephemeral/docker-compose.yaml | 60 ---------------------- deploy/dockerephemeral/run.sh | 13 ----- 3 files changed, 1 insertion(+), 74 deletions(-) diff --git a/Makefile b/Makefile index 1b165316db..ac997ffbdd 100644 --- a/Makefile +++ b/Makefile @@ -86,7 +86,7 @@ endif # Usage: make ci package=brig test=1 # If you want to pass arguments to the test-suite call the script directly. .PHONY: ci -ci: c +ci: c db-migrate ./hack/bin/cabal-run-integration.sh $(package) .PHONY: cabal-fmt diff --git a/deploy/dockerephemeral/docker-compose.yaml b/deploy/dockerephemeral/docker-compose.yaml index cd5788f302..87d21eddae 100644 --- a/deploy/dockerephemeral/docker-compose.yaml +++ b/deploy/dockerephemeral/docker-compose.yaml @@ -198,66 +198,6 @@ services: networks: - demo_wire - db_migrations_brig_schema: - image: quay.io/wire/brig-schema - depends_on: - - cassandra - entrypoint: /scripts/db-migrate/brig-schema.sh - volumes: - - ./:/scripts - links: - - cassandra - networks: - - demo_wire - - db_migrations_brig_index: - image: quay.io/wire/brig-index - depends_on: - - elasticsearch - entrypoint: /scripts/db-migrate/brig-index.sh - volumes: - - ./:/scripts - links: - - elasticsearch - networks: - - demo_wire - - db_migrations_galley: - image: quay.io/wire/galley-schema - depends_on: - - cassandra - entrypoint: /scripts/db-migrate/galley-schema.sh - volumes: - - ./:/scripts - links: - - cassandra - networks: - - demo_wire - - db_migrations_gundeck: - image: quay.io/wire/gundeck-schema - depends_on: - - cassandra - entrypoint: /scripts/db-migrate/gundeck-schema.sh - volumes: - - ./:/scripts - links: - - cassandra - networks: - - demo_wire - - db_migrations_spar: - image: quay.io/wire/spar-schema - depends_on: - - cassandra - entrypoint: /scripts/db-migrate/spar-schema.sh - volumes: - - ./:/scripts - links: - - cassandra - networks: - - demo_wire - # FIXME: replace aws_cli with an image that we build. aws_cli: image: mesosphere/aws-cli:1.14.5 diff --git a/deploy/dockerephemeral/run.sh b/deploy/dockerephemeral/run.sh index e723679596..13a413743b 100755 --- a/deploy/dockerephemeral/run.sh +++ b/deploy/dockerephemeral/run.sh @@ -4,17 +4,4 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" DOCKER_FILE="$SCRIPT_DIR/docker-compose.yaml" -# If there are schema changes and you don't force pull the docker -# migrations, you may run out of sync and you would get this error -# message (or a similar one): -# -# brig: Schema Version too old! Expecting at least: 49, but got: 48 -# -# So we always pull these migration images first. -docker pull quay.io/wire/brig-schema -docker pull quay.io/wire/galley-schema -docker pull quay.io/wire/gundeck-schema -docker pull quay.io/wire/spar-schema -docker pull quay.io/wire/brig-index - docker-compose --file "$DOCKER_FILE" up From 1a36a2e296c7ef4de24876fefeb93d562805ae95 Mon Sep 17 00:00:00 2001 From: Akshay Mankar Date: Mon, 24 Oct 2022 11:38:47 +0200 Subject: [PATCH 3/3] Changelog --- changelog.d/5-internal/integration-db-migrate | 1 + changelog.d/5-internal/make-db-migrate | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog.d/5-internal/integration-db-migrate create mode 100644 changelog.d/5-internal/make-db-migrate diff --git a/changelog.d/5-internal/integration-db-migrate b/changelog.d/5-internal/integration-db-migrate new file mode 100644 index 0000000000..aef7c960d6 --- /dev/null +++ b/changelog.d/5-internal/integration-db-migrate @@ -0,0 +1 @@ +Use locally build schema binaries for db migrations and execute them right before running integration tests. \ No newline at end of file diff --git a/changelog.d/5-internal/make-db-migrate b/changelog.d/5-internal/make-db-migrate new file mode 100644 index 0000000000..8afa843be3 --- /dev/null +++ b/changelog.d/5-internal/make-db-migrate @@ -0,0 +1 @@ +Rename the make targets from `db-migrate-package` and `db-reset-package` to `db-migrate` and `db-reset` and allow migrating and resetting all keyspaces. \ No newline at end of file