Extract the test harness from the Hspec tests

PR-URL: hasura/graphql-engine-mono#6095 Co-authored-by: Gil Mizrahi <[email protected]> GitOrigin-RevId: 51693f7324e62e201a2bdc701255cf6c730745e2
hasura · Oct 4, 2022 · f05f746 · f05f746
1 parent 05b3a64
commit f05f746
Show file tree

Hide file tree

Showing 133 changed files with 789 additions and 742 deletions.
diff --git a/cabal/ci-haddock.project.local b/cabal/ci-haddock.project.local
@@ -24,6 +24,16 @@ package graphql-engine
   executable-dynamic: True
   library-vanilla: False
 
+package test-harness
+  ghc-options: -Werror -j
+  executable-dynamic: True
+  library-vanilla: False
+
+package api-tests
+  ghc-options: -Werror -j
+  executable-dynamic: True
+  library-vanilla: False
+
 -- %%%-BEGIN-PRO-ONLY-CONTENT-%%%
 
 package graphql-engine-pro

diff --git a/cabal/dev-sh.project.local b/cabal/dev-sh.project.local
@@ -48,3 +48,21 @@ package graphql-engine
   executable-dynamic: True
   library-vanilla: False
 
+package test-harness
+  documentation: false
+  coverage: true
+  ghc-options: -Werror
+  ghc-options: -j
+  -- See "package graphql-engine" for more information
+  executable-dynamic: True
+  library-vanilla: False
+
+package api-tests
+  documentation: false
+  coverage: true
+  ghc-options: -Werror
+  ghc-options: -j
+  -- See "package graphql-engine" for more information
+  executable-dynamic: True
+  library-vanilla: False
+
diff --git a/sample.hie.yaml b/sample.hie.yaml
@@ -14,8 +14,8 @@ cradle:
 
     - path: "./server/src-bench-cache"
       component: "graphql-engine:bench:cache"
-    - path: "server/tests-hspec"
-      component: "graphql-engine:test:tests-hspec"
+    - path: "server/lib/api-tests"
+      component: "api-tests:exe:api-tests"
 
     - path: "server/src-emit-metadata-openapi"
       component: "graphql-engine:exe:emit-metadata-openapi"

diff --git a/scripts/make/build.mk b/scripts/make/build.mk
@@ -21,7 +21,7 @@ build-tests: $(GENERATED_CABAL_FILES)
 .PHONY: build-integration-tests
 ## build-integration-tests: build hspec integration tests
 build-integration-tests: $(GENERATED_CABAL_FILES)
-	cabal build tests-hspec
+	cabal build api-tests
 
 .PHONY: build-pro
 ## build-pro: build pro graphql executable

diff --git a/scripts/make/ghcid.mk b/scripts/make/ghcid.mk
@@ -8,7 +8,7 @@ PANE_HEIGHT = $(shell tmux display -p "\#{pane_height}" || echo 30 )
 # once ghcid's window errors are fixed we can remove this explicit width/height
 # nonsense
 # this needs to make it into ghcid: https://github.com/biegunka/terminal-size/pull/16
-define run_ghcid_hspec_tests
+define run_ghcid_api_tests
 	@if [[ $$(uname -p) == 'arm' ]]; then \
 		HSPEC_MATCH="$(2)" ghcid -c "DYLD_LIBRARY_PATH=$$DYLD_LIBRARY_PATH cabal repl $(1) $(GHCID_TESTS_FLAGS)" \
 			--test "main" \
@@ -53,46 +53,46 @@ ghcid-library:
 ghcid-tests:
 	$(call run_ghcid,graphql-engine:test:graphql-engine-tests)
 
-.PHONY: ghcid-hspec
-## ghcid-hspec: build and watch tests-hspec in ghcid
-ghcid-hspec:
-	$(call run_ghcid,graphql-engine:tests-hspec)
+.PHONY: ghcid-api-tests
+## ghcid-api-tests: build and watch api-tests in ghcid
+ghcid-api-tests:
+	$(call run_ghcid,api-tests)
 
 .PHONY: ghcid-test-backends
-## ghcid-test-backends: run all hspec tests in ghcid
+## ghcid-test-backends: run all api tests in ghcid
 ghcid-test-backends: remove-tix-file
-	$(call run_ghcid_hspec_tests,graphql-engine:tests-hspec)
+	$(call run_ghcid_api_tests,api-tests)
 
 .PHONY: ghcid-test-bigquery
 ## ghcid-test-bigquery: run tests for BigQuery backend in ghcid
-# will require some setup detailed here: https://github.com/hasura/graphql-engine-mono/tree/main/server/tests-hspec#required-setup-for-bigquery-tests
+# will require some setup detailed here: https://github.com/hasura/graphql-engine-mono/tree/main/server/lib/api-tests#required-setup-for-bigquery-tests
 ghcid-test-bigquery: start-postgres remove-tix-file
-	$(call run_ghcid_hspec_tests,graphql-engine:tests-hspec,BigQuery)
+	$(call run_ghcid_api_tests,api-tests,BigQuery)
 
 .PHONY: ghcid-test-sqlserver
 ## ghcid-test-sqlserver: run tests for SQL Server backend in ghcid
 ghcid-test-sqlserver: start-postgres start-sqlserver remove-tix-file
-	$(call run_ghcid_hspec_tests,graphql-engine:tests-hspec,SQLServer)
+	$(call run_ghcid_api_tests,api-tests,SQLServer)
 
 .PHONY: ghcid-test-mysql
 ## ghcid-test-mysql: run tests for MySQL backend in ghcid
 ghcid-test-mysql: start-postgres start-mysql remove-tix-file
-	$(call run_ghcid_hspec_tests,graphql-engine:tests-hspec,MySQL)
+	$(call run_ghcid_api_tests,api-tests,MySQL)
 
 .PHONY: ghcid-test-citus
 ## ghcid-test-citus: run tests for Citus backend in ghcid
 ghcid-test-citus: start-postgres start-citus remove-tix-file
-	$(call run_ghcid_hspec_tests,graphql-engine:tests-hspec,Citus)
+	$(call run_ghcid_api_tests,api-tests,Citus)
 
 .PHONY: ghcid-test-cockroach
 ## ghcid-test-cockroach: run tests for Cockroach backend in ghcid
 ghcid-test-cockroach: start-postgres start-cockroach remove-tix-file
-	$(call run_ghcid_hspec_tests,graphql-engine:tests-hspec,Cockroach)
+	$(call run_ghcid_api_tests,api-tests,Cockroach)
 
 .PHONY: ghcid-test-data-connectors
 ## ghcid-test-data-connectors: run tests for DataConnectors in ghcid
 ghcid-test-data-connectors: start-postgres start-dc-reference-agent remove-tix-file
-	$(call run_ghcid_hspec_tests,graphql-engine:tests-hspec,DataConnector)
+	$(call run_ghcid_api_tests,api-tests,DataConnector)
 
 .PHONY: ghcid-library-pro
 ## ghcid-library-pro: build and watch pro library in ghcid

diff --git a/scripts/make/test-infrastructure.mk b/scripts/make/test-infrastructure.mk
@@ -1,5 +1,5 @@
 # the following variables are set up in Docker Compose
-# and are also defined in Harness.Constants for use in hspec tests
+# and are also defined in Harness.Constants for use in api tests
 PG_PORT = 65002
 PG_DBNAME = hasura
 PG_DBUSER = hasura
@@ -155,5 +155,5 @@ stop-everything:
 
 .PHONY: remove-tix-file
 remove-tix-file:
-	@ rm -f tests-hspec.tix
+	@ rm -f api-tests.tix
 	@ rm -f graphql-engine-tests.tix
diff --git a/scripts/make/tests.mk b/scripts/make/tests.mk
@@ -1,46 +1,46 @@
 .PHONY: test-bigquery
 ## test-bigquery: run tests for BigQuery backend
-# will require some setup detailed here: https://github.com/hasura/graphql-engine-mono/tree/main/server/tests-hspec#required-setup-for-bigquery-tests
+# will require some setup detailed here: https://github.com/hasura/graphql-engine-mono/tree/main/server/lib/api-tests#required-setup-for-bigquery-tests
 test-bigquery: start-postgres remove-tix-file
 	$(call stop_after, \
-		cabal run tests-hspec -- -m 'BigQuery')
+		cabal run api-tests -- -m 'BigQuery')
 
 .PHONY: test-sqlserver
 ## test-sqlserver: run tests for SQL Server backend
 test-sqlserver: spawn-postgres spawn-sqlserver wait-for-postgres wait-for-sqlserver remove-tix-file
 	$(call stop_after, \
-		cabal run tests-hspec -- -m 'SQLServer')
+		cabal run api-tests -- -m 'SQLServer')
 
 .PHONY: test-mysql
 ## test-mysql: run tests for MySQL backend
 test-mysql: spawn-postgres spawn-mysql wait-for-postgres wait-for-mysql remove-tix-file
 	$(call stop_after, \
-		cabal run tests-hspec -- -m 'MySQL')
+		cabal run api-tests -- -m 'MySQL')
 
 .PHONY: test-citus
 ## test-citus: run tests for Citus backend
 test-citus: spawn-postgres spawn-citus wait-for-postgres wait-for-citus remove-tix-file
 	$(call stop_after, \
-		cabal run tests-hspec -- -m 'Citus')
+		cabal run api-tests -- -m 'Citus')
 
 .PHONY: test-data-connectors
 ## test-data-connectors: run tests for Data Connectors
 test-data-connectors: start-postgres start-dc-reference-agent remove-tix-file
 	$(call stop_after, \
-		cabal run tests-hspec -- -m 'DataConnector')
+		cabal run api-tests -- -m 'DataConnector')
 
 .PHONY: test-cockroach
 ## test-cockroach: run tests for Cockroach backend
 test-cockroach: spawn-postgres spawn-cockroach wait-for-postgres wait-for-cockroach remove-tix-file
 	$(call stop_after, \
-		cabal run tests-hspec -- -m 'Cockroach')
+		cabal run api-tests -- -m 'Cockroach')
 
 .PHONY: test-backends
 ## test-backends: run tests for all backends
-# BigQuery tests will require some setup detailed here: https://github.com/hasura/graphql-engine-mono/tree/main/server/tests-hspec#required-setup-for-bigquery-tests
+# BigQuery tests will require some setup detailed here: https://github.com/hasura/graphql-engine-mono/tree/main/server/lib/api-tests#required-setup-for-bigquery-tests
 test-backends: start-backends remove-tix-file
 	$(call stop_after, \
-		cabal run tests-hspec)
+		cabal run api-tests)
 
 .PHONY: test-unit
 ## test-unit: run unit tests from main suite

diff --git a/scripts/verify-bigquery-creds.sh b/scripts/verify-bigquery-creds.sh
@@ -4,7 +4,7 @@ set -e
 set -u
 
 # Suggested usage:
-# https://github.com/hasura/graphql-engine/blob/master/server/tests-hspec/README.md#required-setup-for-bigquery-tests
+# https://github.com/hasura/graphql-engine/blob/master/server/lib/api-tests/README.md#required-setup-for-bigquery-tests
 # https://cloud.google.com/iam/docs/creating-managing-service-accounts#iam-service-accounts-create-rest
 
 if [[ -z "${HASURA_BIGQUERY_SERVICE_KEY:-}" ]]; then

diff --git a/server/CONTRIBUTING.md b/server/CONTRIBUTING.md
@@ -135,7 +135,7 @@ This will launch a server on port 8080, and it will serve the console assets if
 
 1. A small set of unit tests and integration tests written in Haskell, in `server/src-test`.
 
-2. A new integration test suite written in Haskell, in `server/tests-hspec`.
+2. A new integration test suite written in Haskell, in `server/lib/api-tests`.
 
 3. An extensive set of end-to-end tests written in Python, in `server/tests-py`.
 
@@ -198,10 +198,10 @@ docker compose up
 2. Once the containers are up, you can run the test suite via
 
 ```sh
-cabal test tests-hspec --test-show-details=direct
+cabal run api-tests
 ```
 
-For more details please check out the [README](./tests-hspec/README.md).
+For more details please check out the [README](./lib/api-tests/README.md).
 
 ##### Running unit tests and recompiling