diff --git a/dependency_licenses.txt b/dependency_licenses.txt index 6d6c6e77cc..c292521fc2 100644 --- a/dependency_licenses.txt +++ b/dependency_licenses.txt @@ -2176,7 +2176,7 @@ Apache License ----- -The following software may be included in this product: @aws-sdk/client-cloudwatch-logs, @aws-sdk/client-cognito-identity, @aws-sdk/client-cognito-identity-provider, @aws-sdk/client-comprehend, @aws-sdk/client-dynamodb, @aws-sdk/client-ec2, @aws-sdk/client-firehose, @aws-sdk/client-iam, @aws-sdk/client-kinesis, @aws-sdk/client-kms, @aws-sdk/client-lambda, @aws-sdk/client-lex-runtime-service, @aws-sdk/client-personalize-events, @aws-sdk/client-pinpoint, @aws-sdk/client-polly, @aws-sdk/client-rds, @aws-sdk/client-rekognition, @aws-sdk/client-s3, @aws-sdk/client-secrets-manager, @aws-sdk/client-sfn, @aws-sdk/client-ssm, @aws-sdk/client-sso, @aws-sdk/client-sso-oidc, @aws-sdk/client-sts, @aws-sdk/client-textract, @aws-sdk/client-translate, @aws-sdk/credential-provider-cognito-identity, @aws-sdk/eventstream-codec, @aws-sdk/eventstream-marshaller, @aws-sdk/middleware-apply-body-checksum, @aws-sdk/middleware-bucket-endpoint, @aws-sdk/middleware-endpoint-discovery, @aws-sdk/middleware-eventstream, @aws-sdk/middleware-expect-continue, @aws-sdk/middleware-header-default, @aws-sdk/middleware-location-constraint, @aws-sdk/middleware-retry, @aws-sdk/middleware-sdk-rds, @aws-sdk/middleware-sdk-sts, @aws-sdk/middleware-ssec, @aws-sdk/querystring-builder, @aws-sdk/querystring-parser, @aws-sdk/service-error-classification, @aws-sdk/url-parser, @aws-sdk/url-parser-native, @aws-sdk/util-defaults-mode-browser, @aws-sdk/util-defaults-mode-node, @aws-sdk/util-user-agent-browser, @aws-sdk/util-user-agent-node, @smithy/eventstream-codec, @smithy/middleware-retry, @smithy/querystring-builder, @smithy/querystring-parser, @smithy/service-error-classification, @smithy/url-parser, @smithy/util-defaults-mode-browser, @smithy/util-defaults-mode-node. A copy of the source code may be downloaded from https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-cloudwatch-logs), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-cognito-identity), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-cognito-identity-provider), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-comprehend), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-dynamodb), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-ec2), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-firehose), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-iam), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-kinesis), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-kms), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-lambda), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-lex-runtime-service), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-personalize-events), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-pinpoint), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-polly), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-rds), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-rekognition), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-s3), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-secrets-manager), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-sfn), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-ssm), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-sso), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-sso-oidc), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-sts), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-textract), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-translate), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/credential-provider-cognito-identity), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/eventstream-codec), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/eventstream-marshaller), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/middleware-apply-body-checksum), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/middleware-bucket-endpoint), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/middleware-endpoint-discovery), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/middleware-eventstream), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/middleware-expect-continue), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/middleware-header-default), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/middleware-location-constraint), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/middleware-retry), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/middleware-sdk-rds), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/middleware-sdk-sts), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/middleware-ssec), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/querystring-builder), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/querystring-parser), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/service-error-classification), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/url-parser), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/url-parser-native), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/util-defaults-mode-browser), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/util-defaults-mode-node), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/util-user-agent-browser), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/util-user-agent-node), https://github.com/awslabs/smithy-typescript.git (@smithy/eventstream-codec), https://github.com/awslabs/smithy-typescript.git (@smithy/middleware-retry), https://github.com/awslabs/smithy-typescript.git (@smithy/querystring-builder), https://github.com/awslabs/smithy-typescript.git (@smithy/querystring-parser), https://github.com/awslabs/smithy-typescript.git (@smithy/service-error-classification), https://github.com/awslabs/smithy-typescript.git (@smithy/url-parser), https://github.com/awslabs/smithy-typescript.git (@smithy/util-defaults-mode-browser), https://github.com/awslabs/smithy-typescript.git (@smithy/util-defaults-mode-node). This software contains the following license and notice below: +The following software may be included in this product: @aws-sdk/client-cloudwatch-logs, @aws-sdk/client-cognito-identity, @aws-sdk/client-cognito-identity-provider, @aws-sdk/client-comprehend, @aws-sdk/client-dynamodb, @aws-sdk/client-ec2, @aws-sdk/client-firehose, @aws-sdk/client-iam, @aws-sdk/client-kinesis, @aws-sdk/client-kms, @aws-sdk/client-lambda, @aws-sdk/client-lex-runtime-service, @aws-sdk/client-personalize-events, @aws-sdk/client-pinpoint, @aws-sdk/client-polly, @aws-sdk/client-rds, @aws-sdk/client-rds-data, @aws-sdk/client-rekognition, @aws-sdk/client-s3, @aws-sdk/client-secrets-manager, @aws-sdk/client-sfn, @aws-sdk/client-ssm, @aws-sdk/client-sso, @aws-sdk/client-sso-oidc, @aws-sdk/client-sts, @aws-sdk/client-textract, @aws-sdk/client-translate, @aws-sdk/credential-provider-cognito-identity, @aws-sdk/eventstream-codec, @aws-sdk/eventstream-marshaller, @aws-sdk/middleware-apply-body-checksum, @aws-sdk/middleware-bucket-endpoint, @aws-sdk/middleware-endpoint-discovery, @aws-sdk/middleware-eventstream, @aws-sdk/middleware-expect-continue, @aws-sdk/middleware-header-default, @aws-sdk/middleware-location-constraint, @aws-sdk/middleware-retry, @aws-sdk/middleware-sdk-rds, @aws-sdk/middleware-sdk-sts, @aws-sdk/middleware-ssec, @aws-sdk/querystring-builder, @aws-sdk/querystring-parser, @aws-sdk/service-error-classification, @aws-sdk/url-parser, @aws-sdk/url-parser-native, @aws-sdk/util-defaults-mode-browser, @aws-sdk/util-defaults-mode-node, @aws-sdk/util-user-agent-browser, @aws-sdk/util-user-agent-node, @smithy/eventstream-codec, @smithy/middleware-retry, @smithy/querystring-builder, @smithy/querystring-parser, @smithy/service-error-classification, @smithy/url-parser, @smithy/util-defaults-mode-browser, @smithy/util-defaults-mode-node. A copy of the source code may be downloaded from https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-cloudwatch-logs), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-cognito-identity), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-cognito-identity-provider), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-comprehend), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-dynamodb), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-ec2), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-firehose), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-iam), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-kinesis), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-kms), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-lambda), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-lex-runtime-service), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-personalize-events), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-pinpoint), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-polly), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-rds), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-rds-data), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-rekognition), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-s3), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-secrets-manager), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-sfn), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-ssm), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-sso), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-sso-oidc), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-sts), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-textract), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/client-translate), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/credential-provider-cognito-identity), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/eventstream-codec), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/eventstream-marshaller), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/middleware-apply-body-checksum), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/middleware-bucket-endpoint), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/middleware-endpoint-discovery), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/middleware-eventstream), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/middleware-expect-continue), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/middleware-header-default), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/middleware-location-constraint), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/middleware-retry), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/middleware-sdk-rds), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/middleware-sdk-sts), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/middleware-ssec), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/querystring-builder), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/querystring-parser), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/service-error-classification), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/url-parser), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/url-parser-native), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/util-defaults-mode-browser), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/util-defaults-mode-node), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/util-user-agent-browser), https://github.com/aws/aws-sdk-js-v3.git (@aws-sdk/util-user-agent-node), https://github.com/awslabs/smithy-typescript.git (@smithy/eventstream-codec), https://github.com/awslabs/smithy-typescript.git (@smithy/middleware-retry), https://github.com/awslabs/smithy-typescript.git (@smithy/querystring-builder), https://github.com/awslabs/smithy-typescript.git (@smithy/querystring-parser), https://github.com/awslabs/smithy-typescript.git (@smithy/service-error-classification), https://github.com/awslabs/smithy-typescript.git (@smithy/url-parser), https://github.com/awslabs/smithy-typescript.git (@smithy/util-defaults-mode-browser), https://github.com/awslabs/smithy-typescript.git (@smithy/util-defaults-mode-node). This software contains the following license and notice below: Apache License Version 2.0, January 2004 diff --git a/packages/amplify-e2e-core/package.json b/packages/amplify-e2e-core/package.json index d220cfe9a6..11e5871903 100644 --- a/packages/amplify-e2e-core/package.json +++ b/packages/amplify-e2e-core/package.json @@ -28,6 +28,7 @@ "@aws-sdk/client-secrets-manager": "3.338.0", "@aws-sdk/client-ssm": "3.338.0", "@aws-sdk/client-sts": "3.338.0", + "@aws-sdk/client-rds-data": "3.338.0", "@aws-sdk/credential-providers": "3.338.0", "amplify-headless-interface": "^1.17.7", "axios": "^1.6.0", diff --git a/packages/amplify-e2e-core/src/utils/index.ts b/packages/amplify-e2e-core/src/utils/index.ts index 8d5cacc4ef..1974071eca 100644 --- a/packages/amplify-e2e-core/src/utils/index.ts +++ b/packages/amplify-e2e-core/src/utils/index.ts @@ -20,6 +20,7 @@ export * from './sleep'; export * from './transformConfig'; export * from './rds'; export * from './credentials-rotator'; +export * from './test-regions'; // run dotenv config to update env variable config(); diff --git a/packages/amplify-e2e-core/src/utils/rds.ts b/packages/amplify-e2e-core/src/utils/rds.ts index dd266482af..d965e4bd13 100644 --- a/packages/amplify-e2e-core/src/utils/rds.ts +++ b/packages/amplify-e2e-core/src/utils/rds.ts @@ -1,11 +1,19 @@ import { RDSClient, + CreateDBClusterCommand, CreateDBInstanceCommand, CreateDBInstanceCommandInput, DBInstance, DeleteDBInstanceCommand, waitUntilDBInstanceAvailable, + CreateDBClusterCommandInput, + CreateDBClusterMessage, + waitUntilDBClusterAvailable, + DeleteDBClusterCommand, + DeleteDBClusterCommandInput, } from '@aws-sdk/client-rds'; +import { RDSDataClient, ExecuteStatementCommand, ExecuteStatementCommandInput, Field } from '@aws-sdk/client-rds-data'; +import generator from 'generate-password'; import { EC2Client, AuthorizeSecurityGroupIngressCommand, RevokeSecurityGroupIngressCommand } from '@aws-sdk/client-ec2'; import { SSMClient, @@ -41,6 +49,23 @@ export type RDSConfig = { publiclyAccessible?: boolean; }; +export type ClusterInfo = { + clusterArn: string; + endpoint: string; + port: number; + dbName: string; + secretArn: string; + dbInstance: DBInstance; +}; + +const getRDSEngineType = (engine: SqlEngine): string => { + if (engine == 'postgres') { + return 'aurora-postgresql'; + } else { + throw new Error('Unsupported engine type for cluster'); + } +}; + /** * Creates a new RDS instance using the given input configuration and returns the details of the created RDS instance. * @param config Configuration of the database instance. If password is not passed an RDS managed password will be created. @@ -54,7 +79,7 @@ export const createRDSInstance = async ( dbName: string; dbInstance: DBInstance; password: string; - managedSecretArn: string; + secretArn: string; }> => { const rdsClient = new RDSClient({ region: config.region }); const params: CreateDBInstanceCommandInput = { @@ -71,10 +96,10 @@ export const createRDSInstance = async ( // use RDS managed password, then retrieve the password and store in all other credential store options ManageMasterUserPassword: !config.password, }; - const command = new CreateDBInstanceCommand(params); + const createInstanceCommand = new CreateDBInstanceCommand(params); try { - const rdsResponse = await rdsClient.send(command); + const createInstanceResponse = await rdsClient.send(createInstanceCommand); const availableResponse = await waitUntilDBInstanceAvailable( { @@ -99,7 +124,7 @@ export const createRDSInstance = async ( let password = config.password; let masterUserSecret; if (!config.password) { - masterUserSecret = rdsResponse.DBInstance?.MasterUserSecret; + masterUserSecret = createInstanceResponse.DBInstance?.MasterUserSecret; const secretsManagerClient = new SecretsManagerClient({ region: config.region }); const secretManagerCommand = new GetSecretValueCommand({ SecretId: masterUserSecret.SecretArn, @@ -118,7 +143,7 @@ export const createRDSInstance = async ( dbName: dbInstance.DBName as string, dbInstance, password, - managedSecretArn: masterUserSecret?.SecretArn, + secretArn: masterUserSecret?.SecretArn, }; } catch (error) { console.error(error); @@ -126,6 +151,97 @@ export const createRDSInstance = async ( } }; +/** + * Creates a new RDS Aurora serverless V2 cluster with one DB instance using the given input configuration. + * @param config Configuration of the database cluster. If password is not passed an RDS managed password will be created. + * @returns EndPoint address, port and database name of the created RDS cluster. + */ +export const createRDSCluster = async (config: RDSConfig): Promise => { + const rdsClient = new RDSClient({ region: config.region }); + const initialDBName = 'defaultdb'; + + const params: CreateDBClusterMessage = { + /** input parameters */ + EnableHttpEndpoint: true, + Engine: getRDSEngineType(config.engine), + DatabaseName: initialDBName, + DBClusterIdentifier: config.identifier, + MasterUsername: config.username, + // use RDS managed password, then retrieve the password and store in all other credential store options + ManageMasterUserPassword: true, + ServerlessV2ScalingConfiguration: { + MinCapacity: 4, + MaxCapacity: 10, + }, + }; + + const createClusterCommand = new CreateDBClusterCommand(params); + + const instanceParams: CreateDBInstanceCommandInput = { + DBInstanceClass: 'db.serverless', + DBInstanceIdentifier: createInstanceIdentifier(config.identifier), + Engine: getRDSEngineType(config.engine), + DBClusterIdentifier: config.identifier, + PubliclyAccessible: config.publiclyAccessible ?? true, + }; + + const instanceCommand = new CreateDBInstanceCommand(instanceParams); + + try { + const createClusterResponse = await rdsClient.send(createClusterCommand); + + const availableResponse = await waitUntilDBClusterAvailable( + { + maxWaitTime: 3600, + maxDelay: 120, + minDelay: 60, + client: rdsClient, + }, + { + DBClusterIdentifier: config.identifier, + }, + ); + + if (availableResponse.state !== 'SUCCESS') { + throw new Error('Error in creating a new RDS cluster.'); + } + + const dbCluster = availableResponse.reason.DBClusters[0]; + if (!dbCluster) { + throw new Error('RDS cluster details are missing.'); + } + + const instanceResponse = await rdsClient.send(instanceCommand); + const availableInstanceResponse = await waitUntilDBInstanceAvailable( + { + maxWaitTime: 3600, + maxDelay: 120, + minDelay: 60, + client: rdsClient, + }, + { + DBInstanceIdentifier: instanceParams.DBInstanceIdentifier, + }, + ); + + if (availableInstanceResponse.state !== 'SUCCESS') { + throw new Error('Error in creating a new RDS instance inside the cluster.'); + } + + return { + clusterArn: dbCluster.DBClusterArn as string, + endpoint: dbCluster.Endpoint as string, + port: dbCluster.Port as number, + dbName: dbCluster.DatabaseName as string, + secretArn: createClusterResponse.DBCluster.MasterUserSecret.SecretArn, + dbInstance: instanceResponse?.DBInstance, + }; + } catch (error) { + console.error(error); + throw new Error('Error in creating RDS cluster with an instance.'); + } +}; + /** * Creates a new RDS instance using the given input configuration, runs the given queries and returns the details of the created RDS * instance. @@ -136,7 +252,7 @@ export const createRDSInstance = async ( export const setupRDSInstanceAndData = async ( config: RDSConfig, queries?: string[], -): Promise<{ endpoint: string; port: number; dbName: string; dbInstance: DBInstance; password: string; managedSecretArn: string }> => { +): Promise<{ endpoint: string; port: number; dbName: string; dbInstance: DBInstance; password: string; secretArn: string }> => { console.log(`Creating RDS ${config.engine} instance with identifier ${config.identifier}`); const dbConfig = await createRDSInstance(config); @@ -185,6 +301,69 @@ export const setupRDSInstanceAndData = async ( return dbConfig; }; +/** + * Creates a new RDS Aurora serverless V2 cluster with one DB instance using the given input configuration, runs the given queries and returns the details of the created RDS + * instance. + * @param config Configuration of the database cluster + * @param queries Initial queries to be executed + * @returns Endpoint address, port and database name of the created RDS cluster. + */ + +export const setupRDSClusterAndData = async (config: RDSConfig, queries?: string[]): Promise => { + console.log(`Creating RDS ${config.engine} DB cluster with identifier ${config.identifier}`); + + const dbCluster = await createRDSCluster(config); + + if (!dbCluster.secretArn) { + throw new Error('Failed to store db connection config in secrets manager'); + } + + const client = new RDSDataClient({ region: config.region }); + + // create a new test database with given name + const sanitizedDbName = config.dbname.replace(/[^a-zA-Z0-9_]/g, ''); + + const createDBInput: ExecuteStatementCommandInput = { + resourceArn: dbCluster.clusterArn, + secretArn: dbCluster.secretArn, + sql: `create database ${sanitizedDbName}`, + database: dbCluster.dbName, + }; + + const createDBCommand = new ExecuteStatementCommand(createDBInput); + try { + const createDBResponse = await client.send(createDBCommand); + console.log('Create database response: ' + JSON.stringify(createDBResponse)); + } catch (err) { + console.log(err); + } + + // create the test tables in the test database + queries?.map(async (query) => { + try { + const executeStatementInput: ExecuteStatementCommandInput = { + resourceArn: dbCluster.clusterArn, + secretArn: dbCluster.secretArn, + sql: query, + database: sanitizedDbName, + }; + const executeStatementResponse = await client.send(new ExecuteStatementCommand(executeStatementInput)); + console.log('Create table response: ' + JSON.stringify(executeStatementResponse)); + } catch (err) { + throw new Error(`Error in creating tables in test database: ${err.response.json}`); + } + }); + + return { + clusterArn: dbCluster.clusterArn, + endpoint: dbCluster.endpoint, + port: dbCluster.port, + dbName: sanitizedDbName, + dbInstance: dbCluster.dbInstance, + secretArn: dbCluster.secretArn, + }; +}; + /** * Deletes the given RDS instance * @param identifier RDS Instance identifier to delete @@ -217,10 +396,43 @@ export const deleteDBInstance = async (identifier: string, region: string): Prom // ); } catch (error) { console.log(error); - throw new Error('Error in deleting RDS instance.'); + throw new Error(`Error in deleting RDS instance: ${error.response.json}`); + } +}; + +/** + * Deletes the given RDS cluster and instances it contains + * @param identifier RDS cluster identifier to delete + * @param region RDS cluster region + */ +export const deleteDBCluster = async (identifier: string, region: string): Promise => { + // First the instance deletion is triggered + const instanceID = createInstanceIdentifier(identifier); + console.log(`Deleting instance: ${instanceID}`); + await deleteDBInstance(instanceID, region); + + // Now delete the cluster + const client = new RDSClient({ region }); + const params: DeleteDBClusterCommandInput = { + DBClusterIdentifier: identifier, + SkipFinalSnapshot: true, + }; + console.log(`Deleting cluster: ${identifier}`); + const command = new DeleteDBClusterCommand(params); + try { + await client.send(command); + } catch (error) { + console.log(error); + throw new Error(`Error in deleting RDS cluster ${identifier}: ${error.response.json}`); } }; +const createInstanceIdentifier = (prefix: string) => { + return `${prefix}instance`; +}; + +export const generateDBName = () => generator.generate({ length: 8 }).toLowerCase(); + /** * Adds the given inbound rule to the security group. * @param config Inbound rule configuration diff --git a/packages/amplify-e2e-core/src/utils/test-regions.ts b/packages/amplify-e2e-core/src/utils/test-regions.ts new file mode 100644 index 0000000000..29e14415bb --- /dev/null +++ b/packages/amplify-e2e-core/src/utils/test-regions.ts @@ -0,0 +1,16 @@ +import path from 'path'; +import fs from 'fs-extra'; + +type TestRegion = { + name: string; + optIn: boolean; +}; + +export const isOptInRegion = (region: string): boolean => { + const repoRoot = path.join(__dirname, '..', '..', '..', '..'); + const supportedRegionsPath = path.join(repoRoot, 'scripts', 'e2e-test-regions.json'); + const supportedRegions: TestRegion[] = JSON.parse(fs.readFileSync(supportedRegionsPath, 'utf-8')); + const specificRegion = supportedRegions.find((testRegion) => testRegion.name == region); + + return specificRegion.optIn; +}; diff --git a/packages/amplify-graphql-api-construct-tests/src/__tests__/relationships/postgres-uuid-pk/uuid-pk-ddbprimary-sqlrelated.test.ts b/packages/amplify-graphql-api-construct-tests/src/__tests__/relationships/postgres-uuid-pk/uuid-pk-ddbprimary-sqlrelated.test.ts index d3fca4e6db..720d7db167 100644 --- a/packages/amplify-graphql-api-construct-tests/src/__tests__/relationships/postgres-uuid-pk/uuid-pk-ddbprimary-sqlrelated.test.ts +++ b/packages/amplify-graphql-api-construct-tests/src/__tests__/relationships/postgres-uuid-pk/uuid-pk-ddbprimary-sqlrelated.test.ts @@ -2,7 +2,7 @@ import * as path from 'path'; import * as fs from 'fs-extra'; import * as generator from 'generate-password'; -import { createNewProjectDir, deleteProjectDir } from 'amplify-category-api-e2e-core'; +import { createNewProjectDir, deleteProjectDir, generateDBName } from 'amplify-category-api-e2e-core'; import { DDB_AMPLIFY_MANAGED_DATASOURCE_STRATEGY } from '@aws-amplify/graphql-transformer-core'; import { cdkDeploy, cdkDestroy, initCDKProject } from '../../../commands'; import { SqlDatabaseDetails, SqlDatatabaseController } from '../../../sql-datatabase-controller'; @@ -23,8 +23,8 @@ describe('PostgreSQL tables with UUID primary keys', () => { const region = process.env.CLI_REGION ?? 'us-west-2'; const baseProjFolderName = path.basename(__filename, '.test.ts'); + const dbname = generateDBName(); const [dbUsername, dbIdentifier] = generator.generateMultiple(2); - const dbname = 'postgres'; let dbDetails: SqlDatabaseDetails; // Note that the SQL database is created with slightly non-standard naming conventions, to avoid us having to use `refersTo` in the schema diff --git a/packages/amplify-graphql-api-construct-tests/src/__tests__/relationships/postgres-uuid-pk/uuid-pk-sqlprimary-ddbrelated.test.ts b/packages/amplify-graphql-api-construct-tests/src/__tests__/relationships/postgres-uuid-pk/uuid-pk-sqlprimary-ddbrelated.test.ts index cff6068947..32d6e9156c 100644 --- a/packages/amplify-graphql-api-construct-tests/src/__tests__/relationships/postgres-uuid-pk/uuid-pk-sqlprimary-ddbrelated.test.ts +++ b/packages/amplify-graphql-api-construct-tests/src/__tests__/relationships/postgres-uuid-pk/uuid-pk-sqlprimary-ddbrelated.test.ts @@ -2,7 +2,7 @@ import * as path from 'path'; import * as fs from 'fs-extra'; import * as generator from 'generate-password'; -import { createNewProjectDir, deleteProjectDir } from 'amplify-category-api-e2e-core'; +import { createNewProjectDir, deleteProjectDir, generateDBName } from 'amplify-category-api-e2e-core'; import { DDB_AMPLIFY_MANAGED_DATASOURCE_STRATEGY } from '@aws-amplify/graphql-transformer-core'; import { cdkDeploy, cdkDestroy, initCDKProject } from '../../../commands'; import { SqlDatabaseDetails, SqlDatatabaseController } from '../../../sql-datatabase-controller'; @@ -23,8 +23,8 @@ describe('PostgreSQL tables with UUID primary keys', () => { const region = process.env.CLI_REGION ?? 'us-west-2'; const baseProjFolderName = path.basename(__filename, '.test.ts'); + const dbname = generateDBName(); const [dbUsername, dbIdentifier] = generator.generateMultiple(2); - const dbname = 'postgres'; let dbDetails: SqlDatabaseDetails; // Note that the SQL database is created with slightly non-standard naming conventions, to avoid us having to use `refersTo` in the schema diff --git a/packages/amplify-graphql-api-construct-tests/src/__tests__/relationships/postgres-uuid-pk/uuid-pk-sqlprimary-sqlrelated.test.ts b/packages/amplify-graphql-api-construct-tests/src/__tests__/relationships/postgres-uuid-pk/uuid-pk-sqlprimary-sqlrelated.test.ts index 7e6cdb1288..934201aaaf 100644 --- a/packages/amplify-graphql-api-construct-tests/src/__tests__/relationships/postgres-uuid-pk/uuid-pk-sqlprimary-sqlrelated.test.ts +++ b/packages/amplify-graphql-api-construct-tests/src/__tests__/relationships/postgres-uuid-pk/uuid-pk-sqlprimary-sqlrelated.test.ts @@ -2,7 +2,7 @@ import * as path from 'path'; import * as fs from 'fs-extra'; import * as generator from 'generate-password'; -import { createNewProjectDir, deleteProjectDir } from 'amplify-category-api-e2e-core'; +import { createNewProjectDir, deleteProjectDir, generateDBName } from 'amplify-category-api-e2e-core'; import { cdkDeploy, cdkDestroy, initCDKProject } from '../../../commands'; import { SqlDatabaseDetails, SqlDatatabaseController } from '../../../sql-datatabase-controller'; import { TestDefinition, dbDetailsToModelDataSourceStrategy, writeStackConfig, writeTestDefinitions } from '../../../utils'; @@ -22,8 +22,8 @@ describe('PostgreSQL tables with UUID primary keys', () => { const region = process.env.CLI_REGION ?? 'us-west-2'; const baseProjFolderName = path.basename(__filename, '.test.ts'); + const dbname = generateDBName(); const [dbUsername, dbIdentifier] = generator.generateMultiple(2); - const dbname = 'postgres'; let dbDetails: SqlDatabaseDetails; // Note that the SQL database is created with slightly non-standard naming conventions, to avoid us having to use `refersTo` in the schema diff --git a/packages/amplify-graphql-api-construct-tests/src/__tests__/sql-pg-canary.test.ts b/packages/amplify-graphql-api-construct-tests/src/__tests__/sql-pg-canary.test.ts index 6f412f7f7a..8453be4e06 100644 --- a/packages/amplify-graphql-api-construct-tests/src/__tests__/sql-pg-canary.test.ts +++ b/packages/amplify-graphql-api-construct-tests/src/__tests__/sql-pg-canary.test.ts @@ -1,4 +1,4 @@ -import { createNewProjectDir, deleteProjectDir } from 'amplify-category-api-e2e-core'; +import { createNewProjectDir, deleteProjectDir, generateDBName } from 'amplify-category-api-e2e-core'; import generator from 'generate-password'; import { getResourceNamesForStrategyName } from '@aws-amplify/graphql-transformer-core'; import { SqlDatatabaseController } from '../sql-datatabase-controller'; @@ -11,9 +11,10 @@ jest.setTimeout(DURATION_1_HOUR); describe('Canary using Postgres lambda model datasource strategy', () => { let projRoot: string; const projFolderName = 'pgcanary'; - const [username, password, identifier] = generator.generateMultiple(3); - const region = process.env.CLI_REGION; - const dbname = 'default_db'; + // sufficient password length that meets the requirements for RDS cluster/instance + const [username, password, identifier] = generator.generateMultiple(3, { length: 11 }); + const region = process.env.CLI_REGION ?? 'us-west-2'; + const dbname = generateDBName(); const engine = 'postgres'; const databaseController: SqlDatatabaseController = new SqlDatatabaseController( diff --git a/packages/amplify-graphql-api-construct-tests/src/__tests__/sql-pg-models.test.ts b/packages/amplify-graphql-api-construct-tests/src/__tests__/sql-pg-models.test.ts index 0102ac52fb..7034bd5896 100644 --- a/packages/amplify-graphql-api-construct-tests/src/__tests__/sql-pg-models.test.ts +++ b/packages/amplify-graphql-api-construct-tests/src/__tests__/sql-pg-models.test.ts @@ -1,4 +1,4 @@ -import { createNewProjectDir, deleteProjectDir } from 'amplify-category-api-e2e-core'; +import { createNewProjectDir, deleteProjectDir, generateDBName } from 'amplify-category-api-e2e-core'; import generator from 'generate-password'; import { getResourceNamesForStrategyName } from '@aws-amplify/graphql-transformer-core'; import { SqlDatatabaseController } from '../sql-datatabase-controller'; @@ -12,11 +12,10 @@ describe('CDK GraphQL Transformer deployments with Postgres SQL datasources', () let projRoot: string; const projFolderName = 'pgmodels'; - const [username, password, identifier] = generator.generateMultiple(3); - + // sufficient password length that meets the requirements for RDS cluster/instance + const [username, password, identifier] = generator.generateMultiple(3, { length: 11 }); const region = process.env.CLI_REGION ?? 'us-west-2'; - - const dbname = 'default_db'; + const dbname = generateDBName(); const engine = 'postgres'; const databaseController: SqlDatatabaseController = new SqlDatatabaseController( diff --git a/packages/amplify-graphql-api-construct-tests/src/sql-datatabase-controller.ts b/packages/amplify-graphql-api-construct-tests/src/sql-datatabase-controller.ts index 94a14c4945..0063ac51a0 100644 --- a/packages/amplify-graphql-api-construct-tests/src/sql-datatabase-controller.ts +++ b/packages/amplify-graphql-api-construct-tests/src/sql-datatabase-controller.ts @@ -9,10 +9,14 @@ import { RDSConfig, SqlEngine, setupRDSInstanceAndData, + setupRDSClusterAndData, storeDbConnectionConfig, storeDbConnectionStringConfig, storeDbConnectionConfigWithSecretsManager, + deleteDBCluster, + isOptInRegion, } from 'amplify-category-api-e2e-core'; +import { SecretsManagerClient, CreateSecretCommand, DeleteSecretCommand, GetSecretValueCommand } from '@aws-sdk/client-secrets-manager'; import { isSqlModelDataSourceSecretsManagerDbConnectionConfig, isSqlModelDataSourceSsmDbConnectionConfig, @@ -45,38 +49,55 @@ export interface SqlDatabaseDetails { */ export class SqlDatatabaseController { private databaseDetails: SqlDatabaseDetails | undefined; + private useDataAPI: boolean; - constructor(private readonly setupQueries: Array, private readonly options: RDSConfig) {} + constructor(private readonly setupQueries: Array, private readonly options: RDSConfig) { + // Data API is not supported in opted-in regions + if (options.engine === 'postgres' && !isOptInRegion(options.region)) { + this.useDataAPI = true; + } else { + this.useDataAPI = false; + } + } setupDatabase = async (): Promise => { - console.log(`Setting up database '${this.options.identifier}'`); + let dbConfig; + if (this.useDataAPI) { + dbConfig = await setupRDSClusterAndData(this.options, this.setupQueries); + } else { + dbConfig = await setupRDSInstanceAndData(this.options, this.setupQueries); + } - const dbConfig = await setupRDSInstanceAndData(this.options, this.setupQueries); if (!dbConfig) { throw new Error('Failed to setup RDS instance'); } - const { secretArn } = await storeDbConnectionConfigWithSecretsManager({ - region: this.options.region, - username: this.options.username, - password: dbConfig.password, - secretName: `${this.options.identifier}-secret`, - }); - if (!secretArn) { - throw new Error('Failed to store db connection config for secrets manager'); - } const dbConnectionConfigSecretsManager = { databaseName: this.options.dbname, hostname: dbConfig.endpoint, port: dbConfig.port, - secretArn, + secretArn: dbConfig.secretArn, }; console.log(`Stored db connection config in Secrets manager: ${JSON.stringify(dbConnectionConfigSecretsManager)}`); + if (this.useDataAPI || !this.options.password) { + const secretArn = dbConfig.secretArn; + const secretsManagerClient = new SecretsManagerClient({ region: this.options.region }); + const secretManagerCommand = new GetSecretValueCommand({ + SecretId: secretArn, + }); + const secretsManagerResponse = await secretsManagerClient.send(secretManagerCommand); + const { password: managedPassword } = JSON.parse(secretsManagerResponse.SecretString); + if (!managedPassword) { + throw new Error('Unable to get RDS cluster master user password'); + } + this.options.password = managedPassword; + } + const { secretArn: secretArnWithCustomKey, keyArn } = await storeDbConnectionConfigWithSecretsManager({ region: this.options.region, username: this.options.username, - password: dbConfig.password, + password: this.options.password, secretName: `${this.options.identifier}-secret-custom-key`, useCustomEncryptionKey: true, }); @@ -101,7 +122,7 @@ export class SqlDatatabaseController { port: dbConfig.port, databaseName: this.options.dbname, username: this.options.username, - password: dbConfig.password, + password: this.options.password, }); const dbConnectionStringConfigSSM = await storeDbConnectionStringConfig({ region: this.options.region, @@ -109,7 +130,7 @@ export class SqlDatatabaseController { connectionUri: this.getConnectionUri( engine, this.options.username, - dbConfig.password, + this.options.password, dbConfig.endpoint, dbConfig.port, this.options.dbname, @@ -120,7 +141,7 @@ export class SqlDatatabaseController { pathPrefix, connectionUri: [ 'mysql://username:password@host:3306/dbname', - this.getConnectionUri(engine, this.options.username, dbConfig.password, dbConfig.endpoint, dbConfig.port, this.options.dbname), + this.getConnectionUri(engine, this.options.username, this.options.password, dbConfig.endpoint, dbConfig.port, this.options.dbname), ], }); const parameters = { @@ -150,7 +171,7 @@ export class SqlDatatabaseController { databaseName: this.options.dbname, hostname: dbConfig.endpoint, port: dbConfig.port, - secretArn: dbConfig.managedSecretArn, + secretArn: dbConfig.secretArn, }, connectionUri: dbConnectionStringConfigSSM, connectionUriMultiple: dbConnectionStringConfigMultiple, @@ -166,7 +187,11 @@ export class SqlDatatabaseController { return; } - await deleteDBInstance(this.options.identifier, this.options.region); + if (this.useDataAPI) { + await deleteDBCluster(this.options.identifier, this.options.region); + } else { + await deleteDBInstance(this.options.identifier, this.options.region); + } const { connectionConfigs } = this.databaseDetails; diff --git a/yarn.lock b/yarn.lock index 936523d65f..3c6421ad77 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1758,6 +1758,48 @@ "@aws-sdk/util-utf8-node" "3.6.1" tslib "^2.0.0" +"@aws-sdk/client-rds-data@3.338.0": + version "3.338.0" + resolved "https://registry.npmjs.org/@aws-sdk/client-rds-data/-/client-rds-data-3.338.0.tgz#1a4e3950f85348e90c18781a75aae32e59d3f8f1" + integrity sha512-ys4kWGrEnagY1tgshV8QRKuv562I7xZ6BStSOgJS05zz6baOrYKpMeZYBwT5eVZ/Ua7K9lCHhYC4WC34ID6QVA== + dependencies: + "@aws-crypto/sha256-browser" "3.0.0" + "@aws-crypto/sha256-js" "3.0.0" + "@aws-sdk/client-sts" "3.338.0" + "@aws-sdk/config-resolver" "3.338.0" + "@aws-sdk/credential-provider-node" "3.338.0" + "@aws-sdk/fetch-http-handler" "3.338.0" + "@aws-sdk/hash-node" "3.338.0" + "@aws-sdk/invalid-dependency" "3.338.0" + "@aws-sdk/middleware-content-length" "3.338.0" + "@aws-sdk/middleware-endpoint" "3.338.0" + "@aws-sdk/middleware-host-header" "3.338.0" + "@aws-sdk/middleware-logger" "3.338.0" + "@aws-sdk/middleware-recursion-detection" "3.338.0" + "@aws-sdk/middleware-retry" "3.338.0" + "@aws-sdk/middleware-serde" "3.338.0" + "@aws-sdk/middleware-signing" "3.338.0" + "@aws-sdk/middleware-stack" "3.338.0" + "@aws-sdk/middleware-user-agent" "3.338.0" + "@aws-sdk/node-config-provider" "3.338.0" + "@aws-sdk/node-http-handler" "3.338.0" + "@aws-sdk/smithy-client" "3.338.0" + "@aws-sdk/types" "3.338.0" + "@aws-sdk/url-parser" "3.338.0" + "@aws-sdk/util-base64" "3.310.0" + "@aws-sdk/util-body-length-browser" "3.310.0" + "@aws-sdk/util-body-length-node" "3.310.0" + "@aws-sdk/util-defaults-mode-browser" "3.338.0" + "@aws-sdk/util-defaults-mode-node" "3.338.0" + "@aws-sdk/util-endpoints" "3.338.0" + "@aws-sdk/util-retry" "3.338.0" + "@aws-sdk/util-user-agent-browser" "3.338.0" + "@aws-sdk/util-user-agent-node" "3.338.0" + "@aws-sdk/util-utf8" "3.310.0" + "@smithy/protocol-http" "^1.0.1" + "@smithy/types" "^1.0.0" + tslib "^2.5.0" + "@aws-sdk/client-rds@3.338.0": version "3.338.0" resolved "https://registry.yarnpkg.com/@aws-sdk/client-rds/-/client-rds-3.338.0.tgz#6b20b97e9a638e7dee72ed201a1f729e0cd98383"