diff --git a/deployment/cdk/opensearch-service-migration/dp_pipeline_template.yaml b/deployment/cdk/opensearch-service-migration/dp_pipeline_template.yaml index 9baec2325c..646ee2dc98 100644 --- a/deployment/cdk/opensearch-service-migration/dp_pipeline_template.yaml +++ b/deployment/cdk/opensearch-service-migration/dp_pipeline_template.yaml @@ -1,18 +1,36 @@ +# Name of the Data Prepper pipeline historical-data-migration: + # Source cluster configuration source: opensearch: + # DO NOT CHANGE the hosts value - this will be updated by the CDK hosts: - + # Uncomment the following line to disable TLS + #insecure: true + # Example configuration on how to disable authentication (default: false) disable_authentication: true indices: + # Indices to exclude - exclude system indices by default exclude: - index_name_regex: \.* + # Target cluster configuration sink: - opensearch: - bulk_size: 10 + # DO NOT CHANGE the hosts value - this will be updated by the CDK + # But adjust the protocol (http or https) as appropriate hosts: - https:// + # Derive index name from record metadata index: ${getMetadata("opensearch-index")} + # Use the same document ID as the source cluster document document_id: ${getMetadata("opensearch-document_id")} + # Example configuration for basic auth username: user password: pass + #disable_authentication: true + # Additional pipeline options/optimizations + # For maximum throughput, match workers to number of vCPUs (default: 1) + workers: 1 + # delay is how often the worker threads should process data (default: 3000 ms) + delay: 0 diff --git a/deployment/cdk/opensearch-service-migration/lib/fetch-migration-stack.ts b/deployment/cdk/opensearch-service-migration/lib/fetch-migration-stack.ts index 8a18e6a879..824cdc44a9 100644 --- a/deployment/cdk/opensearch-service-migration/lib/fetch-migration-stack.ts +++ b/deployment/cdk/opensearch-service-migration/lib/fetch-migration-stack.ts @@ -26,7 +26,7 @@ export class FetchMigrationStack extends Stack { super(scope, id, props); // Import required values - const targetClusterEndpoint = StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/osClusterEndpoint`) + const targetClusterEndpoint = StringParameter.valueFromLookup(this, `/migration/${props.stage}/${props.defaultDeployId}/osClusterEndpoint`) const domainAccessGroupId = StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/osAccessSecurityGroupId`) // This SG allows outbound access for ECR access as well as communication with other services in the cluster const serviceConnectGroupId = StringParameter.valueForStringParameter(this, `/migration/${props.stage}/${props.defaultDeployId}/serviceConnectSecurityGroupId`) @@ -38,8 +38,8 @@ export class FetchMigrationStack extends Stack { // ECS Task Definition const fetchMigrationFargateTask = new FargateTaskDefinition(this, "fetchMigrationFargateTask", { - memoryLimitMiB: 2048, - cpu: 512 + memoryLimitMiB: 4096, + cpu: 1024 }); new StringParameter(this, 'SSMParameterFetchMigrationTaskDefArn', {