diff --git a/.gitignore b/.gitignore index ac3afd58..249ee8ac 100644 --- a/.gitignore +++ b/.gitignore @@ -65,6 +65,15 @@ test/patterns/gen-ai/aws-aoss-cw-dashboard/integ-tests/aws-aoss-cw-dashboard.int test/patterns/gen-ai/aws-aoss-cw-dashboard/integ-tests/aws-aoss-cw-dashboard.integ.snapshot/**/manifest.json test/patterns/gen-ai/aws-aoss-cw-dashboard/integ-tests/aws-aoss-cw-dashboard.integ.snapshot/tree.json test/patterns/gen-ai/aws-aoss-cw-dashboard/integ-tests/aws-aoss-cw-dashboard.integ.snapshot/**/tree.json +test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/.tmp +test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.snapshot/asset.* +test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.snapshot/**/asset.* +test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.snapshot/cdk.out +test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.snapshot/**/cdk.out +test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.snapshot/manifest.json +test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.snapshot/**/manifest.json +test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.snapshot/tree.json +test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.snapshot/**/tree.json !/.github/workflows/github-merit-badger.yml !/.github/workflows/monthly-repo-metrics.yml !/.github/workflows/auto-approve.yml diff --git a/.npmignore b/.npmignore index b523d851..1f169094 100644 --- a/.npmignore +++ b/.npmignore @@ -24,6 +24,8 @@ tsconfig.tsbuildinfo !.jsii test/patterns/gen-ai/aws-aoss-cw-dashboard/integ-tests/.tmp test/patterns/gen-ai/aws-aoss-cw-dashboard/integ-tests/aws-aoss-cw-dashboard.integ.snapshot +test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/.tmp +test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.snapshot /docs/ header.js .eslintrc.json diff --git a/.projen/tasks.json b/.projen/tasks.json index 931ef94d..ed8c64df 100644 --- a/.projen/tasks.json +++ b/.projen/tasks.json @@ -227,12 +227,78 @@ } ] }, + "integ:aws-llama-index-data-loader:assert": { + "name": "integ:aws-llama-index-data-loader:assert", + "description": "assert the snapshot of integration test 'aws-llama-index-data-loader'", + "steps": [ + { + "exec": "[ -d \"test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.snapshot\" ] || (echo \"No snapshot available for integration test 'aws-llama-index-data-loader'. Run 'projen integ:aws-llama-index-data-loader:deploy' to capture.\" && exit 1)" + }, + { + "exec": "cdk synth --app \"ts-node -P tsconfig.dev.json test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.ts\" --no-notices --no-version-reporting --no-asset-metadata --no-path-metadata -o test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/.tmp/aws-llama-index-data-loader.integ/assert.cdk.out > /dev/null" + }, + { + "exec": "diff -r -x asset.* -x cdk.out -x manifest.json -x tree.json test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.snapshot/ test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/.tmp/aws-llama-index-data-loader.integ/assert.cdk.out/" + } + ] + }, + "integ:aws-llama-index-data-loader:deploy": { + "name": "integ:aws-llama-index-data-loader:deploy", + "description": "deploy integration test 'aws-llama-index-data-loader' and capture snapshot", + "steps": [ + { + "exec": "rm -fr test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/.tmp/aws-llama-index-data-loader.integ/deploy.cdk.out" + }, + { + "exec": "cdk deploy --app \"ts-node -P tsconfig.dev.json test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.ts\" --no-notices --no-version-reporting --no-asset-metadata --no-path-metadata '**' --require-approval=never -o test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/.tmp/aws-llama-index-data-loader.integ/deploy.cdk.out" + }, + { + "exec": "rm -fr test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.snapshot" + }, + { + "exec": "mv test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/.tmp/aws-llama-index-data-loader.integ/deploy.cdk.out test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.snapshot" + }, + { + "spawn": "integ:aws-llama-index-data-loader:destroy" + } + ] + }, + "integ:aws-llama-index-data-loader:destroy": { + "name": "integ:aws-llama-index-data-loader:destroy", + "description": "destroy integration test 'aws-llama-index-data-loader'", + "steps": [ + { + "exec": "cdk destroy --app test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.snapshot '**' --no-version-reporting" + } + ] + }, + "integ:aws-llama-index-data-loader:snapshot": { + "name": "integ:aws-llama-index-data-loader:snapshot", + "description": "update snapshot for integration test \"aws-llama-index-data-loader\"", + "steps": [ + { + "exec": "cdk synth --app \"ts-node -P tsconfig.dev.json test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.ts\" --no-notices --no-version-reporting --no-asset-metadata --no-path-metadata -o test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.snapshot > /dev/null" + } + ] + }, + "integ:aws-llama-index-data-loader:watch": { + "name": "integ:aws-llama-index-data-loader:watch", + "description": "watch integration test 'aws-llama-index-data-loader' (without updating snapshots)", + "steps": [ + { + "exec": "cdk watch --app \"ts-node -P tsconfig.dev.json test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.ts\" --no-notices --no-version-reporting --no-asset-metadata --no-path-metadata '**' -o test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/.tmp/aws-llama-index-data-loader.integ/deploy.cdk.out" + } + ] + }, "integ:snapshot-all": { "name": "integ:snapshot-all", "description": "update snapshot for all integration tests", "steps": [ { "spawn": "integ:aws-aoss-cw-dashboard:snapshot" + }, + { + "spawn": "integ:aws-llama-index-data-loader:snapshot" } ] }, @@ -375,6 +441,9 @@ }, { "spawn": "integ:aws-aoss-cw-dashboard:assert" + }, + { + "spawn": "integ:aws-llama-index-data-loader:assert" } ] }, diff --git a/README.md b/README.md index 01a4f9c0..ab08f240 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,8 @@ The following constructs are available in the library: | [Content Generation](./src/patterns/gen-ai/aws-contentgen-appsync-lambda/README.md) | Generate images from text using Amazon titan-image-generator-v1 or stability.stable-diffusion-xl-v1 model. | AWS Lambda, Amazon Bedrock, AWS AppSync | | [Web crawler](./src/patterns/gen-ai/aws-web-crawler/README.md) | Crawl websites and RSS feeds on a schedule and store changeset data in an Amazon Simple Storage Service bucket. | AWS Lambda, AWS Batch, AWS Fargate, Amazon DynamoDB | | [Amazon Bedrock Monitoring (Amazon CloudWatch Dashboard)](./src/patterns/gen-ai/aws-bedrock-cw-dashboard/README.md) | Amazon CloudWatch dashboard to monitor model usage from Amazon Bedrock. | Amazon CloudWatch | +| [TXT to SQL](./src/patterns/gen-ai/aws-text-to-sql/README.md) | Leverages generative AI capabilities to facilitate natural language-based SQL query generation. | Amazon Event Bridge, Amazon Bedrock, AWS Lambda, Amazon SQS, AWS Secrets, and database of choice | +| [LlamaIndex Data Loading](./src/patterns/gen-ai/aws-llama-index-data-loader/README.md) | Use LlamaIndex to load data in preparation for generative AI workloads | Amazon ECS Fargate, Amazon SQS, and AWS Systems Manager Parameters | ### L2 Constructs diff --git a/apidocs/README.md b/apidocs/README.md index b2871ae6..8cd51992 100644 --- a/apidocs/README.md +++ b/apidocs/README.md @@ -32,6 +32,7 @@ - [JumpStartModel](classes/JumpStartModel.md) - [JumpStartSageMakerEndpoint](classes/JumpStartSageMakerEndpoint.md) - [LangchainCommonDepsLayer](classes/LangchainCommonDepsLayer.md) +- [LlamaIndexDataLoader](classes/LlamaIndexDataLoader.md) - [QaAppsyncOpensearch](classes/QaAppsyncOpensearch.md) - [RagAppsyncStepfnOpensearch](classes/RagAppsyncStepfnOpensearch.md) - [SageMakerEndpointBase](classes/SageMakerEndpointBase.md) @@ -60,6 +61,7 @@ - [JumpStartSageMakerEndpointProps](interfaces/JumpStartSageMakerEndpointProps.md) - [LangchainLayerProps](interfaces/LangchainLayerProps.md) - [LangchainProps](interfaces/LangchainProps.md) +- [LlamaIndexDataLoaderProps](interfaces/LlamaIndexDataLoaderProps.md) - [ModelMonitoringProps](interfaces/ModelMonitoringProps.md) - [QaAppsyncOpensearchProps](interfaces/QaAppsyncOpensearchProps.md) - [RagAppsyncStepfnOpensearchProps](interfaces/RagAppsyncStepfnOpensearchProps.md) diff --git a/apidocs/classes/BaseClass.md b/apidocs/classes/BaseClass.md index d77f5bf1..47ed4e73 100644 --- a/apidocs/classes/BaseClass.md +++ b/apidocs/classes/BaseClass.md @@ -19,6 +19,7 @@ - [`ContentGenerationAppSyncLambda`](ContentGenerationAppSyncLambda.md) - [`WebCrawler`](WebCrawler.md) - [`TextToSql`](TextToSql.md) +- [`LlamaIndexDataLoader`](LlamaIndexDataLoader.md) ## Constructors diff --git a/apidocs/classes/LlamaIndexDataLoader.md b/apidocs/classes/LlamaIndexDataLoader.md new file mode 100644 index 00000000..6182c12d --- /dev/null +++ b/apidocs/classes/LlamaIndexDataLoader.md @@ -0,0 +1,271 @@ +[**@cdklabs/generative-ai-cdk-constructs**](../README.md) • **Docs** + +*** + +[@cdklabs/generative-ai-cdk-constructs](../README.md) / LlamaIndexDataLoader + +# Class: LlamaIndexDataLoader + +## Extends + +- [`BaseClass`](BaseClass.md) + +## Constructors + +### new LlamaIndexDataLoader() + +> **new LlamaIndexDataLoader**(`scope`, `id`, `props`): [`LlamaIndexDataLoader`](LlamaIndexDataLoader.md) + +#### Parameters + +• **scope**: `Construct` + +• **id**: `string` + +• **props**: [`LlamaIndexDataLoaderProps`](../interfaces/LlamaIndexDataLoaderProps.md) + +#### Returns + +[`LlamaIndexDataLoader`](LlamaIndexDataLoader.md) + +#### Overrides + +[`BaseClass`](BaseClass.md).[`constructor`](BaseClass.md#constructors) + +## Properties + +### constructUsageMetric + +> `readonly` **constructUsageMetric**: `"uksb-1tupboc45"` = `'uksb-1tupboc45'` + +construct usage metric , added in template description + +#### Inherited from + +[`BaseClass`](BaseClass.md).[`constructUsageMetric`](BaseClass.md#constructusagemetric) + +*** + +### enablexray + +> **enablexray**: `boolean` = `true` + +enable disable xray tracing + +#### Default + +```ts +- True +``` + +#### Inherited from + +[`BaseClass`](BaseClass.md).[`enablexray`](BaseClass.md#enablexray) + +*** + +### fieldLogLevel + +> **fieldLogLevel**: `FieldLogLevel` = `appsync.FieldLogLevel.ALL` + +Default log config for all constructs + +#### Inherited from + +[`BaseClass`](BaseClass.md).[`fieldLogLevel`](BaseClass.md#fieldloglevel) + +*** + +### lambdaTracing + +> **lambdaTracing**: `Tracing` = `lambda.Tracing.ACTIVE` + +enable disable lambda tracing + +#### Default + +```ts +- Active +``` + +#### Inherited from + +[`BaseClass`](BaseClass.md).[`lambdaTracing`](BaseClass.md#lambdatracing) + +*** + +### node + +> `readonly` **node**: `Node` + +The tree node. + +#### Inherited from + +[`BaseClass`](BaseClass.md).[`node`](BaseClass.md#node) + +*** + +### outputBucket + +> `readonly` **outputBucket**: `Bucket` + +*** + +### queueProcessingFargateService + +> `readonly` **queueProcessingFargateService**: `QueueProcessingFargateService` + +*** + +### retention + +> **retention**: `RetentionDays` = `logs.RetentionDays.TEN_YEARS` + +Default log retention config for all constructs + +#### Inherited from + +[`BaseClass`](BaseClass.md).[`retention`](BaseClass.md#retention) + +*** + +### stage + +> **stage**: `string` + +Value will be appended to resources name. + +#### Default + +```ts +- _dev +``` + +#### Inherited from + +[`BaseClass`](BaseClass.md).[`stage`](BaseClass.md#stage) + +*** + +### usageMetricMap + +> `protected` `static` **usageMetricMap**: `Record`\<`string`, `number`\> + +Record , maps construct name with number of deployments + +#### Inherited from + +[`BaseClass`](BaseClass.md).[`usageMetricMap`](BaseClass.md#usagemetricmap) + +## Methods + +### addObservabilityToConstruct() + +> `protected` **addObservabilityToConstruct**(`props`): `void` + +#### Parameters + +• **props**: [`BaseClassProps`](../interfaces/BaseClassProps.md) + +#### Returns + +`void` + +#### Inherited from + +[`BaseClass`](BaseClass.md).[`addObservabilityToConstruct`](BaseClass.md#addobservabilitytoconstruct) + +*** + +### toString() + +> **toString**(): `string` + +Returns a string representation of this construct. + +#### Returns + +`string` + +#### Inherited from + +[`BaseClass`](BaseClass.md).[`toString`](BaseClass.md#tostring) + +*** + +### updateConstructUsageMetricCode() + +> `protected` **updateConstructUsageMetricCode**(`props`, `scope`, `lambdaFunctions`): `void` + +#### Parameters + +• **props**: [`BaseClassProps`](../interfaces/BaseClassProps.md) + +• **scope**: `Construct` + +• **lambdaFunctions**: `DockerImageFunction`[] + +#### Returns + +`void` + +#### Inherited from + +[`BaseClass`](BaseClass.md).[`updateConstructUsageMetricCode`](BaseClass.md#updateconstructusagemetriccode) + +*** + +### updateEnvSuffix() + +> `protected` **updateEnvSuffix**(`props`): `void` + +#### Parameters + +• **props**: [`BaseClassProps`](../interfaces/BaseClassProps.md) + +#### Returns + +`void` + +#### Inherited from + +[`BaseClass`](BaseClass.md).[`updateEnvSuffix`](BaseClass.md#updateenvsuffix) + +*** + +### isConstruct() + +> `static` **isConstruct**(`x`): `x is Construct` + +Checks if `x` is a construct. + +Use this method instead of `instanceof` to properly detect `Construct` +instances, even when the construct library is symlinked. + +Explanation: in JavaScript, multiple copies of the `constructs` library on +disk are seen as independent, completely different libraries. As a +consequence, the class `Construct` in each copy of the `constructs` library +is seen as a different class, and an instance of one class will not test as +`instanceof` the other class. `npm install` will not create installations +like this, but users may manually symlink construct libraries together or +use a monorepo tool: in those cases, multiple copies of the `constructs` +library can be accidentally installed, and `instanceof` will behave +unpredictably. It is safest to avoid using `instanceof`, and using +this type-testing method instead. + +#### Parameters + +• **x**: `any` + +Any object + +#### Returns + +`x is Construct` + +true if `x` is an object created from a class which extends `Construct`. + +#### Inherited from + +[`BaseClass`](BaseClass.md).[`isConstruct`](BaseClass.md#isconstruct) diff --git a/apidocs/enumerations/ConstructName.md b/apidocs/enumerations/ConstructName.md index 43d54800..56e42c2a 100644 --- a/apidocs/enumerations/ConstructName.md +++ b/apidocs/enumerations/ConstructName.md @@ -25,6 +25,12 @@ Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. *** +### AWSLLAMAINDEXDATALOADER + +> **AWSLLAMAINDEXDATALOADER**: `"C12"` + +*** + ### AWSMODELDEPLOYMENTSAGEMAKER > **AWSMODELDEPLOYMENTSAGEMAKER**: `"C4"` diff --git a/apidocs/interfaces/LlamaIndexDataLoaderProps.md b/apidocs/interfaces/LlamaIndexDataLoaderProps.md new file mode 100644 index 00000000..2a16ced1 --- /dev/null +++ b/apidocs/interfaces/LlamaIndexDataLoaderProps.md @@ -0,0 +1,120 @@ +[**@cdklabs/generative-ai-cdk-constructs**](../README.md) • **Docs** + +*** + +[@cdklabs/generative-ai-cdk-constructs](../README.md) / LlamaIndexDataLoaderProps + +# Interface: LlamaIndexDataLoaderProps + +## Properties + +### containerLoggingLevel? + +> `readonly` `optional` **containerLoggingLevel**: `string` + +#### Description + +the container's logging level + +#### Default + +```ts +'WARNING' +``` + +*** + +### dockerImageAssetDirectory? + +> `readonly` `optional` **dockerImageAssetDirectory**: `string` + +The directory to build the Docker image + +#### Description + +The directory to build the Docker image. + +#### Default + +```ts +__dirname + '/docker' +``` + +*** + +### memoryLimitMiB? + +> `readonly` `optional` **memoryLimitMiB**: `number` + +The default memory + +#### Description + +The default memory. + +#### Default + +```ts +2048 +``` + +*** + +### observability? + +> `readonly` `optional` **observability**: `boolean` + +Enable observability. Warning: associated cost with the services +used. Best practive to enable by default. + +#### Default + +```ts +- true +``` + +*** + +### outputBucket? + +> `readonly` `optional` **outputBucket**: `Bucket` + +#### Description + +the S3 output to use + +#### Default + +```ts +undefined +``` + +*** + +### stage? + +> `readonly` `optional` **stage**: `string` + +Value will be appended to resources name. + +#### Default + +```ts +- _dev +``` + +*** + +### vpc? + +> `readonly` `optional` **vpc**: `IVpc` + +#### Description + +the VPC to use + +#### Default + +```ts +undefined +``` diff --git a/docs/generative_ai_cdk_constructs.drawio b/docs/generative_ai_cdk_constructs.drawio index d2616e52..0f97752d 100644 --- a/docs/generative_ai_cdk_constructs.drawio +++ b/docs/generative_ai_cdk_constructs.drawio @@ -1,4 +1,4 @@ - + @@ -1012,4 +1012,68 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/package.json b/package.json index 93f54597..92f62541 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,11 @@ "integ:aws-aoss-cw-dashboard:destroy": "npx projen integ:aws-aoss-cw-dashboard:destroy", "integ:aws-aoss-cw-dashboard:snapshot": "npx projen integ:aws-aoss-cw-dashboard:snapshot", "integ:aws-aoss-cw-dashboard:watch": "npx projen integ:aws-aoss-cw-dashboard:watch", + "integ:aws-llama-index-data-loader:assert": "npx projen integ:aws-llama-index-data-loader:assert", + "integ:aws-llama-index-data-loader:deploy": "npx projen integ:aws-llama-index-data-loader:deploy", + "integ:aws-llama-index-data-loader:destroy": "npx projen integ:aws-llama-index-data-loader:destroy", + "integ:aws-llama-index-data-loader:snapshot": "npx projen integ:aws-llama-index-data-loader:snapshot", + "integ:aws-llama-index-data-loader:watch": "npx projen integ:aws-llama-index-data-loader:watch", "integ:snapshot-all": "npx projen integ:snapshot-all", "package": "npx projen package", "package-all": "npx projen package-all", diff --git a/resources/gen-ai/aws-llama-index-data-loader/docker/Dockerfile b/resources/gen-ai/aws-llama-index-data-loader/docker/Dockerfile new file mode 100644 index 00000000..22289ef0 --- /dev/null +++ b/resources/gen-ai/aws-llama-index-data-loader/docker/Dockerfile @@ -0,0 +1,46 @@ +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. +# + +ARG PYTHON_TAG="latest" +# RedundantTargetPlatform check violation +# https://docs.docker.com/reference/build-checks/redundant-target-platform/ +FROM --platform=${BUILDPLATFORM} public.ecr.aws/docker/library/python:${PYTHON_TAG} AS build + +RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get -y install dnsutils +ENV PYTHONUNBUFFERED=1 +# TODO: set verbosity by argument rather than hardcoded. +ENV PYTHONVERBOSE=0 +# 3 + +# matches the current machine. (e.g. linux/amd64) +RUN echo "BUILDPLATFORM - matches the current machine ${BUILDPLATFORM}" +RUN echo ${BUILDOS} +RUN echo ${BUILDARCH} +RUN echo ${BUILDVARIANT} +RUN echo "TARGETPLATFORM — The value set with --platform flag on build ${TARGETPLATFORM}" +RUN echo ${TARGETOS} +RUN echo ${TARGETARCH} +RUN echo ${TARGETVARIANT} + +WORKDIR /usr/src/app + +COPY requirements.txt ./ +RUN pip install --no-cache-dir --requirement requirements.txt + +COPY . . + +RUN chmod a+x runner.sh +RUN chmod a+x healthcheck.sh + +HEALTHCHECK --interval=30s --timeout=30s --start-period=5s --retries=3 CMD [ "/usr/src/app/healthcheck.sh" ] + +ENTRYPOINT [ "/usr/src/app/runner.sh", "/usr/src/app/sqs_consumer.py" ] \ No newline at end of file diff --git a/resources/gen-ai/aws-llama-index-data-loader/docker/__init__.py b/resources/gen-ai/aws-llama-index-data-loader/docker/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/resources/gen-ai/aws-llama-index-data-loader/docker/healthcheck.sh b/resources/gen-ai/aws-llama-index-data-loader/docker/healthcheck.sh new file mode 100755 index 00000000..9c7c3f73 --- /dev/null +++ b/resources/gen-ai/aws-llama-index-data-loader/docker/healthcheck.sh @@ -0,0 +1,8 @@ +#!/bin/sh +set -e + +WORKING_DIRECTORY="/usr/src/app"; + +# Look for the process +pgrep --full "/bin/sh ${WORKING_DIRECTORY}/runner.sh ${WORKING_DIRECTORY}/sqs_consumer.py" 2> /dev/null > /dev/null; +exit ${?}; \ No newline at end of file diff --git a/resources/gen-ai/aws-llama-index-data-loader/docker/requirements.txt b/resources/gen-ai/aws-llama-index-data-loader/docker/requirements.txt new file mode 100644 index 00000000..0aba381e --- /dev/null +++ b/resources/gen-ai/aws-llama-index-data-loader/docker/requirements.txt @@ -0,0 +1,4 @@ +boto3==1.35.36 +cachetools==5.5.0 +jsonpickle==3.3.0 +llama-index-readers-s3==0.2.0 diff --git a/resources/gen-ai/aws-llama-index-data-loader/docker/runner.sh b/resources/gen-ai/aws-llama-index-data-loader/docker/runner.sh new file mode 100755 index 00000000..04cfcc2b --- /dev/null +++ b/resources/gen-ai/aws-llama-index-data-loader/docker/runner.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. +# + +set -e + +echo "${$}" > pid; +echo "STARTING ${0} ${@}..."; +python3 -u ${@}; +echo "...ENDING ${0} ${@}"; +exit 0; \ No newline at end of file diff --git a/resources/gen-ai/aws-llama-index-data-loader/docker/sqs_consumer.py b/resources/gen-ai/aws-llama-index-data-loader/docker/sqs_consumer.py new file mode 100644 index 00000000..aa1ae461 --- /dev/null +++ b/resources/gen-ai/aws-llama-index-data-loader/docker/sqs_consumer.py @@ -0,0 +1,232 @@ +#!/bin/env python3 -x -u +# +# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance +# with the License. A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions +# and limitations under the License. +# + +from base64 import b64encode +from hashlib import sha256 +from json import loads +from json.decoder import JSONDecodeError +from logging import basicConfig +from logging import getLogger +from logging import StreamHandler +from logging import NOTSET +from os import getenv +from os.path import join +from signal import signal +from signal import SIGINT +from signal import SIGTERM +from sys import stdout +from sys import argv +from time import sleep + +from boto3 import client +from boto3 import resource +from botocore.exceptions import ClientError +from cachetools import cached +from cachetools import TTLCache +from jsonpickle import encode +from llama_index.core import SimpleDirectoryReader +from s3fs import S3FileSystem + +MAX_SQS_VISIBILITY_TIMEOUT = 60 * 60 * 12 - 1 # Maximum SQS Visibility Timeout is <12 hours +WAIT_TIME_SECONDS = 1 # 0 - 20 seconds +basicConfig(stream=stdout, level=getenv('LOGGING_LEVEL', NOTSET)) +LOGGER = getLogger() +LOGGER.addHandler(StreamHandler(stream=stdout)) + +S3 = client('s3') +SQS = resource('sqs') +SSM = client('ssm') + +QUEUE_NAME = getenv('QUEUE_NAME') +BUCKET_NAME = getenv('BUCKET_NAME') +PREFIX = getenv('PREFIX', '') +CIRCUIT_BREAKER_SSM_PARAMETER_NAME = getenv('CIRCUIT_BREAKER_SSM_PARAMETER_NAME', '') +CIRCUIT_BREAKER_CACHE_TTL = 60 +WAIT_TIME_SECONDS = 1 + +if QUEUE_NAME is None: + raise AssertionError('QUEUE_NAME environment variable is not set') +else: + LOGGER.debug(f'"QUEUE_NAME" is {QUEUE_NAME}') +if BUCKET_NAME is None: + raise AssertionError('BUCKET_NAME environment variable is not set') +else: + LOGGER.debug(f'"BUCKET_NAME" is {BUCKET_NAME}') + +SQS_QUEUE = SQS.get_queue_by_name( + QueueName=QUEUE_NAME, +) +LOGGER.debug(f'Queue attributes {SQS_QUEUE.attributes}') + +class SignalHandler: + def __init__(self, sig): + self.received_signal = False + signal(sig, self._signal_handler) + + def _signal_handler(self, signal, frame): + LOGGER.warning(f'handling signal {signal}, exiting gracefully') + self.received_signal = True + +def save_document(documents, serialize=True) -> None: + save_result = False + for document in documents: + file_name = f"{document.metadata['file_name']}" if 'file_name' in document.metadata else '' + LOGGER.info(f'document id: {document.doc_id}') + LOGGER.debug(f'document text: {document.text}') + LOGGER.debug(f'document metadata: {document.metadata}') + LOGGER.debug(f'document extra info: {document.extra_info}') + data = document.text.encode() + response = S3.put_object( + Body=data, + Bucket=BUCKET_NAME, + Key=join(PREFIX, file_name, f'{document.doc_id}.txt'), + ChecksumSHA256=b64encode(sha256(data).digest()).decode(), + ChecksumAlgorithm='SHA256', + ) + LOGGER.debug(f'put_object {response}') + save_result = True + if serialize: + frozen = encode(document) + body = frozen.encode() + LOGGER.debug(f'frozen {frozen}') + frozen_response = S3.put_object( + Body=body, + Bucket=BUCKET_NAME, + Key=join(PREFIX, file_name, f'{document.doc_id}.json'), + ChecksumSHA256=b64encode(sha256(body).digest()).decode(), + ChecksumAlgorithm='SHA256', + ) + LOGGER.debug(f'frozen put_object {frozen_response}') + return save_result + +def load_document(filename) -> bool: + load_result = False + LOGGER.debug(f'loading filename {filename}') + reader = SimpleDirectoryReader( + fs=S3FileSystem(), + input_files=[filename], + raise_on_error=True, + ) + LOGGER.debug(f'readed filename {filename}') + documents = reader.load_data() + if len(documents): + LOGGER.debug(f'There are {len(documents)} documents') + else: + LOGGER.warning(f'TODO: There are no documents') + raise AssertionError('No documents found') + LOGGER.debug(f'loaded filename {filename}') + load_result = save_document(documents) + LOGGER.debug(f'saved filename {filename}') + return load_result + +def log_message(sqs_message) -> None: + LOGGER.info(f'log message id: {sqs_message.message_id}') + LOGGER.debug(f'message attributes: {sqs_message.message_attributes}') + LOGGER.debug(f'message available subresources: {sqs_message.get_available_subresources()}') + LOGGER.debug(f'attributes: {sqs_message.attributes}') + LOGGER.debug(f'body: {sqs_message.body}') + LOGGER.debug(f'md5 of body: {sqs_message.md5_of_body}') + LOGGER.debug(f'md5 of message attributes: {sqs_message.md5_of_message_attributes}') + +def process_message(sqs_message) -> bool: + process_result = False + LOGGER.info(f'process message id: {sqs_message.message_id}') + json_message = loads(sqs_message.body) + if 'Type' in json_message and json_message['Type'] == 'Notification': + if 'Message' in json_message and json_message['Message'] != '' and 'Subject' in json_message and json_message['Subject'] == 'Amazon S3 Notification': + message_content = loads(json_message['Message']) + if 'Records' in message_content: + for record in message_content['Records']: + if 'eventName' in record and record['eventName'].startswith('ObjectCreated') and 's3' in record and 'object' in record['s3'] and 'key' in record['s3']['object'] and 'bucket' in record['s3'] and 'name' in record['s3']['bucket']: + LOGGER.info(f's3 object key: {record["s3"]["object"]["key"]} in {record["s3"]["bucket"]["name"]}') + process_result = load_document(join(record['s3']['bucket']['name'], record['s3']['object']['key'])) + else: + LOGGER.warning(f'skipped: {record}') + else: + LOGGER.warning(f'missing records: {message_content}') + else: + LOGGER.warning(f'unknown message: {json_message}') + else: + LOGGER.warning(f'unknown message type: {json_message}') + return process_result + +# The circuit breaker doesn't kill the process it only skips checking the queue... +@cached(cache=TTLCache(maxsize=1, ttl=CIRCUIT_BREAKER_CACHE_TTL)) +def check_circuit_breaker() -> bool: + tripped = False + if CIRCUIT_BREAKER_SSM_PARAMETER_NAME != '': + try: + response_get_parameter = SSM.get_parameter( + Name=CIRCUIT_BREAKER_SSM_PARAMETER_NAME, + ) + circuit_breaker = response_get_parameter['Parameter']['Value'] + LOGGER.info(f'Circuit breaker="{circuit_breaker}" for SSM parameter "{CIRCUIT_BREAKER_SSM_PARAMETER_NAME}" to process the SQS queue named "{QUEUE_NAME}"') + if response_get_parameter['Parameter']['Value'] == 'True': + tripped = True + except ClientError as exc: + if exc.response['Error']['Code'] == 'ParameterNotFound': + pass + else: + raise exc + return tripped + +def main(): + LOGGER.critical('starting...') + LOGGER.warning(f'A circuit breaker SSM parameter "{CIRCUIT_BREAKER_SSM_PARAMETER_NAME}" is being checked TTL of {CIRCUIT_BREAKER_CACHE_TTL} seconds, create it to skip long polling the SQS Queue "{QUEUE_NAME}"') + sigterm_handler = SignalHandler(SIGTERM) + sigint_handler = SignalHandler(SIGINT) + looper = 0 + # On command line, + C and ECS sends SIGTERM + # https://aws.amazon.com/blogs/containers/graceful-shutdowns-with-ecs/ + while not sigterm_handler.received_signal and not sigint_handler.received_signal: + # Only check for a circuit breaker every certain number of loops + if looper > 10: + looper = 0 + else: + looper += 1 + if check_circuit_breaker(): + LOGGER.info('💥' * looper) + sleep(WAIT_TIME_SECONDS) + else: + LOGGER.info('🏃' * looper) + messages = SQS_QUEUE.receive_messages( + MaxNumberOfMessages=10, # 1-10 messages + WaitTimeSeconds=WAIT_TIME_SECONDS, # 0-20 seconds + AttributeNames=['All'], + MessageAttributeNames=['All'], + ) + # Print out the messages - in case some are not processed... + for message in messages: log_message(message) + for message in messages: + LOGGER.info(f'main message id: {message.message_id}') + try: + ## Consider setting the visibility high to avoid running multiple times. + # message.change_visibility( + # VisibilityTimeout=MAX_SQS_VISIBILITY_TIMEOUT, + # ) + if process_message(message): + message.delete() + except Exception as e: + ## Set back to the SQS Queue's default visibility timeout upon an exception + # message.change_visibility( + # VisibilityTimeout=int(SQS_QUEUE.attributes['VisibilityTimeout']), + # ) + LOGGER.error(f'exception while processing message: {repr(e)}') + continue + LOGGER.critical('...ending') + +if __name__ == '__main__': + main() +else: + LOGGER.warning(f'{__name__} is not accounted for') diff --git a/src/common/base-class/base-class.ts b/src/common/base-class/base-class.ts index c8d2e9ce..28775dae 100644 --- a/src/common/base-class/base-class.ts +++ b/src/common/base-class/base-class.ts @@ -62,6 +62,7 @@ export class BaseClass extends Construct { [ConstructName.HUGGINGFACESAGEMAKERENDPOINT]: 0, [ConstructName.JUMPSTARTSAGEMAKERENDPOINT]: 0, [ConstructName.AWSCONTENTGENAPPSYNCLAMBDA]: 0, + [ConstructName.AWSLLAMAINDEXDATALOADER]: 0, }; diff --git a/src/common/base-class/construct-name-enum.ts b/src/common/base-class/construct-name-enum.ts index 908e2122..3c7d3ab5 100644 --- a/src/common/base-class/construct-name-enum.ts +++ b/src/common/base-class/construct-name-enum.ts @@ -21,5 +21,6 @@ export enum ConstructName { AWSCONTENTGENAPPSYNCLAMBDA = 'C8', AWSRAGAPPSYNCSTEPFNKENDRA = 'C9', AWSWEBCRAWLER = 'C10', - AWSTEXTTOSQL= 'C11' + AWSTEXTTOSQL= 'C11', + AWSLLAMAINDEXDATALOADER = 'C12', } diff --git a/src/index.ts b/src/index.ts index 8fb6dc8d..6eccd360 100644 --- a/src/index.ts +++ b/src/index.ts @@ -32,3 +32,4 @@ export * from './common/base-class/base-class'; export * from './common/base-class/construct-name-enum'; export * from './patterns/gen-ai/aws-bedrock-cw-dashboard'; export * from './patterns/gen-ai/aws-aoss-cw-dashboard'; +export * from './patterns/gen-ai/aws-llama-index-data-loader'; diff --git a/src/patterns/gen-ai/aws-llama-index-data-loader/README.md b/src/patterns/gen-ai/aws-llama-index-data-loader/README.md new file mode 100644 index 00000000..392caea9 --- /dev/null +++ b/src/patterns/gen-ai/aws-llama-index-data-loader/README.md @@ -0,0 +1,144 @@ +# aws-llama-index-data-loader + + + +--- + +![Stability: Experimental](https://img.shields.io/badge/stability-Experimental-important.svg?style=for-the-badge) + +> All classes are under active development and subject to non-backward compatible changes or removal in any +> future version. These are not subject to the [Semantic Versioning](https://semver.org/) model. +> This means that while you may use them, you may need to update your source code when upgrading to a newer version of this package. + +--- + + + +## Table of contents + +- [aws-llama-index-data-loader](#aws-llama-index-data-loader) + - [Table of contents](#table-of-contents) + - [Credits](#credits) + - [Overview](#overview) + - [Initializer](#initializer) + - [Pattern Construct Props](#pattern-construct-props) + - [Pattern Properties](#pattern-properties) + - [Methods](#methods) + - [Default properties](#default-properties) + - [Architecture](#architecture) + - [Cost](#cost) + - [Security](#security) + - [Supported AWS Regions](#supported-aws-regions) + - [Quotas](#quotas) + +## Credits + +Thanks to @jtlew for the initial insight for the need on this construct. + +## Overview + +This construct provides a Amazon Elastic Cluster Service using Llamaindex Data Loaders with default properties [here](#default-properties). + +Here is a minimal deployable pattern definition: + +TypeScript + +```typescript +import { Construct } from 'constructs'; +import { Stack, StackProps, Aws } from 'aws-cdk-lib'; +import { LlamaIndexDataLoader } from '@cdklabs/generative-ai-cdk-constructs'; + +const dataLoader = new LlamaIndexDataLoader(this, 'LlamaIndexDataLoader', {}); +``` + +Python + +```python +from constructs import Construct +from cdklabs.generative_ai_cdk_constructs import LlamaIndexDataLoader + +data_loader = LlamaIndexDataLoader(self, 'LlamaIndexDataLoader', ) +``` + +## Initializer + +```typescript +new LlamaIndexDataLoader(scope: Construct, id: string, props: LlamaIndexDataLoaderProps) +``` + +Parameters + +- scope [Construct](https://docs.aws.amazon.com/cdk/api/v2/docs/constructs.Construct.html) +- id string +- props LlamaIndexDataLoaderProps + +## Pattern Construct Props + +| **Name** | **Type** | **Required** | **Description** | +| :--------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| dockerImageAssetDirectory | string | ![Optional](https://img.shields.io/badge/optional-4169E1) | A path to a directory to build a [DockerImageAsset](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ecr_assets.DockerImageAsset.html). _**NOTE**: Currently the platform is fixed to [`Platform.LINUX_AMD64`](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ecr_assets.Platform.html) with the runtime platform to match with `CpuArchitecture.X86_64` and `OperatingSystemFamily.LINUX` for the running Fargate ECS_ | +| memoryLimitMiB | number | ![Optional](https://img.shields.io/badge/optional-4169E1) | The memory configuration for the container, should be a valid `Memory` defined [AWS::ECS::TaskDefinition](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ecs-taskdefinition.html#cfn-ecs-taskdefinition-memory) | +| containerLoggingLevel | string | ![Optional](https://img.shields.io/badge/optional-4169E1) | The [python logging level](https://docs.python.org/3/library/logging.html#levels) for the ECS tasks | +| outputBucket | [Bucket](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_s3.Bucket.html) | ![Optional](https://img.shields.io/badge/optional-4169E1) | The output bucket. _If not passed in one will be created automatically_ | +| vpc | [IVpc](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ec2.IVpc.html) | ![Optional](https://img.shields.io/badge/optional-4169E1) | The VPC to use. _If not passed in, the [QueueProcessingFargateService](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ecs_patterns.QueueProcessingFargateService.html) automatically created one will be used_ | + +## Pattern Properties + +| **Name** | **Type** | **Description** | +| :----------------------------- | :-------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| outputBucket | [Bucket](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_s3.Bucket.html) | The S3 Bucket for the output. | +| queueProcessingFargateService | [QueueProcessingFargateService](https://docs.aws.amazon.com/cdk/api/v2/docs/aws-cdk-lib.aws_ecs_patterns.QueueProcessingFargateService.html) | The ECS Fargate and components assocated with the ECS pattern. | + +## Methods + +N/A + +## Default properties + +Out-of-the-box implementation of the construct without any override will set the following defaults: + +### DockerImageAssetDirectory + +This repositories source [default path](https://github.com/awslabs/generative-ai-cdk-constructs/tree/main/resources/gen-ai/aws-llama-index-data-loader/docker) + +### MemoryLimitMiB + +2048 + +### ContainerLoggingLevel + +'WARNING' + +## Architecture + +![Architecture Diagram](architecture.png) + +## Cost + +You are responsible for the cost of the AWS services used while running this construct. + +We recommend creating a budget through [AWS Cost Explorer](http://aws.amazon.com/aws-cost-management/aws-cost-explorer/) to help manage costs. Prices are subject to change. For full details, refer to the pricing webpage for each AWS service used in this solution: + +## Security + +When you build systems on AWS infrastructure, security responsibilities are shared between you and AWS. This [shared responsibility](http://aws.amazon.com/compliance/shared-responsibility-model/) model reduces your operational burden because AWS operates, manages, and controls the components including the host operating system, virtualization layer, and physical security of the facilities in which the services operate. For more information about AWS security, visit [AWS Cloud Security](http://aws.amazon.com/security/). + +Optionnaly, you can provide existing resources to the constructs (marked optional in the construct pattern props). If you chose to do so, please refer to the official documentation on best practices to secure each service: + +- Amazon S3 + +## Supported AWS Regions + +This solution depends uses the Amazon OpenSearch Serverless and Amazon CloudWatch services, which are not currently available in all AWS Regions. You must launch this construct in an AWS Region where these services are available. For the most current availability of AWS services by Region, see the [AWS Regional Services List](https://aws.amazon.com/about-aws/global-infrastructure/regional-product-services/). + +## Quotas + +Service quotas, also referred to as limits, are the maximum number of service resources or operations for your AWS account. + +Make sure you have sufficient quota for each of the services implemented in this solution. For more information, refer to [AWS service quotas](https://docs.aws.amazon.com/general/latest/gr/aws_service_limits.html). + +To view the service quotas for all AWS services in the documentation without switching pages, view the information in the [Service endpoints and quotas](https://docs.aws.amazon.com/general/latest/gr/aws-general.pdf#aws-service-information) page in the PDF instead. + +--- + +© Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/src/patterns/gen-ai/aws-llama-index-data-loader/architecture.png b/src/patterns/gen-ai/aws-llama-index-data-loader/architecture.png new file mode 100644 index 00000000..3f89e67e Binary files /dev/null and b/src/patterns/gen-ai/aws-llama-index-data-loader/architecture.png differ diff --git a/src/patterns/gen-ai/aws-llama-index-data-loader/index.ts b/src/patterns/gen-ai/aws-llama-index-data-loader/index.ts new file mode 100644 index 00000000..3421c559 --- /dev/null +++ b/src/patterns/gen-ai/aws-llama-index-data-loader/index.ts @@ -0,0 +1,497 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ +import { join } from 'node:path'; +import { Duration, RemovalPolicy, Stack } from 'aws-cdk-lib'; +import { FlowLogDestination, IVpc } from 'aws-cdk-lib/aws-ec2'; +import { DockerImageAsset, Platform } from 'aws-cdk-lib/aws-ecr-assets'; +import { Cluster, ContainerImage, CpuArchitecture, OperatingSystemFamily } from 'aws-cdk-lib/aws-ecs'; +import { QueueProcessingFargateService } from 'aws-cdk-lib/aws-ecs-patterns'; +import { Effect, PolicyStatement, ServicePrincipal } from 'aws-cdk-lib/aws-iam'; +import { Key } from 'aws-cdk-lib/aws-kms'; +import { BlockPublicAccess, Bucket, BucketEncryption, EventType, ObjectLockMode } from 'aws-cdk-lib/aws-s3'; +import { SnsDestination } from 'aws-cdk-lib/aws-s3-notifications'; +import { Topic } from 'aws-cdk-lib/aws-sns'; +import { SqsSubscription } from 'aws-cdk-lib/aws-sns-subscriptions'; +import { Queue } from 'aws-cdk-lib/aws-sqs'; +import { StringParameter } from 'aws-cdk-lib/aws-ssm'; +import { NagSuppressions } from 'cdk-nag'; +import { Construct } from 'constructs'; +import { BaseClass, BaseClassProps, ConstructName } from '../../../common/base-class'; + +export interface LlamaIndexDataLoaderProps { + /** + * The directory to build the Docker image + * @description The directory to build the Docker image. + * @default __dirname + '/docker' + */ + readonly dockerImageAssetDirectory?: string; + + /** + * The default memory + * @description The default memory. + * @default 2048 + */ + readonly memoryLimitMiB?: number | undefined; + + /** + * @description the container's logging level + * @default 'WARNING' + */ + readonly containerLoggingLevel?: string; + + /** + * @description the S3 output to use + * @default undefined + */ + readonly outputBucket?: Bucket; + + /** + * @description the VPC to use + * @default undefined + */ + readonly vpc?: IVpc; + + /** + * Value will be appended to resources name. + * + * @default - _dev + */ + readonly stage?: string; + /** + * Enable observability. Warning: associated cost with the services + * used. Best practive to enable by default. + * + * @default - true + */ + readonly observability?: boolean; + +} + +export class LlamaIndexDataLoader extends BaseClass { + + public readonly outputBucket: Bucket; + public readonly queueProcessingFargateService: QueueProcessingFargateService; + + private readonly logBucket?: Bucket; + private readonly bucketKey?: Key; + private readonly dockerImageAssetDirectory: string; + private readonly memoryLimitMiB: number; + private readonly containerLoggingLevel: string; + + constructor(scope: Construct, id: string, props: LlamaIndexDataLoaderProps) { + super(scope, id); + + const baseProps: BaseClassProps = { + stage: props.stage, + constructName: ConstructName.AWSLLAMAINDEXDATALOADER, + constructId: id, + observability: props.observability, + }; + + this.updateEnvSuffix(baseProps); + this.addObservabilityToConstruct(baseProps); + + // Update the optional properties to their defaults + this.dockerImageAssetDirectory = props.dockerImageAssetDirectory ?? join( + __dirname, '..', '..', '..', '..', 'resources', 'gen-ai', 'aws-llama-index-data-loader', 'docker', + ); + this.memoryLimitMiB = props.memoryLimitMiB ?? 2048; + this.containerLoggingLevel = props.containerLoggingLevel ?? 'WARNING'; + let bucketsInvolved = []; + if (props.outputBucket) { + this.outputBucket = props.outputBucket; + } else { + this.bucketKey = new Key(this, 'LogBucketKey', { + enableKeyRotation: true, + }); + this.logBucket = new Bucket(this, 'LogBucket', { + enforceSSL: true, + versioned: true, + encryption: BucketEncryption.KMS, + encryptionKey: this.bucketKey, + bucketKeyEnabled: true, + removalPolicy: RemovalPolicy.DESTROY, + autoDeleteObjects: true, + objectLockEnabled: true, + objectLockDefaultRetention: { + mode: ObjectLockMode.GOVERNANCE, + duration: Duration.days(1), + }, + blockPublicAccess: BlockPublicAccess.BLOCK_ALL, + }); + bucketsInvolved.push(this.logBucket); + this.outputBucket = new Bucket(this, 'Output', { + enforceSSL: true, + versioned: true, + serverAccessLogsBucket: this.logBucket, + serverAccessLogsPrefix: 'output-bucket-access-logs', + encryption: BucketEncryption.KMS, + encryptionKey: this.bucketKey, + bucketKeyEnabled: true, + blockPublicAccess: BlockPublicAccess.BLOCK_ALL, + removalPolicy: RemovalPolicy.DESTROY, + autoDeleteObjects: true, + objectLockEnabled: true, + objectLockDefaultRetention: { + mode: ObjectLockMode.GOVERNANCE, + duration: Duration.days(1), + }, + lifecycleRules: [ + { + id: 'AbortIncompleteMultipartUpload', + enabled: true, + abortIncompleteMultipartUploadAfter: Duration.days(1), + }, + ], + }); + } + bucketsInvolved.push(this.outputBucket); + // Create a new SSM Parameter holding a String + const circuitBreakerParameter = new StringParameter(this, 'CircuitBreaker', { + stringValue: 'False', + }); + const asset = new DockerImageAsset(this, 'Image', { + directory: this.dockerImageAssetDirectory, + platform: Platform.LINUX_AMD64, + }); + const queue = new Queue(this, 'Queue', { + visibilityTimeout: Duration.seconds(300), + enforceSSL: true, + deadLetterQueue: { + maxReceiveCount: 3, + queue: new Queue(this, 'DeadLetterQueue', { + enforceSSL: true, + }), + }, + }); + const queueProcessingFargateService = new QueueProcessingFargateService(this, 'Service', { + cluster: props.vpc === undefined ? new Cluster(this, 'Cluster', { + containerInsights: true, + }) : undefined, + vpc: props.vpc, + memoryLimitMiB: this.memoryLimitMiB, + runtimePlatform: { + cpuArchitecture: CpuArchitecture.X86_64, + operatingSystemFamily: OperatingSystemFamily.LINUX, + }, + queue: queue, + image: ContainerImage.fromDockerImageAsset(asset), + healthCheck: { + command: ['CMD-SHELL', '/usr/src/app/healthcheck.sh'], + interval: Duration.seconds(30), + timeout: Duration.seconds(30), + retries: 3, + startPeriod: Duration.seconds(5), + }, + enableLogging: true, + environment: { + CIRCUIT_BREAKER_SSM_PARAMETER_NAME: circuitBreakerParameter.parameterName, + LOGGING_LEVEL: this.containerLoggingLevel, + BUCKET_NAME: this.outputBucket.bucketName, + }, + minScalingCapacity: 0, + maxScalingCapacity: 10, + scalingSteps: + [ + { upper: 0, change: -1 }, + { lower: 1, change: +1 }, // because `minScalingCapacity` set at zero + { lower: 100, change: +1 }, + { lower: 500, change: +5 }, + ], + }); + this.queueProcessingFargateService = queueProcessingFargateService; + + // Setup Default S3 Example if Docker asset directory is missing + if (!props.dockerImageAssetDirectory) { + const rawBucket = new Bucket(this, 'Raw', { + enforceSSL: true, + versioned: true, + serverAccessLogsBucket: this.logBucket, + encryption: BucketEncryption.KMS, + encryptionKey: this.bucketKey, + bucketKeyEnabled: true, + serverAccessLogsPrefix: 'raw-bucket-access-logs', + blockPublicAccess: BlockPublicAccess.BLOCK_ALL, + removalPolicy: RemovalPolicy.DESTROY, + autoDeleteObjects: true, + objectLockEnabled: true, + objectLockDefaultRetention: { + mode: ObjectLockMode.GOVERNANCE, + duration: Duration.days(1), + }, + lifecycleRules: [ + { + id: 'AbortIncompleteMultipartUpload', + enabled: true, + abortIncompleteMultipartUploadAfter: Duration.days(1), + }, + ], + }); + bucketsInvolved.push(rawBucket); + const topicKey = new Key(this, 'TopicKey', { + enableKeyRotation: true, + }); + const topic = new Topic(this, 'Topic', { + enforceSSL: true, + masterKey: topicKey, + }); + topicKey.addToResourcePolicy( + new PolicyStatement({ + actions: ['kms:Decrypt', 'kms:GenerateDataKey*'], + resources: ['*'], + principals: [new ServicePrincipal('s3.amazonaws.com')], + }), + ); + topic.grantPublish(new ServicePrincipal('s3.amazonaws.com')); + rawBucket.addEventNotification( + EventType.OBJECT_CREATED, + new SnsDestination(topic), + ); + topic.addSubscription( + new SqsSubscription(queue), + ); + this.queueProcessingFargateService.taskDefinition.addToTaskRolePolicy( + new PolicyStatement({ + effect: Effect.ALLOW, + actions: ['s3:GetObject'], + resources: [rawBucket.bucketArn, rawBucket.bucketArn + '/*'], + }), + ); + if (rawBucket.encryptionKey) { + rawBucket.encryptionKey.grantDecrypt(this.queueProcessingFargateService.taskDefinition.taskRole); + this.queueProcessingFargateService.taskDefinition.taskRole.addToPrincipalPolicy( + new PolicyStatement({ + effect: Effect.ALLOW, + actions: ['kms:GenerateDataKey'], + resources: [rawBucket.encryptionKey.keyArn], + }), + ); + } + } + this.queueProcessingFargateService.cluster.vpc.addFlowLog('FlowLog', { + destination: FlowLogDestination.toS3(this.logBucket, 'vpc-flow-logs'), + }); + circuitBreakerParameter.grantRead(this.queueProcessingFargateService.taskDefinition.taskRole); + this.queueProcessingFargateService.taskDefinition.addToTaskRolePolicy( + new PolicyStatement({ + effect: Effect.ALLOW, + actions: ['s3:PutObject'], + resources: [this.outputBucket.bucketArn, this.outputBucket.bucketArn + '/*'], + }), + ); + if (this.outputBucket.encryptionKey) { + this.outputBucket.encryptionKey.grantDecrypt(this.queueProcessingFargateService.taskDefinition.taskRole); + this.queueProcessingFargateService.taskDefinition.taskRole.addToPrincipalPolicy( + new PolicyStatement({ + effect: Effect.ALLOW, + actions: ['kms:GenerateDataKey'], + resources: [this.outputBucket.encryptionKey.keyArn], + }), + ); + } + + //////////////////////////////////////////////////////////////////////// + // NagSuppressions + //////////////////////////////////////////////////////////////////////// + if (this.logBucket) { + NagSuppressions.addResourceSuppressions(this.logBucket, + [ + { + id: 'AwsSolutions-S1', + reason: 'There is no need to enable access logging for the AccessLogs bucket.', + }, + ], + true, + ); + } + NagSuppressions.addResourceSuppressions(bucketsInvolved, + [ + { + id: 'HIPAA.Security-S3BucketReplicationEnabled', + reason: 'Replication for this use case is uneccessary.', + }, + { + id: 'NIST.800.53.R4-S3BucketReplicationEnabled', + reason: 'Replication for this use case is uneccessary.', + + }, + { + id: 'NIST.800.53.R5-S3BucketReplicationEnabled', + reason: 'Replication for this use case is uneccessary.', + }, + { + id: 'PCI.DSS.321-S3BucketReplicationEnabled', + reason: 'Replication for this use case is uneccessary.', + }, + ], + true, + ); + NagSuppressions.addResourceSuppressions([this.queueProcessingFargateService.cluster.vpc], + [ + { + id: 'HIPAA.Security-VPCSubnetAutoAssignPublicIpDisabled', + reason: 'default vpc creation', + }, + { + id: 'NIST.800.53.R5-VPCSubnetAutoAssignPublicIpDisabled', + reason: 'default vpc creation', + }, + { + id: 'PCI.DSS.321-VPCSubnetAutoAssignPublicIpDisabled', + reason: 'default vpc creation', + }, + { + id: 'HIPAA.Security-VPCNoUnrestrictedRouteToIGW', + reason: 'default vpc creation', + }, + { + id: 'NIST.800.53.R5-VPCNoUnrestrictedRouteToIGW', + reason: 'default vpc creation', + }, + { + id: 'PCI.DSS.321-VPCNoUnrestrictedRouteToIGW', + reason: 'default vpc creation', + }, + ], + true, + ); + + // Suppress findings for the BucketNotificationsHandler + // https://github.com/aws/aws-cdk/issues/9552 + const bucketNotifications = Stack.of(this).node.tryFindChild('BucketNotificationsHandler050a0587b7544547bf325f094a3db834'); + if (bucketNotifications) { + NagSuppressions.addResourceSuppressions(bucketNotifications, + [ + { + id: 'AwsSolutions-IAM4', + reason: 'AWSLambdaBasicExecutionRole is used by `addEventNotification`', + appliesTo: ['Policy::arn::iam::aws:policy/service-role/AWSLambdaBasicExecutionRole'], + }, + { + id: 'AwsSolutions-IAM5', + reason: 'Suppress all AwsSolutions-IAM5 findings for this stack', + appliesTo: ['Resource::*'], + }, + { + id: 'HIPAA.Security-IAMNoInlinePolicy', + reason: 'Inline policies are part of the BucketHandlerNotification', + }, + { + id: 'NIST.800.53.R4-IAMNoInlinePolicy', + reason: 'Inline policies are part of the BucketHandlerNotification', + }, + { + id: 'NIST.800.53.R5-IAMNoInlinePolicy', + reason: 'Inline policies are part of the BucketHandlerNotification', + }, + { + id: 'PCI.DSS.321-IAMNoInlinePolicy', + reason: 'Inline policies are part of the BucketHandlerNotification', + }, + ], + true, + ); + } + NagSuppressions.addResourceSuppressions(this.queueProcessingFargateService, + [ + { + id: 'AwsSolutions-ECS2', + reason: 'Environmental variables are okay', + }, + ], + true, + ); + NagSuppressions.addResourceSuppressions(this.queueProcessingFargateService.taskDefinition, + [ + { + id: 'AwsSolutions-IAM5', + reason: 'The ecr:GetAuthorizationToken requires "*"', + appliesTo: [ + 'Resource::*', + 'Resource::/*', + 'Resource::/*', + ], + }, + ], + true, + ); + + // Inline policies + let taskDefinitionRoles = [ + this.queueProcessingFargateService.taskDefinition.taskRole, + ]; + if (this.queueProcessingFargateService.taskDefinition.executionRole) { + taskDefinitionRoles.push(this.queueProcessingFargateService.taskDefinition.executionRole); + } + NagSuppressions.addResourceSuppressions(taskDefinitionRoles, + [ + { + id: 'HIPAA.Security-IAMNoInlinePolicy', + reason: 'Inline policies are acceptable for this use case.', + }, + { + id: 'NIST.800.53.R4-IAMNoInlinePolicy', + reason: 'Inline policies are acceptable for this use case.', + }, + { + id: 'NIST.800.53.R5-IAMNoInlinePolicy', + reason: 'Inline policies are acceptable for this use case.', + }, + { + id: 'PCI.DSS.321-IAMNoInlinePolicy', + reason: 'Inline policies are acceptable for this use case.', + }, + ], + true, + ); + + // Unencrypted CloudWatch logging + NagSuppressions.addResourceSuppressions(this.queueProcessingFargateService.taskDefinition, + [ + { + id: 'HIPAA.Security-CloudWatchLogGroupEncrypted', + reason: 'CloudWatch logging is not encrypted', + }, + { + id: 'NIST.800.53.R4-CloudWatchLogGroupEncrypted', + reason: 'CloudWatch logging is not encrypted', + }, + { + id: 'NIST.800.53.R5-CloudWatchLogGroupEncrypted', + reason: 'CloudWatch logging is not encrypted', + }, + { + id: 'PCI.DSS.321-CloudWatchLogGroupEncrypted', + reason: 'CloudWatch logging is not encrypted', + }, + { + id: 'HIPAA.Security-CloudWatchLogGroupRetentionPeriod', + reason: 'CloudWatch logging is not encrypted', + }, + { + id: 'NIST.800.53.R4-CloudWatchLogGroupRetentionPeriod', + reason: 'CloudWatch logging is not encrypted', + }, + { + id: 'NIST.800.53.R5-CloudWatchLogGroupRetentionPeriod', + reason: 'CloudWatch logging is not encrypted', + }, + { + id: 'PCI.DSS.321-CloudWatchLogGroupRetentionPeriod', + reason: 'CloudWatch logging is not encrypted', + }, + ], + true, + ); + } +} \ No newline at end of file diff --git a/test/patterns/gen-ai/aws-llama-index-data-loader/aws-llama-index-data-loader.test.ts b/test/patterns/gen-ai/aws-llama-index-data-loader/aws-llama-index-data-loader.test.ts new file mode 100644 index 00000000..5bdc298c --- /dev/null +++ b/test/patterns/gen-ai/aws-llama-index-data-loader/aws-llama-index-data-loader.test.ts @@ -0,0 +1,146 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ +import { + join, +// resolve, +} from 'node:path'; +import * as cdk from 'aws-cdk-lib'; +import { + Template, +// Match, +} from 'aws-cdk-lib/assertions'; +import { Bucket } from 'aws-cdk-lib/aws-s3'; +import { AwsSolutionsChecks } from 'cdk-nag'; +import { + LlamaIndexDataLoader, + LlamaIndexDataLoaderProps, +} from '../../../../src/patterns/gen-ai/aws-llama-index-data-loader'; + +describe('LlamaIndex Data Loader Construct Default', () => { + let app: cdk.App; + let llamaIndexDataLoaderTemplate: Template; + let llamaIndexDataLoaderTestConstruct: LlamaIndexDataLoader; + + afterAll(() => { + console.log('Test completed'); + join(); + }); + + beforeAll(() => { + app = new cdk.App(); + Bucket.toString(); // because typescript errors if not used... + cdk.Aspects.of(app).add(new AwsSolutionsChecks()); + const llamaIndexDataLoaderTestStack = new cdk.Stack(app, 'llamaIndexDataLoaderTestStack', { + env: { account: cdk.Aws.ACCOUNT_ID, region: cdk.Aws.REGION }, + }); + const props: LlamaIndexDataLoaderProps = { + }; + + llamaIndexDataLoaderTestConstruct = new LlamaIndexDataLoader( + llamaIndexDataLoaderTestStack, + 'LlamaIndexDataLoader', + props, + ); + llamaIndexDataLoaderTemplate = Template.fromStack(llamaIndexDataLoaderTestStack); + }); + + test('S3 bucket count', () => { + llamaIndexDataLoaderTemplate.resourceCountIs('AWS::S3::Bucket', 3); + expect(llamaIndexDataLoaderTestConstruct.outputBucket).not.toBeNull; + }); + + test('SNS count', () => { + llamaIndexDataLoaderTemplate.resourceCountIs('AWS::SNS::Topic', 1); + }); + + test('Subscription count', () => { + llamaIndexDataLoaderTemplate.resourceCountIs('AWS::SNS::Subscription', 1); + }); + test('Parameter count', () => { + llamaIndexDataLoaderTemplate.resourceCountIs('AWS::SSM::Parameter', 1); + }); + test('Cluster count', () => { + llamaIndexDataLoaderTemplate.resourceCountIs('AWS::ECS::Cluster', 1); + }); + test('Service count', () => { + llamaIndexDataLoaderTemplate.resourceCountIs('AWS::ECS::Service', 1); + }); + test('Task count', () => { + llamaIndexDataLoaderTemplate.resourceCountIs('AWS::ECS::TaskDefinition', 1); + }); + test('AutoScaling target count', () => { + llamaIndexDataLoaderTemplate.resourceCountIs('AWS::ApplicationAutoScaling::ScalableTarget', 1); + }); + test('AutoScaling policy count', () => { + llamaIndexDataLoaderTemplate.resourceCountIs('AWS::ApplicationAutoScaling::ScalingPolicy', 3); + }); + test('Alarm count', () => { + llamaIndexDataLoaderTemplate.resourceCountIs('AWS::CloudWatch::Alarm', 2); + }); + + test('SQS count', () => { + llamaIndexDataLoaderTemplate.resourceCountIs('AWS::SQS::Queue', 2); + expect(llamaIndexDataLoaderTestConstruct.queueProcessingFargateService.sqsQueue).not.toBeNull; + }); + +}); + +describe('LlamaIndex Data Loader Construct Properties', () => { + let app: cdk.App; + let llamaIndexDataLoaderTemplate: Template; + let llamaIndexDataLoaderTestConstruct: LlamaIndexDataLoader; + + afterAll(() => { + console.log('Test completed'); + }); + + beforeAll(() => { + app = new cdk.App(); + cdk.Aspects.of(app).add(new AwsSolutionsChecks()); + const llamaIndexDataLoaderTestStack = new cdk.Stack(app, 'llamaIndexDataLoaderTestStack', { + env: { account: cdk.Aws.ACCOUNT_ID, region: cdk.Aws.REGION }, + }); + const props: LlamaIndexDataLoaderProps = { + dockerImageAssetDirectory: join( + //resolve(), '..', '..', '..', '..', 'resources', 'gen-ai', 'aws-llama-index-data-loader', 'docker', + 'resources', 'gen-ai', 'aws-llama-index-data-loader', 'docker', + ), + containerLoggingLevel: 'INFO', + memoryLimitMiB: 1024, + outputBucket: new Bucket(llamaIndexDataLoaderTestStack, 'OutputBucket', {}), + observability: false, + stage: 'test', + }; + + llamaIndexDataLoaderTestConstruct = new LlamaIndexDataLoader( + llamaIndexDataLoaderTestStack, + 'LlamaIndexDataLoader', + props, + ); + llamaIndexDataLoaderTemplate = Template.fromStack(llamaIndexDataLoaderTestStack); + }); + + test('Cluster count', () => { + llamaIndexDataLoaderTemplate.resourceCountIs('AWS::ECS::Cluster', 1); + }); + test('Service count', () => { + llamaIndexDataLoaderTemplate.resourceCountIs('AWS::ECS::Service', 1); + }); + test('Task count', () => { + llamaIndexDataLoaderTemplate.resourceCountIs('AWS::ECS::TaskDefinition', 1); + }); + test('Queue', () => { + llamaIndexDataLoaderTemplate.resourceCountIs('AWS::SQS::Queue', 2); + expect(llamaIndexDataLoaderTestConstruct.queueProcessingFargateService.sqsQueue).not.toBeNull; + }); +}); \ No newline at end of file diff --git a/test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.snapshot/ServiceTestDefaultTestDeployAssertE49B1ECE.assets.json b/test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.snapshot/ServiceTestDefaultTestDeployAssertE49B1ECE.assets.json new file mode 100644 index 00000000..4adbb7de --- /dev/null +++ b/test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.snapshot/ServiceTestDefaultTestDeployAssertE49B1ECE.assets.json @@ -0,0 +1,19 @@ +{ + "version": "38.0.1", + "files": { + "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22": { + "source": { + "path": "ServiceTestDefaultTestDeployAssertE49B1ECE.template.json", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "21fbb51d7b23f6a6c262b46a9caee79d744a3ac019fd45422d988b96d44b2a22.json", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + } + }, + "dockerImages": {} +} \ No newline at end of file diff --git a/test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.snapshot/ServiceTestDefaultTestDeployAssertE49B1ECE.template.json b/test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.snapshot/ServiceTestDefaultTestDeployAssertE49B1ECE.template.json new file mode 100644 index 00000000..ad9d0fb7 --- /dev/null +++ b/test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.snapshot/ServiceTestDefaultTestDeployAssertE49B1ECE.template.json @@ -0,0 +1,36 @@ +{ + "Parameters": { + "BootstrapVersion": { + "Type": "AWS::SSM::Parameter::Value", + "Default": "/cdk-bootstrap/hnb659fds/version", + "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]" + } + }, + "Rules": { + "CheckBootstrapVersion": { + "Assertions": [ + { + "Assert": { + "Fn::Not": [ + { + "Fn::Contains": [ + [ + "1", + "2", + "3", + "4", + "5" + ], + { + "Ref": "BootstrapVersion" + } + ] + } + ] + }, + "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI." + } + ] + } + } +} \ No newline at end of file diff --git a/test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.snapshot/aws-llama-index-data-loader-integ-test.assets.json b/test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.snapshot/aws-llama-index-data-loader-integ-test.assets.json new file mode 100644 index 00000000..6f82b4a2 --- /dev/null +++ b/test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.snapshot/aws-llama-index-data-loader-integ-test.assets.json @@ -0,0 +1,46 @@ +{ + "version": "38.0.1", + "files": { + "faa95a81ae7d7373f3e1f242268f904eb748d8d0fdd306e8a6fe515a1905a7d6": { + "source": { + "path": "asset.faa95a81ae7d7373f3e1f242268f904eb748d8d0fdd306e8a6fe515a1905a7d6", + "packaging": "zip" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "faa95a81ae7d7373f3e1f242268f904eb748d8d0fdd306e8a6fe515a1905a7d6.zip", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + }, + "51c815aa4873ecd577f19e12f0731bf14f73527e5a03b65d82151ef0488c5b74": { + "source": { + "path": "aws-llama-index-data-loader-integ-test.template.json", + "packaging": "file" + }, + "destinations": { + "current_account-current_region": { + "bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}", + "objectKey": "51c815aa4873ecd577f19e12f0731bf14f73527e5a03b65d82151ef0488c5b74.json", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + } + }, + "dockerImages": { + "dbdfbc71da28e53c4e106883b62a14fd19bbe4d7e6dfa907f9454b8f3e2360f1": { + "source": { + "directory": "asset.dbdfbc71da28e53c4e106883b62a14fd19bbe4d7e6dfa907f9454b8f3e2360f1", + "platform": "linux/amd64" + }, + "destinations": { + "current_account-current_region": { + "repositoryName": "cdk-hnb659fds-container-assets-${AWS::AccountId}-${AWS::Region}", + "imageTag": "dbdfbc71da28e53c4e106883b62a14fd19bbe4d7e6dfa907f9454b8f3e2360f1", + "assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-image-publishing-role-${AWS::AccountId}-${AWS::Region}" + } + } + } + } +} \ No newline at end of file diff --git a/test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.snapshot/aws-llama-index-data-loader-integ-test.template.json b/test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.snapshot/aws-llama-index-data-loader-integ-test.template.json new file mode 100644 index 00000000..98384213 --- /dev/null +++ b/test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.snapshot/aws-llama-index-data-loader-integ-test.template.json @@ -0,0 +1,3686 @@ +{ + "Resources": { + "testLogBucketKey32CC4D1D": { + "Type": "AWS::KMS::Key", + "Properties": { + "EnableKeyRotation": true, + "KeyPolicy": { + "Statement": [ + { + "Action": "kms:*", + "Effect": "Allow", + "Principal": { + "AWS": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::", + { + "Ref": "AWS::AccountId" + }, + ":root" + ] + ] + } + }, + "Resource": "*" + }, + { + "Action": [ + "kms:Decrypt", + "kms:Encrypt", + "kms:ReEncrypt*", + "kms:GenerateDataKey*" + ], + "Effect": "Allow", + "Principal": { + "Service": "logging.s3.amazonaws.com" + }, + "Resource": "*" + } + ], + "Version": "2012-10-17" + } + }, + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain" + }, + "testLogBucket7F298F3B": { + "Type": "AWS::S3::Bucket", + "Properties": { + "AccessControl": "LogDeliveryWrite", + "BucketEncryption": { + "ServerSideEncryptionConfiguration": [ + { + "BucketKeyEnabled": true, + "ServerSideEncryptionByDefault": { + "KMSMasterKeyID": { + "Fn::GetAtt": [ + "testLogBucketKey32CC4D1D", + "Arn" + ] + }, + "SSEAlgorithm": "aws:kms" + } + } + ] + }, + "ObjectLockConfiguration": { + "ObjectLockEnabled": "Enabled", + "Rule": { + "DefaultRetention": { + "Days": 1, + "Mode": "GOVERNANCE" + } + } + }, + "ObjectLockEnabled": true, + "OwnershipControls": { + "Rules": [ + { + "ObjectOwnership": "ObjectWriter" + } + ] + }, + "PublicAccessBlockConfiguration": { + "BlockPublicAcls": true, + "BlockPublicPolicy": true, + "IgnorePublicAcls": true, + "RestrictPublicBuckets": true + }, + "Tags": [ + { + "Key": "aws-cdk:auto-delete-objects", + "Value": "true" + } + ], + "VersioningConfiguration": { + "Status": "Enabled" + } + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "There is no need to enable access logging for the AccessLogs bucket.", + "id": "AwsSolutions-S1" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "HIPAA.Security-S3BucketReplicationEnabled" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "NIST.800.53.R4-S3BucketReplicationEnabled" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "NIST.800.53.R5-S3BucketReplicationEnabled" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "PCI.DSS.321-S3BucketReplicationEnabled" + } + ] + } + } + }, + "testLogBucketPolicy8C5750D7": { + "Type": "AWS::S3::BucketPolicy", + "Properties": { + "Bucket": { + "Ref": "testLogBucket7F298F3B" + }, + "PolicyDocument": { + "Statement": [ + { + "Action": "s3:*", + "Condition": { + "Bool": { + "aws:SecureTransport": "false" + } + }, + "Effect": "Deny", + "Principal": { + "AWS": "*" + }, + "Resource": [ + { + "Fn::GetAtt": [ + "testLogBucket7F298F3B", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "testLogBucket7F298F3B", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, + { + "Action": [ + "s3:PutBucketPolicy", + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*" + ], + "Effect": "Allow", + "Principal": { + "AWS": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092", + "Arn" + ] + } + }, + "Resource": [ + { + "Fn::GetAtt": [ + "testLogBucket7F298F3B", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "testLogBucket7F298F3B", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + } + }, + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "There is no need to enable access logging for the AccessLogs bucket.", + "id": "AwsSolutions-S1" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "HIPAA.Security-S3BucketReplicationEnabled" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "NIST.800.53.R4-S3BucketReplicationEnabled" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "NIST.800.53.R5-S3BucketReplicationEnabled" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "PCI.DSS.321-S3BucketReplicationEnabled" + } + ] + } + } + }, + "testLogBucketAutoDeleteObjectsCustomResource14E7FB3C": { + "Type": "Custom::S3AutoDeleteObjects", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderHandler9D90184F", + "Arn" + ] + }, + "BucketName": { + "Ref": "testLogBucket7F298F3B" + } + }, + "DependsOn": [ + "testLogBucketPolicy8C5750D7" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "There is no need to enable access logging for the AccessLogs bucket.", + "id": "AwsSolutions-S1" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "HIPAA.Security-S3BucketReplicationEnabled" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "NIST.800.53.R4-S3BucketReplicationEnabled" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "NIST.800.53.R5-S3BucketReplicationEnabled" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "PCI.DSS.321-S3BucketReplicationEnabled" + } + ] + } + } + }, + "testOutput66980038": { + "Type": "AWS::S3::Bucket", + "Properties": { + "BucketEncryption": { + "ServerSideEncryptionConfiguration": [ + { + "BucketKeyEnabled": true, + "ServerSideEncryptionByDefault": { + "KMSMasterKeyID": { + "Fn::GetAtt": [ + "testLogBucketKey32CC4D1D", + "Arn" + ] + }, + "SSEAlgorithm": "aws:kms" + } + } + ] + }, + "LifecycleConfiguration": { + "Rules": [ + { + "AbortIncompleteMultipartUpload": { + "DaysAfterInitiation": 1 + }, + "Id": "AbortIncompleteMultipartUpload", + "Status": "Enabled" + } + ] + }, + "LoggingConfiguration": { + "DestinationBucketName": { + "Ref": "testLogBucket7F298F3B" + }, + "LogFilePrefix": "output-bucket-access-logs" + }, + "ObjectLockConfiguration": { + "ObjectLockEnabled": "Enabled", + "Rule": { + "DefaultRetention": { + "Days": 1, + "Mode": "GOVERNANCE" + } + } + }, + "ObjectLockEnabled": true, + "PublicAccessBlockConfiguration": { + "BlockPublicAcls": true, + "BlockPublicPolicy": true, + "IgnorePublicAcls": true, + "RestrictPublicBuckets": true + }, + "Tags": [ + { + "Key": "aws-cdk:auto-delete-objects", + "Value": "true" + } + ], + "VersioningConfiguration": { + "Status": "Enabled" + } + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "Replication for this use case is uneccessary.", + "id": "HIPAA.Security-S3BucketReplicationEnabled" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "NIST.800.53.R4-S3BucketReplicationEnabled" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "NIST.800.53.R5-S3BucketReplicationEnabled" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "PCI.DSS.321-S3BucketReplicationEnabled" + } + ] + } + } + }, + "testOutputPolicy4B2D7F76": { + "Type": "AWS::S3::BucketPolicy", + "Properties": { + "Bucket": { + "Ref": "testOutput66980038" + }, + "PolicyDocument": { + "Statement": [ + { + "Action": "s3:*", + "Condition": { + "Bool": { + "aws:SecureTransport": "false" + } + }, + "Effect": "Deny", + "Principal": { + "AWS": "*" + }, + "Resource": [ + { + "Fn::GetAtt": [ + "testOutput66980038", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "testOutput66980038", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, + { + "Action": [ + "s3:PutBucketPolicy", + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*" + ], + "Effect": "Allow", + "Principal": { + "AWS": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092", + "Arn" + ] + } + }, + "Resource": [ + { + "Fn::GetAtt": [ + "testOutput66980038", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "testOutput66980038", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + } + }, + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "Replication for this use case is uneccessary.", + "id": "HIPAA.Security-S3BucketReplicationEnabled" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "NIST.800.53.R4-S3BucketReplicationEnabled" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "NIST.800.53.R5-S3BucketReplicationEnabled" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "PCI.DSS.321-S3BucketReplicationEnabled" + } + ] + } + } + }, + "testOutputAutoDeleteObjectsCustomResource019BF1CF": { + "Type": "Custom::S3AutoDeleteObjects", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderHandler9D90184F", + "Arn" + ] + }, + "BucketName": { + "Ref": "testOutput66980038" + } + }, + "DependsOn": [ + "testOutputPolicy4B2D7F76" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "Replication for this use case is uneccessary.", + "id": "HIPAA.Security-S3BucketReplicationEnabled" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "NIST.800.53.R4-S3BucketReplicationEnabled" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "NIST.800.53.R5-S3BucketReplicationEnabled" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "PCI.DSS.321-S3BucketReplicationEnabled" + } + ] + } + } + }, + "testCircuitBreaker564CBB3F": { + "Type": "AWS::SSM::Parameter", + "Properties": { + "Type": "String", + "Value": "False" + } + }, + "testDeadLetterQueueA5F7D72D": { + "Type": "AWS::SQS::Queue", + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "testDeadLetterQueuePolicy14C080A7": { + "Type": "AWS::SQS::QueuePolicy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "sqs:*", + "Condition": { + "Bool": { + "aws:SecureTransport": "false" + } + }, + "Effect": "Deny", + "Principal": { + "AWS": "*" + }, + "Resource": { + "Fn::GetAtt": [ + "testDeadLetterQueueA5F7D72D", + "Arn" + ] + } + } + ], + "Version": "2012-10-17" + }, + "Queues": [ + { + "Ref": "testDeadLetterQueueA5F7D72D" + } + ] + } + }, + "testQueueA1A031F3": { + "Type": "AWS::SQS::Queue", + "Properties": { + "RedrivePolicy": { + "deadLetterTargetArn": { + "Fn::GetAtt": [ + "testDeadLetterQueueA5F7D72D", + "Arn" + ] + }, + "maxReceiveCount": 3 + }, + "VisibilityTimeout": 300 + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete" + }, + "testQueuePolicy2FC53456": { + "Type": "AWS::SQS::QueuePolicy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "sqs:*", + "Condition": { + "Bool": { + "aws:SecureTransport": "false" + } + }, + "Effect": "Deny", + "Principal": { + "AWS": "*" + }, + "Resource": { + "Fn::GetAtt": [ + "testQueueA1A031F3", + "Arn" + ] + } + }, + { + "Action": "sqs:SendMessage", + "Condition": { + "ArnEquals": { + "aws:SourceArn": { + "Ref": "testTopic20AA7407" + } + } + }, + "Effect": "Allow", + "Principal": { + "Service": "sns.amazonaws.com" + }, + "Resource": { + "Fn::GetAtt": [ + "testQueueA1A031F3", + "Arn" + ] + } + } + ], + "Version": "2012-10-17" + }, + "Queues": [ + { + "Ref": "testQueueA1A031F3" + } + ] + } + }, + "testQueueawsllamaindexdataloaderintegtestTopic13943F88EC3EDAB2": { + "Type": "AWS::SNS::Subscription", + "Properties": { + "Endpoint": { + "Fn::GetAtt": [ + "testQueueA1A031F3", + "Arn" + ] + }, + "Protocol": "sqs", + "TopicArn": { + "Ref": "testTopic20AA7407" + } + }, + "DependsOn": [ + "testQueuePolicy2FC53456" + ] + }, + "testClusterE41B7119": { + "Type": "AWS::ECS::Cluster", + "Properties": { + "ClusterSettings": [ + { + "Name": "containerInsights", + "Value": "enabled" + } + ] + } + }, + "testClusterVpc0A7EFC4B": { + "Type": "AWS::EC2::VPC", + "Properties": { + "CidrBlock": "10.0.0.0/16", + "EnableDnsHostnames": true, + "EnableDnsSupport": true, + "InstanceTenancy": "default", + "Tags": [ + { + "Key": "Name", + "Value": "aws-llama-index-data-loader-integ-test/test/Cluster/Vpc" + } + ] + }, + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCNoUnrestrictedRouteToIGW" + } + ] + } + } + }, + "testClusterVpcPublicSubnet1SubnetC8224979": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "AvailabilityZone": { + "Fn::Select": [ + 0, + { + "Fn::GetAZs": "" + } + ] + }, + "CidrBlock": "10.0.0.0/18", + "MapPublicIpOnLaunch": true, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Public" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Public" + }, + { + "Key": "Name", + "Value": "aws-llama-index-data-loader-integ-test/test/Cluster/Vpc/PublicSubnet1" + } + ], + "VpcId": { + "Ref": "testClusterVpc0A7EFC4B" + } + }, + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCNoUnrestrictedRouteToIGW" + } + ] + } + } + }, + "testClusterVpcPublicSubnet1RouteTable95F06E01": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "Tags": [ + { + "Key": "Name", + "Value": "aws-llama-index-data-loader-integ-test/test/Cluster/Vpc/PublicSubnet1" + } + ], + "VpcId": { + "Ref": "testClusterVpc0A7EFC4B" + } + }, + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCNoUnrestrictedRouteToIGW" + } + ] + } + } + }, + "testClusterVpcPublicSubnet1RouteTableAssociation84B5DA23": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "testClusterVpcPublicSubnet1RouteTable95F06E01" + }, + "SubnetId": { + "Ref": "testClusterVpcPublicSubnet1SubnetC8224979" + } + }, + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCNoUnrestrictedRouteToIGW" + } + ] + } + } + }, + "testClusterVpcPublicSubnet1DefaultRouteF4ABA912": { + "Type": "AWS::EC2::Route", + "Properties": { + "DestinationCidrBlock": "0.0.0.0/0", + "GatewayId": { + "Ref": "testClusterVpcIGWAAAC5F13" + }, + "RouteTableId": { + "Ref": "testClusterVpcPublicSubnet1RouteTable95F06E01" + } + }, + "DependsOn": [ + "testClusterVpcVPCGWF6B79041" + ], + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCNoUnrestrictedRouteToIGW" + } + ] + } + } + }, + "testClusterVpcPublicSubnet1EIP5C9D39EB": { + "Type": "AWS::EC2::EIP", + "Properties": { + "Domain": "vpc", + "Tags": [ + { + "Key": "Name", + "Value": "aws-llama-index-data-loader-integ-test/test/Cluster/Vpc/PublicSubnet1" + } + ] + }, + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCNoUnrestrictedRouteToIGW" + } + ] + } + } + }, + "testClusterVpcPublicSubnet1NATGateway754A80BD": { + "Type": "AWS::EC2::NatGateway", + "Properties": { + "AllocationId": { + "Fn::GetAtt": [ + "testClusterVpcPublicSubnet1EIP5C9D39EB", + "AllocationId" + ] + }, + "SubnetId": { + "Ref": "testClusterVpcPublicSubnet1SubnetC8224979" + }, + "Tags": [ + { + "Key": "Name", + "Value": "aws-llama-index-data-loader-integ-test/test/Cluster/Vpc/PublicSubnet1" + } + ] + }, + "DependsOn": [ + "testClusterVpcPublicSubnet1DefaultRouteF4ABA912", + "testClusterVpcPublicSubnet1RouteTableAssociation84B5DA23" + ], + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCNoUnrestrictedRouteToIGW" + } + ] + } + } + }, + "testClusterVpcPublicSubnet2SubnetF8C60632": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "AvailabilityZone": { + "Fn::Select": [ + 1, + { + "Fn::GetAZs": "" + } + ] + }, + "CidrBlock": "10.0.64.0/18", + "MapPublicIpOnLaunch": true, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Public" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Public" + }, + { + "Key": "Name", + "Value": "aws-llama-index-data-loader-integ-test/test/Cluster/Vpc/PublicSubnet2" + } + ], + "VpcId": { + "Ref": "testClusterVpc0A7EFC4B" + } + }, + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCNoUnrestrictedRouteToIGW" + } + ] + } + } + }, + "testClusterVpcPublicSubnet2RouteTable0DE2163B": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "Tags": [ + { + "Key": "Name", + "Value": "aws-llama-index-data-loader-integ-test/test/Cluster/Vpc/PublicSubnet2" + } + ], + "VpcId": { + "Ref": "testClusterVpc0A7EFC4B" + } + }, + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCNoUnrestrictedRouteToIGW" + } + ] + } + } + }, + "testClusterVpcPublicSubnet2RouteTableAssociation9B22B47D": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "testClusterVpcPublicSubnet2RouteTable0DE2163B" + }, + "SubnetId": { + "Ref": "testClusterVpcPublicSubnet2SubnetF8C60632" + } + }, + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCNoUnrestrictedRouteToIGW" + } + ] + } + } + }, + "testClusterVpcPublicSubnet2DefaultRouteC2E8FD46": { + "Type": "AWS::EC2::Route", + "Properties": { + "DestinationCidrBlock": "0.0.0.0/0", + "GatewayId": { + "Ref": "testClusterVpcIGWAAAC5F13" + }, + "RouteTableId": { + "Ref": "testClusterVpcPublicSubnet2RouteTable0DE2163B" + } + }, + "DependsOn": [ + "testClusterVpcVPCGWF6B79041" + ], + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCNoUnrestrictedRouteToIGW" + } + ] + } + } + }, + "testClusterVpcPublicSubnet2EIP1F54B40A": { + "Type": "AWS::EC2::EIP", + "Properties": { + "Domain": "vpc", + "Tags": [ + { + "Key": "Name", + "Value": "aws-llama-index-data-loader-integ-test/test/Cluster/Vpc/PublicSubnet2" + } + ] + }, + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCNoUnrestrictedRouteToIGW" + } + ] + } + } + }, + "testClusterVpcPublicSubnet2NATGateway58CC8B29": { + "Type": "AWS::EC2::NatGateway", + "Properties": { + "AllocationId": { + "Fn::GetAtt": [ + "testClusterVpcPublicSubnet2EIP1F54B40A", + "AllocationId" + ] + }, + "SubnetId": { + "Ref": "testClusterVpcPublicSubnet2SubnetF8C60632" + }, + "Tags": [ + { + "Key": "Name", + "Value": "aws-llama-index-data-loader-integ-test/test/Cluster/Vpc/PublicSubnet2" + } + ] + }, + "DependsOn": [ + "testClusterVpcPublicSubnet2DefaultRouteC2E8FD46", + "testClusterVpcPublicSubnet2RouteTableAssociation9B22B47D" + ], + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCNoUnrestrictedRouteToIGW" + } + ] + } + } + }, + "testClusterVpcPrivateSubnet1SubnetD8A3DD84": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "AvailabilityZone": { + "Fn::Select": [ + 0, + { + "Fn::GetAZs": "" + } + ] + }, + "CidrBlock": "10.0.128.0/18", + "MapPublicIpOnLaunch": false, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Private" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Private" + }, + { + "Key": "Name", + "Value": "aws-llama-index-data-loader-integ-test/test/Cluster/Vpc/PrivateSubnet1" + } + ], + "VpcId": { + "Ref": "testClusterVpc0A7EFC4B" + } + }, + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCNoUnrestrictedRouteToIGW" + } + ] + } + } + }, + "testClusterVpcPrivateSubnet1RouteTableAB209658": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "Tags": [ + { + "Key": "Name", + "Value": "aws-llama-index-data-loader-integ-test/test/Cluster/Vpc/PrivateSubnet1" + } + ], + "VpcId": { + "Ref": "testClusterVpc0A7EFC4B" + } + }, + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCNoUnrestrictedRouteToIGW" + } + ] + } + } + }, + "testClusterVpcPrivateSubnet1RouteTableAssociation707F217E": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "testClusterVpcPrivateSubnet1RouteTableAB209658" + }, + "SubnetId": { + "Ref": "testClusterVpcPrivateSubnet1SubnetD8A3DD84" + } + }, + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCNoUnrestrictedRouteToIGW" + } + ] + } + } + }, + "testClusterVpcPrivateSubnet1DefaultRouteB2722C76": { + "Type": "AWS::EC2::Route", + "Properties": { + "DestinationCidrBlock": "0.0.0.0/0", + "NatGatewayId": { + "Ref": "testClusterVpcPublicSubnet1NATGateway754A80BD" + }, + "RouteTableId": { + "Ref": "testClusterVpcPrivateSubnet1RouteTableAB209658" + } + }, + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCNoUnrestrictedRouteToIGW" + } + ] + } + } + }, + "testClusterVpcPrivateSubnet2Subnet9652B709": { + "Type": "AWS::EC2::Subnet", + "Properties": { + "AvailabilityZone": { + "Fn::Select": [ + 1, + { + "Fn::GetAZs": "" + } + ] + }, + "CidrBlock": "10.0.192.0/18", + "MapPublicIpOnLaunch": false, + "Tags": [ + { + "Key": "aws-cdk:subnet-name", + "Value": "Private" + }, + { + "Key": "aws-cdk:subnet-type", + "Value": "Private" + }, + { + "Key": "Name", + "Value": "aws-llama-index-data-loader-integ-test/test/Cluster/Vpc/PrivateSubnet2" + } + ], + "VpcId": { + "Ref": "testClusterVpc0A7EFC4B" + } + }, + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCNoUnrestrictedRouteToIGW" + } + ] + } + } + }, + "testClusterVpcPrivateSubnet2RouteTableA45C2A3F": { + "Type": "AWS::EC2::RouteTable", + "Properties": { + "Tags": [ + { + "Key": "Name", + "Value": "aws-llama-index-data-loader-integ-test/test/Cluster/Vpc/PrivateSubnet2" + } + ], + "VpcId": { + "Ref": "testClusterVpc0A7EFC4B" + } + }, + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCNoUnrestrictedRouteToIGW" + } + ] + } + } + }, + "testClusterVpcPrivateSubnet2RouteTableAssociation63F1ABBF": { + "Type": "AWS::EC2::SubnetRouteTableAssociation", + "Properties": { + "RouteTableId": { + "Ref": "testClusterVpcPrivateSubnet2RouteTableA45C2A3F" + }, + "SubnetId": { + "Ref": "testClusterVpcPrivateSubnet2Subnet9652B709" + } + }, + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCNoUnrestrictedRouteToIGW" + } + ] + } + } + }, + "testClusterVpcPrivateSubnet2DefaultRoute6882F728": { + "Type": "AWS::EC2::Route", + "Properties": { + "DestinationCidrBlock": "0.0.0.0/0", + "NatGatewayId": { + "Ref": "testClusterVpcPublicSubnet2NATGateway58CC8B29" + }, + "RouteTableId": { + "Ref": "testClusterVpcPrivateSubnet2RouteTableA45C2A3F" + } + }, + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCNoUnrestrictedRouteToIGW" + } + ] + } + } + }, + "testClusterVpcIGWAAAC5F13": { + "Type": "AWS::EC2::InternetGateway", + "Properties": { + "Tags": [ + { + "Key": "Name", + "Value": "aws-llama-index-data-loader-integ-test/test/Cluster/Vpc" + } + ] + }, + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCNoUnrestrictedRouteToIGW" + } + ] + } + } + }, + "testClusterVpcVPCGWF6B79041": { + "Type": "AWS::EC2::VPCGatewayAttachment", + "Properties": { + "InternetGatewayId": { + "Ref": "testClusterVpcIGWAAAC5F13" + }, + "VpcId": { + "Ref": "testClusterVpc0A7EFC4B" + } + }, + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCNoUnrestrictedRouteToIGW" + } + ] + } + } + }, + "testClusterVpcFlowLog9503638E": { + "Type": "AWS::EC2::FlowLog", + "Properties": { + "LogDestination": { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "testLogBucket7F298F3B", + "Arn" + ] + }, + "/vpc-flow-logs" + ] + ] + }, + "LogDestinationType": "s3", + "ResourceId": { + "Ref": "testClusterVpc0A7EFC4B" + }, + "ResourceType": "VPC", + "Tags": [ + { + "Key": "Name", + "Value": "aws-llama-index-data-loader-integ-test/test/Cluster/Vpc/FlowLog" + } + ], + "TrafficType": "ALL" + }, + "DependsOn": [ + "testLogBucketAutoDeleteObjectsCustomResource14E7FB3C", + "testLogBucketPolicy8C5750D7" + ], + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCSubnetAutoAssignPublicIpDisabled" + }, + { + "reason": "default vpc creation", + "id": "HIPAA.Security-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "NIST.800.53.R5-VPCNoUnrestrictedRouteToIGW" + }, + { + "reason": "default vpc creation", + "id": "PCI.DSS.321-VPCNoUnrestrictedRouteToIGW" + } + ] + } + } + }, + "testServiceQueueProcessingTaskDefTaskRoleBB1E3493": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ecs-tasks.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + }, + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "Environmental variables are okay", + "id": "AwsSolutions-ECS2" + }, + { + "reason": "The ecr:GetAuthorizationToken requires \"*\"", + "id": "AwsSolutions-IAM5", + "applies_to": [ + "Resource::*", + "Resource::/*", + "Resource::/*" + ] + }, + { + "reason": "Inline policies are acceptable for this use case.", + "id": "HIPAA.Security-IAMNoInlinePolicy" + }, + { + "reason": "Inline policies are acceptable for this use case.", + "id": "NIST.800.53.R4-IAMNoInlinePolicy" + }, + { + "reason": "Inline policies are acceptable for this use case.", + "id": "NIST.800.53.R5-IAMNoInlinePolicy" + }, + { + "reason": "Inline policies are acceptable for this use case.", + "id": "PCI.DSS.321-IAMNoInlinePolicy" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "HIPAA.Security-CloudWatchLogGroupEncrypted" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "NIST.800.53.R4-CloudWatchLogGroupEncrypted" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "NIST.800.53.R5-CloudWatchLogGroupEncrypted" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "PCI.DSS.321-CloudWatchLogGroupEncrypted" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "HIPAA.Security-CloudWatchLogGroupRetentionPeriod" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "NIST.800.53.R4-CloudWatchLogGroupRetentionPeriod" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "NIST.800.53.R5-CloudWatchLogGroupRetentionPeriod" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "PCI.DSS.321-CloudWatchLogGroupRetentionPeriod" + } + ] + } + } + }, + "testServiceQueueProcessingTaskDefTaskRoleDefaultPolicyA2314C2C": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "sqs:ReceiveMessage", + "sqs:ChangeMessageVisibility", + "sqs:GetQueueUrl", + "sqs:DeleteMessage", + "sqs:GetQueueAttributes" + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "testQueueA1A031F3", + "Arn" + ] + } + }, + { + "Action": "s3:GetObject", + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "testRawB02F8A9B", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "testRawB02F8A9B", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, + { + "Action": "kms:Decrypt", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "testLogBucketKey32CC4D1D", + "Arn" + ] + } + }, + { + "Action": "kms:GenerateDataKey", + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "testLogBucketKey32CC4D1D", + "Arn" + ] + } + }, + { + "Action": [ + "ssm:DescribeParameters", + "ssm:GetParameters", + "ssm:GetParameter", + "ssm:GetParameterHistory" + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ssm:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":parameter/", + { + "Ref": "testCircuitBreaker564CBB3F" + } + ] + ] + } + }, + { + "Action": "s3:PutObject", + "Effect": "Allow", + "Resource": [ + { + "Fn::GetAtt": [ + "testOutput66980038", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "testOutput66980038", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "testServiceQueueProcessingTaskDefTaskRoleDefaultPolicyA2314C2C", + "Roles": [ + { + "Ref": "testServiceQueueProcessingTaskDefTaskRoleBB1E3493" + } + ] + }, + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "Environmental variables are okay", + "id": "AwsSolutions-ECS2" + }, + { + "reason": "The ecr:GetAuthorizationToken requires \"*\"", + "id": "AwsSolutions-IAM5", + "applies_to": [ + "Resource::*", + "Resource::/*", + "Resource::/*" + ] + }, + { + "reason": "Inline policies are acceptable for this use case.", + "id": "HIPAA.Security-IAMNoInlinePolicy" + }, + { + "reason": "Inline policies are acceptable for this use case.", + "id": "NIST.800.53.R4-IAMNoInlinePolicy" + }, + { + "reason": "Inline policies are acceptable for this use case.", + "id": "NIST.800.53.R5-IAMNoInlinePolicy" + }, + { + "reason": "Inline policies are acceptable for this use case.", + "id": "PCI.DSS.321-IAMNoInlinePolicy" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "HIPAA.Security-CloudWatchLogGroupEncrypted" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "NIST.800.53.R4-CloudWatchLogGroupEncrypted" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "NIST.800.53.R5-CloudWatchLogGroupEncrypted" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "PCI.DSS.321-CloudWatchLogGroupEncrypted" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "HIPAA.Security-CloudWatchLogGroupRetentionPeriod" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "NIST.800.53.R4-CloudWatchLogGroupRetentionPeriod" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "NIST.800.53.R5-CloudWatchLogGroupRetentionPeriod" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "PCI.DSS.321-CloudWatchLogGroupRetentionPeriod" + } + ] + } + } + }, + "testServiceQueueProcessingTaskDefA998758D": { + "Type": "AWS::ECS::TaskDefinition", + "Properties": { + "ContainerDefinitions": [ + { + "Environment": [ + { + "Name": "CIRCUIT_BREAKER_SSM_PARAMETER_NAME", + "Value": { + "Ref": "testCircuitBreaker564CBB3F" + } + }, + { + "Name": "LOGGING_LEVEL", + "Value": "WARNING" + }, + { + "Name": "BUCKET_NAME", + "Value": { + "Ref": "testOutput66980038" + } + }, + { + "Name": "QUEUE_NAME", + "Value": { + "Fn::GetAtt": [ + "testQueueA1A031F3", + "QueueName" + ] + } + } + ], + "Essential": true, + "HealthCheck": { + "Command": [ + "CMD-SHELL", + "/usr/src/app/healthcheck.sh" + ], + "Interval": 30, + "Retries": 3, + "StartPeriod": 5, + "Timeout": 30 + }, + "Image": { + "Fn::Sub": "${AWS::AccountId}.dkr.ecr.${AWS::Region}.${AWS::URLSuffix}/cdk-hnb659fds-container-assets-${AWS::AccountId}-${AWS::Region}:dbdfbc71da28e53c4e106883b62a14fd19bbe4d7e6dfa907f9454b8f3e2360f1" + }, + "LogConfiguration": { + "LogDriver": "awslogs", + "Options": { + "awslogs-group": { + "Ref": "testServiceQueueProcessingTaskDefQueueProcessingContainerLogGroupD7294126" + }, + "awslogs-stream-prefix": "Service", + "awslogs-region": { + "Ref": "AWS::Region" + } + } + }, + "Name": "QueueProcessingContainer" + } + ], + "Cpu": "256", + "ExecutionRoleArn": { + "Fn::GetAtt": [ + "testServiceQueueProcessingTaskDefExecutionRole95E572DA", + "Arn" + ] + }, + "Family": "awsllamaindexdataloaderintegtestServiceQueueProcessingTaskDefFCE41A75", + "Memory": "2048", + "NetworkMode": "awsvpc", + "RequiresCompatibilities": [ + "FARGATE" + ], + "RuntimePlatform": { + "CpuArchitecture": "X86_64", + "OperatingSystemFamily": "LINUX" + }, + "TaskRoleArn": { + "Fn::GetAtt": [ + "testServiceQueueProcessingTaskDefTaskRoleBB1E3493", + "Arn" + ] + } + }, + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "Environmental variables are okay", + "id": "AwsSolutions-ECS2" + }, + { + "reason": "The ecr:GetAuthorizationToken requires \"*\"", + "id": "AwsSolutions-IAM5", + "applies_to": [ + "Resource::*", + "Resource::/*", + "Resource::/*" + ] + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "HIPAA.Security-CloudWatchLogGroupEncrypted" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "NIST.800.53.R4-CloudWatchLogGroupEncrypted" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "NIST.800.53.R5-CloudWatchLogGroupEncrypted" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "PCI.DSS.321-CloudWatchLogGroupEncrypted" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "HIPAA.Security-CloudWatchLogGroupRetentionPeriod" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "NIST.800.53.R4-CloudWatchLogGroupRetentionPeriod" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "NIST.800.53.R5-CloudWatchLogGroupRetentionPeriod" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "PCI.DSS.321-CloudWatchLogGroupRetentionPeriod" + } + ] + } + } + }, + "testServiceQueueProcessingTaskDefQueueProcessingContainerLogGroupD7294126": { + "Type": "AWS::Logs::LogGroup", + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain", + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "Environmental variables are okay", + "id": "AwsSolutions-ECS2" + }, + { + "reason": "The ecr:GetAuthorizationToken requires \"*\"", + "id": "AwsSolutions-IAM5", + "applies_to": [ + "Resource::*", + "Resource::/*", + "Resource::/*" + ] + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "HIPAA.Security-CloudWatchLogGroupEncrypted" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "NIST.800.53.R4-CloudWatchLogGroupEncrypted" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "NIST.800.53.R5-CloudWatchLogGroupEncrypted" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "PCI.DSS.321-CloudWatchLogGroupEncrypted" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "HIPAA.Security-CloudWatchLogGroupRetentionPeriod" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "NIST.800.53.R4-CloudWatchLogGroupRetentionPeriod" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "NIST.800.53.R5-CloudWatchLogGroupRetentionPeriod" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "PCI.DSS.321-CloudWatchLogGroupRetentionPeriod" + } + ] + } + } + }, + "testServiceQueueProcessingTaskDefExecutionRole95E572DA": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "ecs-tasks.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + } + }, + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "Environmental variables are okay", + "id": "AwsSolutions-ECS2" + }, + { + "reason": "The ecr:GetAuthorizationToken requires \"*\"", + "id": "AwsSolutions-IAM5", + "applies_to": [ + "Resource::*", + "Resource::/*", + "Resource::/*" + ] + }, + { + "reason": "Inline policies are acceptable for this use case.", + "id": "HIPAA.Security-IAMNoInlinePolicy" + }, + { + "reason": "Inline policies are acceptable for this use case.", + "id": "NIST.800.53.R4-IAMNoInlinePolicy" + }, + { + "reason": "Inline policies are acceptable for this use case.", + "id": "NIST.800.53.R5-IAMNoInlinePolicy" + }, + { + "reason": "Inline policies are acceptable for this use case.", + "id": "PCI.DSS.321-IAMNoInlinePolicy" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "HIPAA.Security-CloudWatchLogGroupEncrypted" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "NIST.800.53.R4-CloudWatchLogGroupEncrypted" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "NIST.800.53.R5-CloudWatchLogGroupEncrypted" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "PCI.DSS.321-CloudWatchLogGroupEncrypted" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "HIPAA.Security-CloudWatchLogGroupRetentionPeriod" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "NIST.800.53.R4-CloudWatchLogGroupRetentionPeriod" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "NIST.800.53.R5-CloudWatchLogGroupRetentionPeriod" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "PCI.DSS.321-CloudWatchLogGroupRetentionPeriod" + } + ] + } + } + }, + "testServiceQueueProcessingTaskDefExecutionRoleDefaultPolicy45353975": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": [ + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage" + ], + "Effect": "Allow", + "Resource": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":ecr:", + { + "Ref": "AWS::Region" + }, + ":", + { + "Ref": "AWS::AccountId" + }, + ":repository/", + { + "Fn::Sub": "cdk-hnb659fds-container-assets-${AWS::AccountId}-${AWS::Region}" + } + ] + ] + } + }, + { + "Action": "ecr:GetAuthorizationToken", + "Effect": "Allow", + "Resource": "*" + }, + { + "Action": [ + "logs:CreateLogStream", + "logs:PutLogEvents" + ], + "Effect": "Allow", + "Resource": { + "Fn::GetAtt": [ + "testServiceQueueProcessingTaskDefQueueProcessingContainerLogGroupD7294126", + "Arn" + ] + } + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "testServiceQueueProcessingTaskDefExecutionRoleDefaultPolicy45353975", + "Roles": [ + { + "Ref": "testServiceQueueProcessingTaskDefExecutionRole95E572DA" + } + ] + }, + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "Environmental variables are okay", + "id": "AwsSolutions-ECS2" + }, + { + "reason": "The ecr:GetAuthorizationToken requires \"*\"", + "id": "AwsSolutions-IAM5", + "applies_to": [ + "Resource::*", + "Resource::/*", + "Resource::/*" + ] + }, + { + "reason": "Inline policies are acceptable for this use case.", + "id": "HIPAA.Security-IAMNoInlinePolicy" + }, + { + "reason": "Inline policies are acceptable for this use case.", + "id": "NIST.800.53.R4-IAMNoInlinePolicy" + }, + { + "reason": "Inline policies are acceptable for this use case.", + "id": "NIST.800.53.R5-IAMNoInlinePolicy" + }, + { + "reason": "Inline policies are acceptable for this use case.", + "id": "PCI.DSS.321-IAMNoInlinePolicy" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "HIPAA.Security-CloudWatchLogGroupEncrypted" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "NIST.800.53.R4-CloudWatchLogGroupEncrypted" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "NIST.800.53.R5-CloudWatchLogGroupEncrypted" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "PCI.DSS.321-CloudWatchLogGroupEncrypted" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "HIPAA.Security-CloudWatchLogGroupRetentionPeriod" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "NIST.800.53.R4-CloudWatchLogGroupRetentionPeriod" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "NIST.800.53.R5-CloudWatchLogGroupRetentionPeriod" + }, + { + "reason": "CloudWatch logging is not encrypted", + "id": "PCI.DSS.321-CloudWatchLogGroupRetentionPeriod" + } + ] + } + } + }, + "testServiceQueueProcessingFargateService288F5A76": { + "Type": "AWS::ECS::Service", + "Properties": { + "Cluster": { + "Ref": "testClusterE41B7119" + }, + "DeploymentConfiguration": { + "Alarms": { + "AlarmNames": [], + "Enable": false, + "Rollback": false + }, + "MaximumPercent": 200, + "MinimumHealthyPercent": 50 + }, + "EnableECSManagedTags": false, + "LaunchType": "FARGATE", + "NetworkConfiguration": { + "AwsvpcConfiguration": { + "AssignPublicIp": "DISABLED", + "SecurityGroups": [ + { + "Fn::GetAtt": [ + "testServiceQueueProcessingFargateServiceSecurityGroup84A3E440", + "GroupId" + ] + } + ], + "Subnets": [ + { + "Ref": "testClusterVpcPrivateSubnet1SubnetD8A3DD84" + }, + { + "Ref": "testClusterVpcPrivateSubnet2Subnet9652B709" + } + ] + } + }, + "TaskDefinition": { + "Ref": "testServiceQueueProcessingTaskDefA998758D" + } + }, + "DependsOn": [ + "testServiceQueueProcessingTaskDefTaskRoleDefaultPolicyA2314C2C", + "testServiceQueueProcessingTaskDefTaskRoleBB1E3493" + ], + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "Environmental variables are okay", + "id": "AwsSolutions-ECS2" + } + ] + } + } + }, + "testServiceQueueProcessingFargateServiceSecurityGroup84A3E440": { + "Type": "AWS::EC2::SecurityGroup", + "Properties": { + "GroupDescription": "aws-llama-index-data-loader-integ-test/test/Service/QueueProcessingFargateService/SecurityGroup", + "SecurityGroupEgress": [ + { + "CidrIp": "0.0.0.0/0", + "Description": "Allow all outbound traffic by default", + "IpProtocol": "-1" + } + ], + "VpcId": { + "Ref": "testClusterVpc0A7EFC4B" + } + }, + "DependsOn": [ + "testServiceQueueProcessingTaskDefTaskRoleDefaultPolicyA2314C2C", + "testServiceQueueProcessingTaskDefTaskRoleBB1E3493" + ], + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "Environmental variables are okay", + "id": "AwsSolutions-ECS2" + } + ] + } + } + }, + "testServiceQueueProcessingFargateServiceTaskCountTarget359EAD66": { + "Type": "AWS::ApplicationAutoScaling::ScalableTarget", + "Properties": { + "MaxCapacity": 10, + "MinCapacity": 0, + "ResourceId": { + "Fn::Join": [ + "", + [ + "service/", + { + "Ref": "testClusterE41B7119" + }, + "/", + { + "Fn::GetAtt": [ + "testServiceQueueProcessingFargateService288F5A76", + "Name" + ] + } + ] + ] + }, + "RoleARN": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::", + { + "Ref": "AWS::AccountId" + }, + ":role/aws-service-role/ecs.application-autoscaling.amazonaws.com/AWSServiceRoleForApplicationAutoScaling_ECSService" + ] + ] + }, + "ScalableDimension": "ecs:service:DesiredCount", + "ServiceNamespace": "ecs" + }, + "DependsOn": [ + "testServiceQueueProcessingTaskDefTaskRoleDefaultPolicyA2314C2C", + "testServiceQueueProcessingTaskDefTaskRoleBB1E3493" + ], + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "Environmental variables are okay", + "id": "AwsSolutions-ECS2" + } + ] + } + } + }, + "testServiceQueueProcessingFargateServiceTaskCountTargetCpuScalingDA2C4394": { + "Type": "AWS::ApplicationAutoScaling::ScalingPolicy", + "Properties": { + "PolicyName": "awsllamaindexdataloaderintegtestServiceQueueProcessingFargateServiceTaskCountTargetCpuScalingE026A7FA", + "PolicyType": "TargetTrackingScaling", + "ScalingTargetId": { + "Ref": "testServiceQueueProcessingFargateServiceTaskCountTarget359EAD66" + }, + "TargetTrackingScalingPolicyConfiguration": { + "PredefinedMetricSpecification": { + "PredefinedMetricType": "ECSServiceAverageCPUUtilization" + }, + "TargetValue": 50 + } + }, + "DependsOn": [ + "testServiceQueueProcessingTaskDefTaskRoleDefaultPolicyA2314C2C", + "testServiceQueueProcessingTaskDefTaskRoleBB1E3493" + ], + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "Environmental variables are okay", + "id": "AwsSolutions-ECS2" + } + ] + } + } + }, + "testServiceQueueProcessingFargateServiceTaskCountTargetQueueMessagesVisibleScalingLowerPolicy459B950A": { + "Type": "AWS::ApplicationAutoScaling::ScalingPolicy", + "Properties": { + "PolicyName": "awsllamaindexdataloaderintegtestServiceQueueProcessingFargateServiceTaskCountTargetQueueMessagesVisibleScalingLowerPolicy60563297", + "PolicyType": "StepScaling", + "ScalingTargetId": { + "Ref": "testServiceQueueProcessingFargateServiceTaskCountTarget359EAD66" + }, + "StepScalingPolicyConfiguration": { + "AdjustmentType": "ChangeInCapacity", + "MetricAggregationType": "Maximum", + "StepAdjustments": [ + { + "MetricIntervalUpperBound": 0, + "ScalingAdjustment": -1 + } + ] + } + }, + "DependsOn": [ + "testServiceQueueProcessingTaskDefTaskRoleDefaultPolicyA2314C2C", + "testServiceQueueProcessingTaskDefTaskRoleBB1E3493" + ], + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "Environmental variables are okay", + "id": "AwsSolutions-ECS2" + } + ] + } + } + }, + "testServiceQueueProcessingFargateServiceTaskCountTargetQueueMessagesVisibleScalingLowerAlarm07FF0B76": { + "Type": "AWS::CloudWatch::Alarm", + "Properties": { + "AlarmActions": [ + { + "Ref": "testServiceQueueProcessingFargateServiceTaskCountTargetQueueMessagesVisibleScalingLowerPolicy459B950A" + } + ], + "AlarmDescription": "Lower threshold scaling alarm", + "ComparisonOperator": "LessThanOrEqualToThreshold", + "Dimensions": [ + { + "Name": "QueueName", + "Value": { + "Fn::GetAtt": [ + "testQueueA1A031F3", + "QueueName" + ] + } + } + ], + "EvaluationPeriods": 1, + "MetricName": "ApproximateNumberOfMessagesVisible", + "Namespace": "AWS/SQS", + "Period": 300, + "Statistic": "Maximum", + "Threshold": 0 + }, + "DependsOn": [ + "testServiceQueueProcessingTaskDefTaskRoleDefaultPolicyA2314C2C", + "testServiceQueueProcessingTaskDefTaskRoleBB1E3493" + ], + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "Environmental variables are okay", + "id": "AwsSolutions-ECS2" + } + ] + } + } + }, + "testServiceQueueProcessingFargateServiceTaskCountTargetQueueMessagesVisibleScalingUpperPolicy7C58C4AB": { + "Type": "AWS::ApplicationAutoScaling::ScalingPolicy", + "Properties": { + "PolicyName": "awsllamaindexdataloaderintegtestServiceQueueProcessingFargateServiceTaskCountTargetQueueMessagesVisibleScalingUpperPolicyB35BAAEC", + "PolicyType": "StepScaling", + "ScalingTargetId": { + "Ref": "testServiceQueueProcessingFargateServiceTaskCountTarget359EAD66" + }, + "StepScalingPolicyConfiguration": { + "AdjustmentType": "ChangeInCapacity", + "MetricAggregationType": "Maximum", + "StepAdjustments": [ + { + "MetricIntervalLowerBound": 0, + "MetricIntervalUpperBound": 99, + "ScalingAdjustment": 1 + }, + { + "MetricIntervalLowerBound": 99, + "MetricIntervalUpperBound": 499, + "ScalingAdjustment": 1 + }, + { + "MetricIntervalLowerBound": 499, + "ScalingAdjustment": 5 + } + ] + } + }, + "DependsOn": [ + "testServiceQueueProcessingTaskDefTaskRoleDefaultPolicyA2314C2C", + "testServiceQueueProcessingTaskDefTaskRoleBB1E3493" + ], + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "Environmental variables are okay", + "id": "AwsSolutions-ECS2" + } + ] + } + } + }, + "testServiceQueueProcessingFargateServiceTaskCountTargetQueueMessagesVisibleScalingUpperAlarm2A34497B": { + "Type": "AWS::CloudWatch::Alarm", + "Properties": { + "AlarmActions": [ + { + "Ref": "testServiceQueueProcessingFargateServiceTaskCountTargetQueueMessagesVisibleScalingUpperPolicy7C58C4AB" + } + ], + "AlarmDescription": "Upper threshold scaling alarm", + "ComparisonOperator": "GreaterThanOrEqualToThreshold", + "Dimensions": [ + { + "Name": "QueueName", + "Value": { + "Fn::GetAtt": [ + "testQueueA1A031F3", + "QueueName" + ] + } + } + ], + "EvaluationPeriods": 1, + "MetricName": "ApproximateNumberOfMessagesVisible", + "Namespace": "AWS/SQS", + "Period": 300, + "Statistic": "Maximum", + "Threshold": 1 + }, + "DependsOn": [ + "testServiceQueueProcessingTaskDefTaskRoleDefaultPolicyA2314C2C", + "testServiceQueueProcessingTaskDefTaskRoleBB1E3493" + ], + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "Environmental variables are okay", + "id": "AwsSolutions-ECS2" + } + ] + } + } + }, + "testRawB02F8A9B": { + "Type": "AWS::S3::Bucket", + "Properties": { + "BucketEncryption": { + "ServerSideEncryptionConfiguration": [ + { + "BucketKeyEnabled": true, + "ServerSideEncryptionByDefault": { + "KMSMasterKeyID": { + "Fn::GetAtt": [ + "testLogBucketKey32CC4D1D", + "Arn" + ] + }, + "SSEAlgorithm": "aws:kms" + } + } + ] + }, + "LifecycleConfiguration": { + "Rules": [ + { + "AbortIncompleteMultipartUpload": { + "DaysAfterInitiation": 1 + }, + "Id": "AbortIncompleteMultipartUpload", + "Status": "Enabled" + } + ] + }, + "LoggingConfiguration": { + "DestinationBucketName": { + "Ref": "testLogBucket7F298F3B" + }, + "LogFilePrefix": "raw-bucket-access-logs" + }, + "ObjectLockConfiguration": { + "ObjectLockEnabled": "Enabled", + "Rule": { + "DefaultRetention": { + "Days": 1, + "Mode": "GOVERNANCE" + } + } + }, + "ObjectLockEnabled": true, + "PublicAccessBlockConfiguration": { + "BlockPublicAcls": true, + "BlockPublicPolicy": true, + "IgnorePublicAcls": true, + "RestrictPublicBuckets": true + }, + "Tags": [ + { + "Key": "aws-cdk:auto-delete-objects", + "Value": "true" + } + ], + "VersioningConfiguration": { + "Status": "Enabled" + } + }, + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "Replication for this use case is uneccessary.", + "id": "HIPAA.Security-S3BucketReplicationEnabled" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "NIST.800.53.R4-S3BucketReplicationEnabled" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "NIST.800.53.R5-S3BucketReplicationEnabled" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "PCI.DSS.321-S3BucketReplicationEnabled" + } + ] + } + } + }, + "testRawPolicy5CC638A9": { + "Type": "AWS::S3::BucketPolicy", + "Properties": { + "Bucket": { + "Ref": "testRawB02F8A9B" + }, + "PolicyDocument": { + "Statement": [ + { + "Action": "s3:*", + "Condition": { + "Bool": { + "aws:SecureTransport": "false" + } + }, + "Effect": "Deny", + "Principal": { + "AWS": "*" + }, + "Resource": [ + { + "Fn::GetAtt": [ + "testRawB02F8A9B", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "testRawB02F8A9B", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + }, + { + "Action": [ + "s3:PutBucketPolicy", + "s3:GetBucket*", + "s3:List*", + "s3:DeleteObject*" + ], + "Effect": "Allow", + "Principal": { + "AWS": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092", + "Arn" + ] + } + }, + "Resource": [ + { + "Fn::GetAtt": [ + "testRawB02F8A9B", + "Arn" + ] + }, + { + "Fn::Join": [ + "", + [ + { + "Fn::GetAtt": [ + "testRawB02F8A9B", + "Arn" + ] + }, + "/*" + ] + ] + } + ] + } + ], + "Version": "2012-10-17" + } + }, + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "Replication for this use case is uneccessary.", + "id": "HIPAA.Security-S3BucketReplicationEnabled" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "NIST.800.53.R4-S3BucketReplicationEnabled" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "NIST.800.53.R5-S3BucketReplicationEnabled" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "PCI.DSS.321-S3BucketReplicationEnabled" + } + ] + } + } + }, + "testRawAutoDeleteObjectsCustomResource75A03DE7": { + "Type": "Custom::S3AutoDeleteObjects", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderHandler9D90184F", + "Arn" + ] + }, + "BucketName": { + "Ref": "testRawB02F8A9B" + } + }, + "DependsOn": [ + "testRawPolicy5CC638A9" + ], + "UpdateReplacePolicy": "Delete", + "DeletionPolicy": "Delete", + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "Replication for this use case is uneccessary.", + "id": "HIPAA.Security-S3BucketReplicationEnabled" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "NIST.800.53.R4-S3BucketReplicationEnabled" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "NIST.800.53.R5-S3BucketReplicationEnabled" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "PCI.DSS.321-S3BucketReplicationEnabled" + } + ] + } + } + }, + "testRawNotificationsE49FC2F5": { + "Type": "Custom::S3BucketNotifications", + "Properties": { + "ServiceToken": { + "Fn::GetAtt": [ + "BucketNotificationsHandler050a0587b7544547bf325f094a3db8347ECC3691", + "Arn" + ] + }, + "BucketName": { + "Ref": "testRawB02F8A9B" + }, + "NotificationConfiguration": { + "TopicConfigurations": [ + { + "Events": [ + "s3:ObjectCreated:*" + ], + "TopicArn": { + "Ref": "testTopic20AA7407" + } + } + ] + }, + "Managed": true, + "SkipDestinationValidation": false + }, + "DependsOn": [ + "testRawPolicy5CC638A9", + "testTopicPolicy6FF7D7A8", + "testTopic20AA7407" + ], + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "Replication for this use case is uneccessary.", + "id": "HIPAA.Security-S3BucketReplicationEnabled" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "NIST.800.53.R4-S3BucketReplicationEnabled" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "NIST.800.53.R5-S3BucketReplicationEnabled" + }, + { + "reason": "Replication for this use case is uneccessary.", + "id": "PCI.DSS.321-S3BucketReplicationEnabled" + } + ] + } + } + }, + "testTopicKey6AD3C1A9": { + "Type": "AWS::KMS::Key", + "Properties": { + "EnableKeyRotation": true, + "KeyPolicy": { + "Statement": [ + { + "Action": "kms:*", + "Effect": "Allow", + "Principal": { + "AWS": { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::", + { + "Ref": "AWS::AccountId" + }, + ":root" + ] + ] + } + }, + "Resource": "*" + }, + { + "Action": [ + "kms:Decrypt", + "kms:GenerateDataKey*" + ], + "Effect": "Allow", + "Principal": { + "Service": "s3.amazonaws.com" + }, + "Resource": "*" + } + ], + "Version": "2012-10-17" + } + }, + "UpdateReplacePolicy": "Retain", + "DeletionPolicy": "Retain" + }, + "testTopic20AA7407": { + "Type": "AWS::SNS::Topic", + "Properties": { + "KmsMasterKeyId": { + "Fn::GetAtt": [ + "testTopicKey6AD3C1A9", + "Arn" + ] + } + } + }, + "testTopicPolicy6FF7D7A8": { + "Type": "AWS::SNS::TopicPolicy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "sns:Publish", + "Effect": "Allow", + "Principal": { + "Service": "s3.amazonaws.com" + }, + "Resource": { + "Ref": "testTopic20AA7407" + }, + "Sid": "0" + }, + { + "Action": "sns:Publish", + "Condition": { + "Bool": { + "aws:SecureTransport": "false" + } + }, + "Effect": "Deny", + "Principal": "*", + "Resource": { + "Ref": "testTopic20AA7407" + }, + "Sid": "AllowPublishThroughSSLOnly" + }, + { + "Action": "sns:Publish", + "Condition": { + "ArnLike": { + "aws:SourceArn": { + "Fn::GetAtt": [ + "testRawB02F8A9B", + "Arn" + ] + } + } + }, + "Effect": "Allow", + "Principal": { + "Service": "s3.amazonaws.com" + }, + "Resource": { + "Ref": "testTopic20AA7407" + }, + "Sid": "2" + } + ], + "Version": "2012-10-17" + }, + "Topics": [ + { + "Ref": "testTopic20AA7407" + } + ] + } + }, + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Version": "2012-10-17", + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ] + }, + "ManagedPolicyArns": [ + { + "Fn::Sub": "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + } + ] + } + }, + "CustomS3AutoDeleteObjectsCustomResourceProviderHandler9D90184F": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Code": { + "S3Bucket": { + "Fn::Sub": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}" + }, + "S3Key": "faa95a81ae7d7373f3e1f242268f904eb748d8d0fdd306e8a6fe515a1905a7d6.zip" + }, + "Timeout": 900, + "MemorySize": 128, + "Handler": "index.handler", + "Role": { + "Fn::GetAtt": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092", + "Arn" + ] + }, + "Runtime": { + "Fn::FindInMap": [ + "LatestNodeRuntimeMap", + { + "Ref": "AWS::Region" + }, + "value" + ] + }, + "Description": { + "Fn::Join": [ + "", + [ + "Lambda function for auto-deleting objects in ", + { + "Ref": "testLogBucket7F298F3B" + }, + " S3 bucket." + ] + ] + } + }, + "DependsOn": [ + "CustomS3AutoDeleteObjectsCustomResourceProviderRole3B1BD092" + ] + }, + "BucketNotificationsHandler050a0587b7544547bf325f094a3db834RoleB6FB88EC": { + "Type": "AWS::IAM::Role", + "Properties": { + "AssumeRolePolicyDocument": { + "Statement": [ + { + "Action": "sts:AssumeRole", + "Effect": "Allow", + "Principal": { + "Service": "lambda.amazonaws.com" + } + } + ], + "Version": "2012-10-17" + }, + "ManagedPolicyArns": [ + { + "Fn::Join": [ + "", + [ + "arn:", + { + "Ref": "AWS::Partition" + }, + ":iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ] + ] + } + ] + }, + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "AWSLambdaBasicExecutionRole is used by `addEventNotification`", + "id": "AwsSolutions-IAM4", + "applies_to": [ + "Policy::arn::iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ] + }, + { + "reason": "Suppress all AwsSolutions-IAM5 findings for this stack", + "id": "AwsSolutions-IAM5", + "applies_to": [ + "Resource::*" + ] + }, + { + "reason": "Inline policies are part of the BucketHandlerNotification", + "id": "HIPAA.Security-IAMNoInlinePolicy" + }, + { + "reason": "Inline policies are part of the BucketHandlerNotification", + "id": "NIST.800.53.R4-IAMNoInlinePolicy" + }, + { + "reason": "Inline policies are part of the BucketHandlerNotification", + "id": "NIST.800.53.R5-IAMNoInlinePolicy" + }, + { + "reason": "Inline policies are part of the BucketHandlerNotification", + "id": "PCI.DSS.321-IAMNoInlinePolicy" + } + ] + } + } + }, + "BucketNotificationsHandler050a0587b7544547bf325f094a3db834RoleDefaultPolicy2CF63D36": { + "Type": "AWS::IAM::Policy", + "Properties": { + "PolicyDocument": { + "Statement": [ + { + "Action": "s3:PutBucketNotification", + "Effect": "Allow", + "Resource": "*" + } + ], + "Version": "2012-10-17" + }, + "PolicyName": "BucketNotificationsHandler050a0587b7544547bf325f094a3db834RoleDefaultPolicy2CF63D36", + "Roles": [ + { + "Ref": "BucketNotificationsHandler050a0587b7544547bf325f094a3db834RoleB6FB88EC" + } + ] + }, + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "AWSLambdaBasicExecutionRole is used by `addEventNotification`", + "id": "AwsSolutions-IAM4", + "applies_to": [ + "Policy::arn::iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ] + }, + { + "reason": "Suppress all AwsSolutions-IAM5 findings for this stack", + "id": "AwsSolutions-IAM5", + "applies_to": [ + "Resource::*" + ] + }, + { + "reason": "Inline policies are part of the BucketHandlerNotification", + "id": "HIPAA.Security-IAMNoInlinePolicy" + }, + { + "reason": "Inline policies are part of the BucketHandlerNotification", + "id": "NIST.800.53.R4-IAMNoInlinePolicy" + }, + { + "reason": "Inline policies are part of the BucketHandlerNotification", + "id": "NIST.800.53.R5-IAMNoInlinePolicy" + }, + { + "reason": "Inline policies are part of the BucketHandlerNotification", + "id": "PCI.DSS.321-IAMNoInlinePolicy" + } + ] + } + } + }, + "BucketNotificationsHandler050a0587b7544547bf325f094a3db8347ECC3691": { + "Type": "AWS::Lambda::Function", + "Properties": { + "Description": "AWS CloudFormation handler for \"Custom::S3BucketNotifications\" resources (@aws-cdk/aws-s3)", + "Code": { + "ZipFile": "import boto3 # type: ignore\nimport json\nimport logging\nimport urllib.request\n\ns3 = boto3.client(\"s3\")\n\nEVENTBRIDGE_CONFIGURATION = 'EventBridgeConfiguration'\nCONFIGURATION_TYPES = [\"TopicConfigurations\", \"QueueConfigurations\", \"LambdaFunctionConfigurations\"]\n\ndef handler(event: dict, context):\n response_status = \"SUCCESS\"\n error_message = \"\"\n try:\n props = event[\"ResourceProperties\"]\n notification_configuration = props[\"NotificationConfiguration\"]\n managed = props.get('Managed', 'true').lower() == 'true'\n skipDestinationValidation = props.get('SkipDestinationValidation', 'false').lower() == 'true'\n stack_id = event['StackId']\n old = event.get(\"OldResourceProperties\", {}).get(\"NotificationConfiguration\", {})\n if managed:\n config = handle_managed(event[\"RequestType\"], notification_configuration)\n else:\n config = handle_unmanaged(props[\"BucketName\"], stack_id, event[\"RequestType\"], notification_configuration, old)\n s3.put_bucket_notification_configuration(Bucket=props[\"BucketName\"], NotificationConfiguration=config, SkipDestinationValidation=skipDestinationValidation)\n except Exception as e:\n logging.exception(\"Failed to put bucket notification configuration\")\n response_status = \"FAILED\"\n error_message = f\"Error: {str(e)}. \"\n finally:\n submit_response(event, context, response_status, error_message)\n\ndef handle_managed(request_type, notification_configuration):\n if request_type == 'Delete':\n return {}\n return notification_configuration\n\ndef handle_unmanaged(bucket, stack_id, request_type, notification_configuration, old):\n def get_id(n):\n n['Id'] = ''\n sorted_notifications = sort_filter_rules(n)\n strToHash=json.dumps(sorted_notifications, sort_keys=True).replace('\"Name\": \"prefix\"', '\"Name\": \"Prefix\"').replace('\"Name\": \"suffix\"', '\"Name\": \"Suffix\"')\n return f\"{stack_id}-{hash(strToHash)}\"\n def with_id(n):\n n['Id'] = get_id(n)\n return n\n\n external_notifications = {}\n existing_notifications = s3.get_bucket_notification_configuration(Bucket=bucket)\n for t in CONFIGURATION_TYPES:\n if request_type == 'Update':\n old_incoming_ids = [get_id(n) for n in old.get(t, [])]\n external_notifications[t] = [n for n in existing_notifications.get(t, []) if not get_id(n) in old_incoming_ids] \n elif request_type == 'Delete':\n external_notifications[t] = [n for n in existing_notifications.get(t, []) if not n['Id'].startswith(f\"{stack_id}-\")]\n elif request_type == 'Create':\n external_notifications[t] = [n for n in existing_notifications.get(t, [])]\n if EVENTBRIDGE_CONFIGURATION in existing_notifications:\n external_notifications[EVENTBRIDGE_CONFIGURATION] = existing_notifications[EVENTBRIDGE_CONFIGURATION]\n\n if request_type == 'Delete':\n return external_notifications\n\n notifications = {}\n for t in CONFIGURATION_TYPES:\n external = external_notifications.get(t, [])\n incoming = [with_id(n) for n in notification_configuration.get(t, [])]\n notifications[t] = external + incoming\n\n if EVENTBRIDGE_CONFIGURATION in notification_configuration:\n notifications[EVENTBRIDGE_CONFIGURATION] = notification_configuration[EVENTBRIDGE_CONFIGURATION]\n elif EVENTBRIDGE_CONFIGURATION in external_notifications:\n notifications[EVENTBRIDGE_CONFIGURATION] = external_notifications[EVENTBRIDGE_CONFIGURATION]\n\n return notifications\n\ndef submit_response(event: dict, context, response_status: str, error_message: str):\n response_body = json.dumps(\n {\n \"Status\": response_status,\n \"Reason\": f\"{error_message}See the details in CloudWatch Log Stream: {context.log_stream_name}\",\n \"PhysicalResourceId\": event.get(\"PhysicalResourceId\") or event[\"LogicalResourceId\"],\n \"StackId\": event[\"StackId\"],\n \"RequestId\": event[\"RequestId\"],\n \"LogicalResourceId\": event[\"LogicalResourceId\"],\n \"NoEcho\": False,\n }\n ).encode(\"utf-8\")\n headers = {\"content-type\": \"\", \"content-length\": str(len(response_body))}\n try:\n req = urllib.request.Request(url=event[\"ResponseURL\"], headers=headers, data=response_body, method=\"PUT\")\n with urllib.request.urlopen(req) as response:\n print(response.read().decode(\"utf-8\"))\n print(\"Status code: \" + response.reason)\n except Exception as e:\n print(\"send(..) failed executing request.urlopen(..): \" + str(e))\n\ndef sort_filter_rules(json_obj):\n if not isinstance(json_obj, dict):\n return json_obj\n for key, value in json_obj.items():\n if isinstance(value, dict):\n json_obj[key] = sort_filter_rules(value)\n elif isinstance(value, list):\n json_obj[key] = [sort_filter_rules(item) for item in value]\n if \"Filter\" in json_obj and \"Key\" in json_obj[\"Filter\"] and \"FilterRules\" in json_obj[\"Filter\"][\"Key\"]:\n filter_rules = json_obj[\"Filter\"][\"Key\"][\"FilterRules\"]\n sorted_filter_rules = sorted(filter_rules, key=lambda x: x[\"Name\"])\n json_obj[\"Filter\"][\"Key\"][\"FilterRules\"] = sorted_filter_rules\n return json_obj" + }, + "Handler": "index.handler", + "Role": { + "Fn::GetAtt": [ + "BucketNotificationsHandler050a0587b7544547bf325f094a3db834RoleB6FB88EC", + "Arn" + ] + }, + "Runtime": "python3.11", + "Timeout": 300 + }, + "DependsOn": [ + "BucketNotificationsHandler050a0587b7544547bf325f094a3db834RoleDefaultPolicy2CF63D36", + "BucketNotificationsHandler050a0587b7544547bf325f094a3db834RoleB6FB88EC" + ], + "Metadata": { + "cdk_nag": { + "rules_to_suppress": [ + { + "reason": "AWSLambdaBasicExecutionRole is used by `addEventNotification`", + "id": "AwsSolutions-IAM4", + "applies_to": [ + "Policy::arn::iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + ] + }, + { + "reason": "Suppress all AwsSolutions-IAM5 findings for this stack", + "id": "AwsSolutions-IAM5", + "applies_to": [ + "Resource::*" + ] + }, + { + "reason": "Inline policies are part of the BucketHandlerNotification", + "id": "HIPAA.Security-IAMNoInlinePolicy" + }, + { + "reason": "Inline policies are part of the BucketHandlerNotification", + "id": "NIST.800.53.R4-IAMNoInlinePolicy" + }, + { + "reason": "Inline policies are part of the BucketHandlerNotification", + "id": "NIST.800.53.R5-IAMNoInlinePolicy" + }, + { + "reason": "Inline policies are part of the BucketHandlerNotification", + "id": "PCI.DSS.321-IAMNoInlinePolicy" + } + ] + } + } + } + }, + "Outputs": { + "testServiceSQSQueue2827F52D": { + "Value": { + "Fn::GetAtt": [ + "testQueueA1A031F3", + "QueueName" + ] + } + }, + "testServiceSQSQueueArnCF38F668": { + "Value": { + "Fn::GetAtt": [ + "testQueueA1A031F3", + "Arn" + ] + } + } + }, + "Mappings": { + "LatestNodeRuntimeMap": { + "af-south-1": { + "value": "nodejs20.x" + }, + "ap-east-1": { + "value": "nodejs20.x" + }, + "ap-northeast-1": { + "value": "nodejs20.x" + }, + "ap-northeast-2": { + "value": "nodejs20.x" + }, + "ap-northeast-3": { + "value": "nodejs20.x" + }, + "ap-south-1": { + "value": "nodejs20.x" + }, + "ap-south-2": { + "value": "nodejs20.x" + }, + "ap-southeast-1": { + "value": "nodejs20.x" + }, + "ap-southeast-2": { + "value": "nodejs20.x" + }, + "ap-southeast-3": { + "value": "nodejs20.x" + }, + "ap-southeast-4": { + "value": "nodejs20.x" + }, + "ap-southeast-5": { + "value": "nodejs20.x" + }, + "ap-southeast-7": { + "value": "nodejs20.x" + }, + "ca-central-1": { + "value": "nodejs20.x" + }, + "ca-west-1": { + "value": "nodejs20.x" + }, + "cn-north-1": { + "value": "nodejs18.x" + }, + "cn-northwest-1": { + "value": "nodejs18.x" + }, + "eu-central-1": { + "value": "nodejs20.x" + }, + "eu-central-2": { + "value": "nodejs20.x" + }, + "eu-isoe-west-1": { + "value": "nodejs18.x" + }, + "eu-north-1": { + "value": "nodejs20.x" + }, + "eu-south-1": { + "value": "nodejs20.x" + }, + "eu-south-2": { + "value": "nodejs20.x" + }, + "eu-west-1": { + "value": "nodejs20.x" + }, + "eu-west-2": { + "value": "nodejs20.x" + }, + "eu-west-3": { + "value": "nodejs20.x" + }, + "il-central-1": { + "value": "nodejs20.x" + }, + "me-central-1": { + "value": "nodejs20.x" + }, + "me-south-1": { + "value": "nodejs20.x" + }, + "mx-central-1": { + "value": "nodejs20.x" + }, + "sa-east-1": { + "value": "nodejs20.x" + }, + "us-east-1": { + "value": "nodejs20.x" + }, + "us-east-2": { + "value": "nodejs20.x" + }, + "us-gov-east-1": { + "value": "nodejs18.x" + }, + "us-gov-west-1": { + "value": "nodejs18.x" + }, + "us-iso-east-1": { + "value": "nodejs18.x" + }, + "us-iso-west-1": { + "value": "nodejs18.x" + }, + "us-isob-east-1": { + "value": "nodejs18.x" + }, + "us-west-1": { + "value": "nodejs20.x" + }, + "us-west-2": { + "value": "nodejs20.x" + } + } + }, + "Parameters": { + "BootstrapVersion": { + "Type": "AWS::SSM::Parameter::Value", + "Default": "/cdk-bootstrap/hnb659fds/version", + "Description": "Version of the CDK Bootstrap resources in this environment, automatically retrieved from SSM Parameter Store. [cdk:skip]" + } + }, + "Rules": { + "CheckBootstrapVersion": { + "Assertions": [ + { + "Assert": { + "Fn::Not": [ + { + "Fn::Contains": [ + [ + "1", + "2", + "3", + "4", + "5" + ], + { + "Ref": "BootstrapVersion" + } + ] + } + ] + }, + "AssertDescription": "CDK bootstrap stack version 6 required. Please run 'cdk bootstrap' with a recent version of the CDK CLI." + } + ] + } + } +} \ No newline at end of file diff --git a/test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.snapshot/integ.json b/test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.snapshot/integ.json new file mode 100644 index 00000000..2d280671 --- /dev/null +++ b/test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.snapshot/integ.json @@ -0,0 +1,19 @@ +{ + "version": "38.0.1", + "testCases": { + "ServiceTest/DefaultTest": { + "stacks": [ + "aws-llama-index-data-loader-integ-test" + ], + "cdkCommandOptions": { + "destroy": { + "args": { + "force": true + } + } + }, + "assertionStack": "ServiceTest/DefaultTest/DeployAssert", + "assertionStackName": "ServiceTestDefaultTestDeployAssertE49B1ECE" + } + } +} \ No newline at end of file diff --git a/test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.ts b/test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.ts new file mode 100644 index 00000000..dd19b82e --- /dev/null +++ b/test/patterns/gen-ai/aws-llama-index-data-loader/integ-tests/aws-llama-index-data-loader.integ.ts @@ -0,0 +1,35 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance + * with the License. A copy of the License is located at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * or in the 'license' file accompanying this file. This file is distributed on an 'AS IS' BASIS, WITHOUT WARRANTIES + * OR CONDITIONS OF ANY KIND, express or implied. See the License for the specific language governing permissions + * and limitations under the License. + */ +import * as integ from '@aws-cdk/integ-tests-alpha'; +import * as cdk from 'aws-cdk-lib'; +import { + LlamaIndexDataLoader, +} from '../../../../../src/patterns/gen-ai/aws-llama-index-data-loader'; + +const app = new cdk.App(); +const stack = new cdk.Stack(app, 'aws-llama-index-data-loader-integ-test'); +new LlamaIndexDataLoader(stack, 'test', {}); + +// const integ_case = +new integ.IntegTest(app, 'ServiceTest', { + testCases: [stack], + cdkCommandOptions: { + destroy: { + args: { + force: true, + }, + }, + }, +}); + +app.synth();