feat(client-firehose): Features : Adding support for new data ingesti…

…on source to Kinesis Firehose - AWS Managed Services Kafka.
aws · Sep 27, 2023 · f392d88 · f392d88
1 parent 114788d
commit f392d88
Show file tree

Hide file tree

Showing 10 changed files with 344 additions and 41 deletions.
diff --git a/clients/client-firehose/src/commands/CreateDeliveryStreamCommand.ts b/clients/client-firehose/src/commands/CreateDeliveryStreamCommand.ts
@@ -118,7 +118,7 @@ export interface CreateDeliveryStreamCommandOutput extends CreateDeliveryStreamO
  * const client = new FirehoseClient(config);
  * const input = { // CreateDeliveryStreamInput
  *   DeliveryStreamName: "STRING_VALUE", // required
- *   DeliveryStreamType: "DirectPut" || "KinesisStreamAsSource",
+ *   DeliveryStreamType: "DirectPut" || "KinesisStreamAsSource" || "MSKAsSource",
  *   KinesisStreamSourceConfiguration: { // KinesisStreamSourceConfiguration
  *     KinesisStreamARN: "STRING_VALUE", // required
  *     RoleARN: "STRING_VALUE", // required
@@ -174,10 +174,10 @@ export interface CreateDeliveryStreamCommandOutput extends CreateDeliveryStreamO
  *       Enabled: true || false,
  *       Processors: [ // ProcessorList
  *         { // Processor
- *           Type: "RecordDeAggregation" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
+ *           Type: "RecordDeAggregation" || "Decompression" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
  *           Parameters: [ // ProcessorParameterList
  *             { // ProcessorParameter
- *               ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter", // required
+ *               ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter" || "CompressionFormat", // required
  *               ParameterValue: "STRING_VALUE", // required
  *             },
  *           ],
@@ -306,10 +306,10 @@ export interface CreateDeliveryStreamCommandOutput extends CreateDeliveryStreamO
  *       Enabled: true || false,
  *       Processors: [
  *         {
- *           Type: "RecordDeAggregation" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
+ *           Type: "RecordDeAggregation" || "Decompression" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
  *           Parameters: [
  *             {
- *               ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter", // required
+ *               ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter" || "CompressionFormat", // required
  *               ParameterValue: "STRING_VALUE", // required
  *             },
  *           ],
@@ -370,10 +370,10 @@ export interface CreateDeliveryStreamCommandOutput extends CreateDeliveryStreamO
  *       Enabled: true || false,
  *       Processors: [
  *         {
- *           Type: "RecordDeAggregation" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
+ *           Type: "RecordDeAggregation" || "Decompression" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
  *           Parameters: [
  *             {
- *               ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter", // required
+ *               ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter" || "CompressionFormat", // required
  *               ParameterValue: "STRING_VALUE", // required
  *             },
  *           ],
@@ -414,10 +414,10 @@ export interface CreateDeliveryStreamCommandOutput extends CreateDeliveryStreamO
  *       Enabled: true || false,
  *       Processors: [
  *         {
- *           Type: "RecordDeAggregation" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
+ *           Type: "RecordDeAggregation" || "Decompression" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
  *           Parameters: [
  *             {
- *               ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter", // required
+ *               ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter" || "CompressionFormat", // required
  *               ParameterValue: "STRING_VALUE", // required
  *             },
  *           ],
@@ -452,10 +452,10 @@ export interface CreateDeliveryStreamCommandOutput extends CreateDeliveryStreamO
  *       Enabled: true || false,
  *       Processors: [
  *         {
- *           Type: "RecordDeAggregation" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
+ *           Type: "RecordDeAggregation" || "Decompression" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
  *           Parameters: [
  *             {
- *               ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter", // required
+ *               ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter" || "CompressionFormat", // required
  *               ParameterValue: "STRING_VALUE", // required
  *             },
  *           ],
@@ -523,6 +523,14 @@ export interface CreateDeliveryStreamCommandOutput extends CreateDeliveryStreamO
  *       ],
  *     },
  *   },
+ *   MSKSourceConfiguration: { // MSKSourceConfiguration
+ *     MSKClusterARN: "STRING_VALUE", // required
+ *     TopicName: "STRING_VALUE", // required
+ *     AuthenticationConfiguration: { // AuthenticationConfiguration
+ *       RoleARN: "STRING_VALUE", // required
+ *       Connectivity: "PUBLIC" || "PRIVATE", // required
+ *     },
+ *   },
  * };
  * const command = new CreateDeliveryStreamCommand(input);
  * const response = await client.send(command);

diff --git a/clients/client-firehose/src/commands/DescribeDeliveryStreamCommand.ts b/clients/client-firehose/src/commands/DescribeDeliveryStreamCommand.ts
@@ -79,7 +79,7 @@ export interface DescribeDeliveryStreamCommandOutput extends DescribeDeliveryStr
  * //         Details: "STRING_VALUE", // required
  * //       },
  * //     },
- * //     DeliveryStreamType: "DirectPut" || "KinesisStreamAsSource", // required
+ * //     DeliveryStreamType: "DirectPut" || "KinesisStreamAsSource" || "MSKAsSource", // required
  * //     VersionId: "STRING_VALUE", // required
  * //     CreateTimestamp: new Date("TIMESTAMP"),
  * //     LastUpdateTimestamp: new Date("TIMESTAMP"),
@@ -89,6 +89,15 @@ export interface DescribeDeliveryStreamCommandOutput extends DescribeDeliveryStr
  * //         RoleARN: "STRING_VALUE",
  * //         DeliveryStartTimestamp: new Date("TIMESTAMP"),
  * //       },
+ * //       MSKSourceDescription: { // MSKSourceDescription
+ * //         MSKClusterARN: "STRING_VALUE",
+ * //         TopicName: "STRING_VALUE",
+ * //         AuthenticationConfiguration: { // AuthenticationConfiguration
+ * //           RoleARN: "STRING_VALUE", // required
+ * //           Connectivity: "PUBLIC" || "PRIVATE", // required
+ * //         },
+ * //         DeliveryStartTimestamp: new Date("TIMESTAMP"),
+ * //       },
  * //     },
  * //     Destinations: [ // DestinationDescriptionList // required
  * //       { // DestinationDescription
@@ -140,10 +149,10 @@ export interface DescribeDeliveryStreamCommandOutput extends DescribeDeliveryStr
  * //             Enabled: true || false,
  * //             Processors: [ // ProcessorList
  * //               { // Processor
- * //                 Type: "RecordDeAggregation" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
+ * //                 Type: "RecordDeAggregation" || "Decompression" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
  * //                 Parameters: [ // ProcessorParameterList
  * //                   { // ProcessorParameter
- * //                     ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter", // required
+ * //                     ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter" || "CompressionFormat", // required
  * //                     ParameterValue: "STRING_VALUE", // required
  * //                   },
  * //                 ],
@@ -271,10 +280,10 @@ export interface DescribeDeliveryStreamCommandOutput extends DescribeDeliveryStr
  * //             Enabled: true || false,
  * //             Processors: [
  * //               {
- * //                 Type: "RecordDeAggregation" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
+ * //                 Type: "RecordDeAggregation" || "Decompression" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
  * //                 Parameters: [
  * //                   {
- * //                     ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter", // required
+ * //                     ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter" || "CompressionFormat", // required
  * //                     ParameterValue: "STRING_VALUE", // required
  * //                   },
  * //                 ],
@@ -335,10 +344,10 @@ export interface DescribeDeliveryStreamCommandOutput extends DescribeDeliveryStr
  * //             Enabled: true || false,
  * //             Processors: [
  * //               {
- * //                 Type: "RecordDeAggregation" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
+ * //                 Type: "RecordDeAggregation" || "Decompression" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
  * //                 Parameters: [
  * //                   {
- * //                     ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter", // required
+ * //                     ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter" || "CompressionFormat", // required
  * //                     ParameterValue: "STRING_VALUE", // required
  * //                   },
  * //                 ],
@@ -380,10 +389,10 @@ export interface DescribeDeliveryStreamCommandOutput extends DescribeDeliveryStr
  * //             Enabled: true || false,
  * //             Processors: [
  * //               {
- * //                 Type: "RecordDeAggregation" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
+ * //                 Type: "RecordDeAggregation" || "Decompression" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
  * //                 Parameters: [
  * //                   {
- * //                     ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter", // required
+ * //                     ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter" || "CompressionFormat", // required
  * //                     ParameterValue: "STRING_VALUE", // required
  * //                   },
  * //                 ],
@@ -419,10 +428,10 @@ export interface DescribeDeliveryStreamCommandOutput extends DescribeDeliveryStr
  * //             Enabled: true || false,
  * //             Processors: [
  * //               {
- * //                 Type: "RecordDeAggregation" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
+ * //                 Type: "RecordDeAggregation" || "Decompression" || "Lambda" || "MetadataExtraction" || "AppendDelimiterToRecord", // required
  * //                 Parameters: [
  * //                   {
- * //                     ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter", // required
+ * //                     ParameterName: "LambdaArn" || "NumberOfRetries" || "MetadataExtractionQuery" || "JsonParsingEngine" || "RoleArn" || "BufferSizeInMBs" || "BufferIntervalInSeconds" || "SubRecordType" || "Delimiter" || "CompressionFormat", // required
  * //                     ParameterValue: "STRING_VALUE", // required
  * //                   },
  * //                 ],

diff --git a/clients/client-firehose/src/commands/ListDeliveryStreamsCommand.ts b/clients/client-firehose/src/commands/ListDeliveryStreamsCommand.ts
@@ -53,7 +53,7 @@ export interface ListDeliveryStreamsCommandOutput extends ListDeliveryStreamsOut
  * const client = new FirehoseClient(config);
  * const input = { // ListDeliveryStreamsInput
  *   Limit: Number("int"),
- *   DeliveryStreamType: "DirectPut" || "KinesisStreamAsSource",
+ *   DeliveryStreamType: "DirectPut" || "KinesisStreamAsSource" || "MSKAsSource",
  *   ExclusiveStartDeliveryStreamName: "STRING_VALUE",
  * };
  * const command = new ListDeliveryStreamsCommand(input);

diff --git a/clients/client-firehose/src/commands/PutRecordBatchCommand.ts b/clients/client-firehose/src/commands/PutRecordBatchCommand.ts
@@ -41,6 +41,7 @@ export interface PutRecordBatchCommandOutput extends PutRecordBatchOutput, __Met
  *          achieve higher throughput per producer than when writing single records. To write single
  *          data records into a delivery stream, use <a>PutRecord</a>. Applications using
  *          these operations are referred to as producers.</p>
+ *          <p>Kinesis Data Firehose accumulates and publishes a particular metric for a customer account in one minute intervals. It is possible that the bursts of incoming bytes/records ingested to a delivery stream last only for a few seconds. Due to this, the actual spikes in the traffic might not be fully visible in the customer's 1 minute CloudWatch metrics.</p>
  *          <p>For information about service quota, see <a href="https://docs.aws.amazon.com/firehose/latest/dev/limits.html">Amazon Kinesis Data Firehose
  *          Quota</a>.</p>
  *          <p>Each <a>PutRecordBatch</a> request supports up to 500 records. Each record
@@ -77,8 +78,11 @@ export interface PutRecordBatchCommandOutput extends PutRecordBatchOutput, __Met
  *          duplicate records and also reduces the total bytes sent (and corresponding charges). We
  *          recommend that you handle any duplicates at the destination.</p>
  *          <p>If <a>PutRecordBatch</a> throws <code>ServiceUnavailableException</code>,
- *          back off and retry. If the exception persists, it is possible that the throughput limits
- *          have been exceeded for the delivery stream.</p>
+ *          the API is automatically reinvoked (retried) 3 times. If the exception persists, it is
+ *          possible that the throughput limits have been exceeded for the delivery stream.</p>
+ *          <p>Re-invoking the Put API operations (for example, PutRecord and PutRecordBatch) can
+ *          result in data duplicates. For larger data assets, allow for a longer time out before
+ *          retrying Put API operations.</p>
  *          <p>Data records sent to Kinesis Data Firehose are stored for 24 hours from the time they
  *          are added to a delivery stream as it attempts to send the records to the destination. If
  *          the destination is unreachable for more than 24 hours, the data is no longer

diff --git a/clients/client-firehose/src/commands/PutRecordCommand.ts b/clients/client-firehose/src/commands/PutRecordCommand.ts
@@ -46,6 +46,7 @@ export interface PutRecordCommandOutput extends PutRecordOutput, __MetadataBeare
  *          operations for each delivery stream. For more information about limits and how to request
  *          an increase, see <a href="https://docs.aws.amazon.com/firehose/latest/dev/limits.html">Amazon
  *             Kinesis Data Firehose Limits</a>. </p>
+ *          <p>Kinesis Data Firehose accumulates and publishes a particular metric for a customer account in one minute intervals. It is possible that the bursts of incoming bytes/records ingested to a delivery stream last only for a few seconds. Due to this, the actual spikes in the traffic might not be fully visible in the customer's 1 minute CloudWatch metrics.</p>
  *          <p>You must specify the name of the delivery stream and the data record when using <a>PutRecord</a>. The data record consists of a data blob that can be up to 1,000
  *          KiB in size, and any kind of data. For example, it can be a segment from a log file,
  *          geographic location data, website clickstream data, and so on.</p>
@@ -58,8 +59,12 @@ export interface PutRecordCommandOutput extends PutRecordOutput, __MetadataBeare
  *          unique string assigned to each record. Producer applications can use this ID for purposes
  *          such as auditability and investigation.</p>
  *          <p>If the <code>PutRecord</code> operation throws a
- *             <code>ServiceUnavailableException</code>, back off and retry. If the exception persists,
- *          it is possible that the throughput limits have been exceeded for the delivery stream. </p>
+ *             <code>ServiceUnavailableException</code>, the API is automatically reinvoked (retried) 3
+ *          times. If the exception persists, it is possible that the throughput limits have been
+ *          exceeded for the delivery stream. </p>
+ *          <p>Re-invoking the Put API operations (for example, PutRecord and PutRecordBatch) can
+ *          result in data duplicates. For larger data assets, allow for a longer time out before
+ *          retrying Put API operations.</p>
  *          <p>Data records sent to Kinesis Data Firehose are stored for 24 hours from the time they
  *          are added to a delivery stream as it tries to send the records to the destination. If the
  *          destination is unreachable for more than 24 hours, the data is no longer

diff --git a/clients/client-firehose/src/commands/StartDeliveryStreamEncryptionCommand.ts b/clients/client-firehose/src/commands/StartDeliveryStreamEncryptionCommand.ts
@@ -62,6 +62,7 @@ export interface StartDeliveryStreamEncryptionCommandOutput
  *          CMK is of type <code>CUSTOMER_MANAGED_CMK</code>, Kinesis Data Firehose creates a grant
  *          that enables it to use the new CMK to encrypt and decrypt data and to manage the
  *          grant.</p>
+ *          <p>For the KMS grant creation to be successful, Kinesis Data Firehose APIs <code>StartDeliveryStreamEncryption</code> and <code>CreateDeliveryStream</code> should not be called with session credentials that are more than 6 hours old.</p>
  *          <p>If a delivery stream already has encryption enabled and then you invoke this operation
  *          to change the ARN of the CMK or both its type and ARN and you get
  *             <code>ENABLING_FAILED</code>, this only means that the attempt to change the CMK failed.