feat(client-glue): AWS Glue is introducing two new optimizers for Apa…

…che Iceberg tables: snapshot retention and orphan file deletion. Customers can enable these optimizers and customize their configurations to perform daily maintenance tasks on their Iceberg tables based on their specific requirements.
aws · Sep 12, 2024 · 0dac89e · 0dac89e
1 parent ba923e6
commit 0dac89e
Show file tree

Hide file tree

Showing 22 changed files with 1,752 additions and 869 deletions.
diff --git a/clients/client-glue/src/commands/BatchGetTableOptimizerCommand.ts b/clients/client-glue/src/commands/BatchGetTableOptimizerCommand.ts
@@ -41,7 +41,7 @@ export interface BatchGetTableOptimizerCommandOutput extends BatchGetTableOptimi
  *       catalogId: "STRING_VALUE",
  *       databaseName: "STRING_VALUE",
  *       tableName: "STRING_VALUE",
- *       type: "compaction",
+ *       type: "compaction" || "retention" || "orphan_file_deletion",
  *     },
  *   ],
  * };
@@ -54,10 +54,23 @@ export interface BatchGetTableOptimizerCommandOutput extends BatchGetTableOptimi
  * //       databaseName: "STRING_VALUE",
  * //       tableName: "STRING_VALUE",
  * //       tableOptimizer: { // TableOptimizer
- * //         type: "compaction",
+ * //         type: "compaction" || "retention" || "orphan_file_deletion",
  * //         configuration: { // TableOptimizerConfiguration
  * //           roleArn: "STRING_VALUE",
  * //           enabled: true || false,
+ * //           retentionConfiguration: { // RetentionConfiguration
+ * //             icebergConfiguration: { // IcebergRetentionConfiguration
+ * //               snapshotRetentionPeriodInDays: Number("int"),
+ * //               numberOfSnapshotsToRetain: Number("int"),
+ * //               cleanExpiredFiles: true || false,
+ * //             },
+ * //           },
+ * //           orphanFileDeletionConfiguration: { // OrphanFileDeletionConfiguration
+ * //             icebergConfiguration: { // IcebergOrphanFileDeletionConfiguration
+ * //               orphanFileRetentionPeriodInDays: Number("int"),
+ * //               location: "STRING_VALUE",
+ * //             },
+ * //           },
  * //         },
  * //         lastRun: { // TableOptimizerRun
  * //           eventType: "starting" || "completed" || "failed" || "in_progress",
@@ -70,6 +83,30 @@ export interface BatchGetTableOptimizerCommandOutput extends BatchGetTableOptimi
  * //             JobDurationInHour: "STRING_VALUE",
  * //           },
  * //           error: "STRING_VALUE",
+ * //           compactionMetrics: { // CompactionMetrics
+ * //             IcebergMetrics: { // IcebergCompactionMetrics
+ * //               NumberOfBytesCompacted: Number("long"),
+ * //               NumberOfFilesCompacted: Number("long"),
+ * //               NumberOfDpus: Number("int"),
+ * //               JobDurationInHour: Number("double"),
+ * //             },
+ * //           },
+ * //           retentionMetrics: { // RetentionMetrics
+ * //             IcebergMetrics: { // IcebergRetentionMetrics
+ * //               NumberOfDataFilesDeleted: Number("long"),
+ * //               NumberOfManifestFilesDeleted: Number("long"),
+ * //               NumberOfManifestListsDeleted: Number("long"),
+ * //               NumberOfDpus: Number("int"),
+ * //               JobDurationInHour: Number("double"),
+ * //             },
+ * //           },
+ * //           orphanFileDeletionMetrics: { // OrphanFileDeletionMetrics
+ * //             IcebergMetrics: { // IcebergOrphanFileDeletionMetrics
+ * //               NumberOfOrphanFilesDeleted: Number("long"),
+ * //               NumberOfDpus: Number("int"),
+ * //               JobDurationInHour: Number("double"),
+ * //             },
+ * //           },
  * //         },
  * //       },
  * //     },
@@ -83,7 +120,7 @@ export interface BatchGetTableOptimizerCommandOutput extends BatchGetTableOptimi
  * //       catalogId: "STRING_VALUE",
  * //       databaseName: "STRING_VALUE",
  * //       tableName: "STRING_VALUE",
- * //       type: "compaction",
+ * //       type: "compaction" || "retention" || "orphan_file_deletion",
  * //     },
  * //   ],
  * // };
@@ -96,9 +133,21 @@ export interface BatchGetTableOptimizerCommandOutput extends BatchGetTableOptimi
  * @see {@link BatchGetTableOptimizerCommandOutput} for command's `response` shape.
  * @see {@link GlueClientResolvedConfig | config} for GlueClient's `config` shape.
  *
+ * @throws {@link AccessDeniedException} (client fault)
+ *  <p>Access to a resource was denied.</p>
+ *
+ * @throws {@link EntityNotFoundException} (client fault)
+ *  <p>A specified entity does not exist</p>
+ *
  * @throws {@link InternalServiceException} (server fault)
  *  <p>An internal service error occurred.</p>
  *
+ * @throws {@link InvalidInputException} (client fault)
+ *  <p>The input provided was not valid.</p>
+ *
+ * @throws {@link ThrottlingException} (client fault)
+ *  <p>The throttling threshhold was exceeded.</p>
+ *
  * @throws {@link GlueServiceException}
  * <p>Base exception class for all service exceptions from Glue service.</p>
  *

diff --git a/clients/client-glue/src/commands/CancelDataQualityRulesetEvaluationRunCommand.ts b/clients/client-glue/src/commands/CancelDataQualityRulesetEvaluationRunCommand.ts
@@ -9,7 +9,7 @@ import { GlueClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from
 import {
   CancelDataQualityRulesetEvaluationRunRequest,
   CancelDataQualityRulesetEvaluationRunResponse,
-} from "../models/models_0";
+} from "../models/models_1";
 import {
   de_CancelDataQualityRulesetEvaluationRunCommand,
   se_CancelDataQualityRulesetEvaluationRunCommand,

diff --git a/clients/client-glue/src/commands/CancelMLTaskRunCommand.ts b/clients/client-glue/src/commands/CancelMLTaskRunCommand.ts
@@ -6,7 +6,7 @@ import { MetadataBearer as __MetadataBearer } from "@smithy/types";
 
 import { commonParams } from "../endpoint/EndpointParameters";
 import { GlueClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from "../GlueClient";
-import { CancelMLTaskRunRequest, CancelMLTaskRunResponse } from "../models/models_0";
+import { CancelMLTaskRunRequest, CancelMLTaskRunResponse } from "../models/models_1";
 import { de_CancelMLTaskRunCommand, se_CancelMLTaskRunCommand } from "../protocols/Aws_json1_1";
 
 /**

diff --git a/clients/client-glue/src/commands/CancelStatementCommand.ts b/clients/client-glue/src/commands/CancelStatementCommand.ts
@@ -6,7 +6,7 @@ import { MetadataBearer as __MetadataBearer } from "@smithy/types";
 
 import { commonParams } from "../endpoint/EndpointParameters";
 import { GlueClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from "../GlueClient";
-import { CancelStatementRequest, CancelStatementResponse } from "../models/models_0";
+import { CancelStatementRequest, CancelStatementResponse } from "../models/models_1";
 import { de_CancelStatementCommand, se_CancelStatementCommand } from "../protocols/Aws_json1_1";
 
 /**

diff --git a/clients/client-glue/src/commands/CheckSchemaVersionValidityCommand.ts b/clients/client-glue/src/commands/CheckSchemaVersionValidityCommand.ts
@@ -6,7 +6,7 @@ import { MetadataBearer as __MetadataBearer } from "@smithy/types";
 
 import { commonParams } from "../endpoint/EndpointParameters";
 import { GlueClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from "../GlueClient";
-import { CheckSchemaVersionValidityInput, CheckSchemaVersionValidityResponse } from "../models/models_0";
+import { CheckSchemaVersionValidityInput, CheckSchemaVersionValidityResponse } from "../models/models_1";
 import { de_CheckSchemaVersionValidityCommand, se_CheckSchemaVersionValidityCommand } from "../protocols/Aws_json1_1";
 
 /**

diff --git a/clients/client-glue/src/commands/CreateTableOptimizerCommand.ts b/clients/client-glue/src/commands/CreateTableOptimizerCommand.ts
@@ -39,10 +39,23 @@ export interface CreateTableOptimizerCommandOutput extends CreateTableOptimizerR
  *   CatalogId: "STRING_VALUE", // required
  *   DatabaseName: "STRING_VALUE", // required
  *   TableName: "STRING_VALUE", // required
- *   Type: "compaction", // required
+ *   Type: "compaction" || "retention" || "orphan_file_deletion", // required
  *   TableOptimizerConfiguration: { // TableOptimizerConfiguration
  *     roleArn: "STRING_VALUE",
  *     enabled: true || false,
+ *     retentionConfiguration: { // RetentionConfiguration
+ *       icebergConfiguration: { // IcebergRetentionConfiguration
+ *         snapshotRetentionPeriodInDays: Number("int"),
+ *         numberOfSnapshotsToRetain: Number("int"),
+ *         cleanExpiredFiles: true || false,
+ *       },
+ *     },
+ *     orphanFileDeletionConfiguration: { // OrphanFileDeletionConfiguration
+ *       icebergConfiguration: { // IcebergOrphanFileDeletionConfiguration
+ *         orphanFileRetentionPeriodInDays: Number("int"),
+ *         location: "STRING_VALUE",
+ *       },
+ *     },
  *   },
  * };
  * const command = new CreateTableOptimizerCommand(input);
@@ -72,6 +85,12 @@ export interface CreateTableOptimizerCommandOutput extends CreateTableOptimizerR
  * @throws {@link InvalidInputException} (client fault)
  *  <p>The input provided was not valid.</p>
  *
+ * @throws {@link ThrottlingException} (client fault)
+ *  <p>The throttling threshhold was exceeded.</p>
+ *
+ * @throws {@link ValidationException} (client fault)
+ *  <p>A value could not be validated.</p>
+ *
  * @throws {@link GlueServiceException}
  * <p>Base exception class for all service exceptions from Glue service.</p>
  *

diff --git a/clients/client-glue/src/commands/DeleteTableOptimizerCommand.ts b/clients/client-glue/src/commands/DeleteTableOptimizerCommand.ts
@@ -39,7 +39,7 @@ export interface DeleteTableOptimizerCommandOutput extends DeleteTableOptimizerR
  *   CatalogId: "STRING_VALUE", // required
  *   DatabaseName: "STRING_VALUE", // required
  *   TableName: "STRING_VALUE", // required
- *   Type: "compaction", // required
+ *   Type: "compaction" || "retention" || "orphan_file_deletion", // required
  * };
  * const command = new DeleteTableOptimizerCommand(input);
  * const response = await client.send(command);
@@ -65,6 +65,9 @@ export interface DeleteTableOptimizerCommandOutput extends DeleteTableOptimizerR
  * @throws {@link InvalidInputException} (client fault)
  *  <p>The input provided was not valid.</p>
  *
+ * @throws {@link ThrottlingException} (client fault)
+ *  <p>The throttling threshhold was exceeded.</p>
+ *
  * @throws {@link GlueServiceException}
  * <p>Base exception class for all service exceptions from Glue service.</p>
  *

diff --git a/clients/client-glue/src/commands/GetMLTransformCommand.ts b/clients/client-glue/src/commands/GetMLTransformCommand.ts
@@ -6,7 +6,7 @@ import { MetadataBearer as __MetadataBearer } from "@smithy/types";
 
 import { commonParams } from "../endpoint/EndpointParameters";
 import { GlueClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from "../GlueClient";
-import { GetMLTransformRequest, GetMLTransformResponse } from "../models/models_1";
+import { GetMLTransformRequest, GetMLTransformResponse } from "../models/models_2";
 import { de_GetMLTransformCommand, se_GetMLTransformCommand } from "../protocols/Aws_json1_1";
 
 /**

diff --git a/clients/client-glue/src/commands/GetTableOptimizerCommand.ts b/clients/client-glue/src/commands/GetTableOptimizerCommand.ts
@@ -39,7 +39,7 @@ export interface GetTableOptimizerCommandOutput extends GetTableOptimizerRespons
  *   CatalogId: "STRING_VALUE", // required
  *   DatabaseName: "STRING_VALUE", // required
  *   TableName: "STRING_VALUE", // required
- *   Type: "compaction", // required
+ *   Type: "compaction" || "retention" || "orphan_file_deletion", // required
  * };
  * const command = new GetTableOptimizerCommand(input);
  * const response = await client.send(command);
@@ -48,10 +48,23 @@ export interface GetTableOptimizerCommandOutput extends GetTableOptimizerRespons
  * //   DatabaseName: "STRING_VALUE",
  * //   TableName: "STRING_VALUE",
  * //   TableOptimizer: { // TableOptimizer
- * //     type: "compaction",
+ * //     type: "compaction" || "retention" || "orphan_file_deletion",
  * //     configuration: { // TableOptimizerConfiguration
  * //       roleArn: "STRING_VALUE",
  * //       enabled: true || false,
+ * //       retentionConfiguration: { // RetentionConfiguration
+ * //         icebergConfiguration: { // IcebergRetentionConfiguration
+ * //           snapshotRetentionPeriodInDays: Number("int"),
+ * //           numberOfSnapshotsToRetain: Number("int"),
+ * //           cleanExpiredFiles: true || false,
+ * //         },
+ * //       },
+ * //       orphanFileDeletionConfiguration: { // OrphanFileDeletionConfiguration
+ * //         icebergConfiguration: { // IcebergOrphanFileDeletionConfiguration
+ * //           orphanFileRetentionPeriodInDays: Number("int"),
+ * //           location: "STRING_VALUE",
+ * //         },
+ * //       },
  * //     },
  * //     lastRun: { // TableOptimizerRun
  * //       eventType: "starting" || "completed" || "failed" || "in_progress",
@@ -64,6 +77,30 @@ export interface GetTableOptimizerCommandOutput extends GetTableOptimizerRespons
  * //         JobDurationInHour: "STRING_VALUE",
  * //       },
  * //       error: "STRING_VALUE",
+ * //       compactionMetrics: { // CompactionMetrics
+ * //         IcebergMetrics: { // IcebergCompactionMetrics
+ * //           NumberOfBytesCompacted: Number("long"),
+ * //           NumberOfFilesCompacted: Number("long"),
+ * //           NumberOfDpus: Number("int"),
+ * //           JobDurationInHour: Number("double"),
+ * //         },
+ * //       },
+ * //       retentionMetrics: { // RetentionMetrics
+ * //         IcebergMetrics: { // IcebergRetentionMetrics
+ * //           NumberOfDataFilesDeleted: Number("long"),
+ * //           NumberOfManifestFilesDeleted: Number("long"),
+ * //           NumberOfManifestListsDeleted: Number("long"),
+ * //           NumberOfDpus: Number("int"),
+ * //           JobDurationInHour: Number("double"),
+ * //         },
+ * //       },
+ * //       orphanFileDeletionMetrics: { // OrphanFileDeletionMetrics
+ * //         IcebergMetrics: { // IcebergOrphanFileDeletionMetrics
+ * //           NumberOfOrphanFilesDeleted: Number("long"),
+ * //           NumberOfDpus: Number("int"),
+ * //           JobDurationInHour: Number("double"),
+ * //         },
+ * //       },
  * //     },
  * //   },
  * // };
@@ -88,6 +125,9 @@ export interface GetTableOptimizerCommandOutput extends GetTableOptimizerRespons
  * @throws {@link InvalidInputException} (client fault)
  *  <p>The input provided was not valid.</p>
  *
+ * @throws {@link ThrottlingException} (client fault)
+ *  <p>The throttling threshhold was exceeded.</p>
+ *
  * @throws {@link GlueServiceException}
  * <p>Base exception class for all service exceptions from Glue service.</p>
  *

diff --git a/clients/client-glue/src/commands/ListTableOptimizerRunsCommand.ts b/clients/client-glue/src/commands/ListTableOptimizerRunsCommand.ts
@@ -39,7 +39,7 @@ export interface ListTableOptimizerRunsCommandOutput extends ListTableOptimizerR
  *   CatalogId: "STRING_VALUE", // required
  *   DatabaseName: "STRING_VALUE", // required
  *   TableName: "STRING_VALUE", // required
- *   Type: "compaction", // required
+ *   Type: "compaction" || "retention" || "orphan_file_deletion", // required
  *   MaxResults: Number("int"),
  *   NextToken: "STRING_VALUE",
  * };
@@ -62,6 +62,30 @@ export interface ListTableOptimizerRunsCommandOutput extends ListTableOptimizerR
  * //         JobDurationInHour: "STRING_VALUE",
  * //       },
  * //       error: "STRING_VALUE",
+ * //       compactionMetrics: { // CompactionMetrics
+ * //         IcebergMetrics: { // IcebergCompactionMetrics
+ * //           NumberOfBytesCompacted: Number("long"),
+ * //           NumberOfFilesCompacted: Number("long"),
+ * //           NumberOfDpus: Number("int"),
+ * //           JobDurationInHour: Number("double"),
+ * //         },
+ * //       },
+ * //       retentionMetrics: { // RetentionMetrics
+ * //         IcebergMetrics: { // IcebergRetentionMetrics
+ * //           NumberOfDataFilesDeleted: Number("long"),
+ * //           NumberOfManifestFilesDeleted: Number("long"),
+ * //           NumberOfManifestListsDeleted: Number("long"),
+ * //           NumberOfDpus: Number("int"),
+ * //           JobDurationInHour: Number("double"),
+ * //         },
+ * //       },
+ * //       orphanFileDeletionMetrics: { // OrphanFileDeletionMetrics
+ * //         IcebergMetrics: { // IcebergOrphanFileDeletionMetrics
+ * //           NumberOfOrphanFilesDeleted: Number("long"),
+ * //           NumberOfDpus: Number("int"),
+ * //           JobDurationInHour: Number("double"),
+ * //         },
+ * //       },
  * //     },
  * //   ],
  * // };
@@ -86,6 +110,12 @@ export interface ListTableOptimizerRunsCommandOutput extends ListTableOptimizerR
  * @throws {@link InvalidInputException} (client fault)
  *  <p>The input provided was not valid.</p>
  *
+ * @throws {@link ThrottlingException} (client fault)
+ *  <p>The throttling threshhold was exceeded.</p>
+ *
+ * @throws {@link ValidationException} (client fault)
+ *  <p>A value could not be validated.</p>
+ *
  * @throws {@link GlueServiceException}
  * <p>Base exception class for all service exceptions from Glue service.</p>
  *

diff --git a/clients/client-glue/src/commands/UpdateTableCommand.ts b/clients/client-glue/src/commands/UpdateTableCommand.ts
@@ -6,7 +6,8 @@ import { MetadataBearer as __MetadataBearer } from "@smithy/types";
 
 import { commonParams } from "../endpoint/EndpointParameters";
 import { GlueClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from "../GlueClient";
-import { UpdateTableRequest, UpdateTableResponse } from "../models/models_2";
+import { UpdateTableRequest } from "../models/models_2";
+import { UpdateTableResponse } from "../models/models_3";
 import { de_UpdateTableCommand, se_UpdateTableCommand } from "../protocols/Aws_json1_1";
 
 /**

diff --git a/clients/client-glue/src/commands/UpdateTableOptimizerCommand.ts b/clients/client-glue/src/commands/UpdateTableOptimizerCommand.ts
@@ -6,7 +6,7 @@ import { MetadataBearer as __MetadataBearer } from "@smithy/types";
 
 import { commonParams } from "../endpoint/EndpointParameters";
 import { GlueClientResolvedConfig, ServiceInputTypes, ServiceOutputTypes } from "../GlueClient";
-import { UpdateTableOptimizerRequest, UpdateTableOptimizerResponse } from "../models/models_2";
+import { UpdateTableOptimizerRequest, UpdateTableOptimizerResponse } from "../models/models_3";
 import { de_UpdateTableOptimizerCommand, se_UpdateTableOptimizerCommand } from "../protocols/Aws_json1_1";
 
 /**
@@ -39,10 +39,23 @@ export interface UpdateTableOptimizerCommandOutput extends UpdateTableOptimizerR
  *   CatalogId: "STRING_VALUE", // required
  *   DatabaseName: "STRING_VALUE", // required
  *   TableName: "STRING_VALUE", // required
- *   Type: "compaction", // required
+ *   Type: "compaction" || "retention" || "orphan_file_deletion", // required
  *   TableOptimizerConfiguration: { // TableOptimizerConfiguration
  *     roleArn: "STRING_VALUE",
  *     enabled: true || false,
+ *     retentionConfiguration: { // RetentionConfiguration
+ *       icebergConfiguration: { // IcebergRetentionConfiguration
+ *         snapshotRetentionPeriodInDays: Number("int"),
+ *         numberOfSnapshotsToRetain: Number("int"),
+ *         cleanExpiredFiles: true || false,
+ *       },
+ *     },
+ *     orphanFileDeletionConfiguration: { // OrphanFileDeletionConfiguration
+ *       icebergConfiguration: { // IcebergOrphanFileDeletionConfiguration
+ *         orphanFileRetentionPeriodInDays: Number("int"),
+ *         location: "STRING_VALUE",
+ *       },
+ *     },
  *   },
  * };
  * const command = new UpdateTableOptimizerCommand(input);
@@ -60,6 +73,9 @@ export interface UpdateTableOptimizerCommandOutput extends UpdateTableOptimizerR
  * @throws {@link AccessDeniedException} (client fault)
  *  <p>Access to a resource was denied.</p>
  *
+ * @throws {@link ConcurrentModificationException} (client fault)
+ *  <p>Two processes are trying to modify a resource simultaneously.</p>
+ *
  * @throws {@link EntityNotFoundException} (client fault)
  *  <p>A specified entity does not exist</p>
  *
@@ -69,6 +85,12 @@ export interface UpdateTableOptimizerCommandOutput extends UpdateTableOptimizerR
  * @throws {@link InvalidInputException} (client fault)
  *  <p>The input provided was not valid.</p>
  *
+ * @throws {@link ThrottlingException} (client fault)
+ *  <p>The throttling threshhold was exceeded.</p>
+ *
+ * @throws {@link ValidationException} (client fault)
+ *  <p>A value could not be validated.</p>
+ *
  * @throws {@link GlueServiceException}
  * <p>Base exception class for all service exceptions from Glue service.</p>
  *