Remove aws-java-sdk-bundle from release (#204)

memiiso · Jun 1, 2023 · 85abcd7 · 85abcd7
1 parent 6804857
commit 85abcd7
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 2 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -19,7 +19,6 @@ on:
 
 env:
   SPARK_LOCAL_IP: 127.0.0.1
-  AWS_REGION: us-east-1
 
 jobs:
   build:

diff --git a/debezium-server-iceberg-sink/pom.xml b/debezium-server-iceberg-sink/pom.xml
@@ -109,6 +109,18 @@
             <groupId>org.apache.hadoop</groupId>
             <artifactId>hadoop-aws</artifactId>
             <version>${version.hadoop}</version>
+            <exclusions>
+                <exclusion>
+                    <groupId>com.amazonaws</groupId>
+                    <artifactId>aws-java-sdk-bundle</artifactId>
+                </exclusion>
+            </exclusions>
+        </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-java-sdk-bundle</artifactId>
+            <version>1.11.1026</version>
+            <scope>test</scope>
         </dependency>
         <!-- Hadoop -->
         <dependency>

diff --git a/...er-iceberg-sink/src/test/java/io/debezium/server/iceberg/testresources/BaseSparkTest.java b/...er-iceberg-sink/src/test/java/io/debezium/server/iceberg/testresources/BaseSparkTest.java
@@ -9,6 +9,7 @@
 package io.debezium.server.iceberg.testresources;
 
 import io.debezium.server.iceberg.IcebergUtil;
+import io.debezium.server.iceberg.TestConfigSource;
 
 import java.util.HashMap;
 import java.util.Map;
@@ -51,7 +52,15 @@ static void setup() {
         .set("spark.sql.catalog.spark_catalog.type", "hadoop")
         .set("spark.sql.catalog.spark_catalog.warehouse", S3_BUCKET)
         .set("spark.sql.catalog.spark_catalog.default-namespaces", CATALOG_TABLE_NAMESPACE)
+        .set("spark.sql.catalog.spark_catalog.io-impl", "org.apache.iceberg.aws.s3.S3FileIO")
         .set("spark.sql.warehouse.dir", S3_BUCKET)
+        .set("spark.sql.catalog.spark_catalog.s3.endpoint", "http://localhost:" + S3Minio.getMappedPort().toString())
+        .set("spark.sql.catalog.spark_catalog.s3.path-style-access", "true")
+        .set("spark.sql.catalog.spark_catalog.s3.access-key-id", S3Minio.MINIO_ACCESS_KEY)
+        .set("spark.sql.catalog.spark_catalog.s3.secret-access-key", S3Minio.MINIO_SECRET_KEY)
+        .set("spark.sql.catalog.spark_catalog.client.region", TestConfigSource.S3_REGION)
+        .set("spark.sql.catalog.spark_catalog.io-impl", "org.apache.iceberg.aws.s3.S3FileIO")
+        .set("spark.sql.catalog.spark_catalog.warehouse", S3_BUCKET)
 //        // JdbcCatalog catalog, add additional catalog
 //        .set("spark.sql.defaultCatalog", ICEBERG_CATALOG_NAME)
 //        .set("spark.sql.catalog." + ICEBERG_CATALOG_NAME, "org.apache.iceberg.spark.SparkCatalog")

diff --git a/...m-server-iceberg-sink/src/test/java/io/debezium/server/iceberg/testresources/S3Minio.java b/...m-server-iceberg-sink/src/test/java/io/debezium/server/iceberg/testresources/S3Minio.java
@@ -135,7 +135,7 @@ public Map<String, String> start() {
     config.put("debezium.sink.iceberg.s3.path-style-access", "true");
     config.put("debezium.sink.iceberg.s3.access-key-id", S3Minio.MINIO_ACCESS_KEY);
     config.put("debezium.sink.iceberg.s3.secret-access-key", S3Minio.MINIO_SECRET_KEY);
-    config.put("debezium.sink.iceberg.s3.region", TestConfigSource.S3_REGION);
+    config.put("debezium.sink.iceberg.client.region", TestConfigSource.S3_REGION);
     config.put("debezium.sink.iceberg.io-impl", "org.apache.iceberg.aws.s3.S3FileIO");
     config.put("debezium.sink.iceberg.warehouse", S3_BUCKET);
     // FOR HADOOP CATALOG