-
Notifications
You must be signed in to change notification settings - Fork 35
/
S3Minio.java
152 lines (134 loc) · 5.66 KB
/
S3Minio.java
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
/*
*
* * Copyright memiiso Authors.
* *
* * Licensed under the Apache Software License version 2.0, available at http://www.apache.org/licenses/LICENSE-2.0
*
*/
package io.debezium.server.iceberg.testresources;
import io.debezium.server.iceberg.TestConfigSource;
import io.minio.ListObjectsArgs;
import io.minio.MakeBucketArgs;
import io.minio.MinioClient;
import io.minio.Result;
import io.minio.messages.Bucket;
import io.minio.messages.Item;
import io.quarkus.test.common.QuarkusTestResourceLifecycleManager;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.wait.strategy.HttpWaitStrategy;
import org.testcontainers.utility.DockerImageName;
import static io.debezium.server.iceberg.TestConfigSource.S3_BUCKET;
import static io.debezium.server.iceberg.TestConfigSource.S3_BUCKET_NAME;
public class S3Minio implements QuarkusTestResourceLifecycleManager {
public static final String MINIO_ACCESS_KEY = "admin";
public static final String MINIO_SECRET_KEY = "12345678";
protected static final Logger LOGGER = LoggerFactory.getLogger(S3Minio.class);
static final int MINIO_DEFAULT_PORT = 9000;
static final String DEFAULT_IMAGE = "minio/minio:latest";
static final String DEFAULT_STORAGE_DIRECTORY = "/data";
static final String HEALTH_ENDPOINT = "/minio/health/ready";
static private final GenericContainer<?> container = new GenericContainer<>(DockerImageName.parse(DEFAULT_IMAGE))
.withExposedPorts(MINIO_DEFAULT_PORT)
.waitingFor(new HttpWaitStrategy()
.forPath(HEALTH_ENDPOINT)
.forPort(MINIO_DEFAULT_PORT)
.withStartupTimeout(Duration.ofSeconds(30)))
.withEnv("MINIO_ACCESS_KEY", MINIO_ACCESS_KEY)
.withEnv("MINIO_SECRET_KEY", MINIO_SECRET_KEY)
.withEnv("MINIO_REGION_NAME", TestConfigSource.S3_REGION)
.withCommand("server " + DEFAULT_STORAGE_DIRECTORY)
.withExposedPorts(MINIO_DEFAULT_PORT);
public static MinioClient client;
public static List<Item> getObjectList(String bucketName) {
List<Item> objects = new ArrayList<>();
try {
Iterable<Result<Item>> results = client.listObjects(ListObjectsArgs.builder().bucket(bucketName).recursive(true).build());
for (Result<Item> result : results) {
Item item = result.get();
objects.add(item);
}
} catch (Exception e) {
e.printStackTrace();
}
return objects;
}
public static void listFiles() {
LOGGER.info("-----------------------------------------------------------------");
try {
List<Bucket> bucketList = client.listBuckets();
for (Bucket bucket : bucketList) {
System.out.printf("Bucket:%s ROOT\n", bucket.name());
Iterable<Result<Item>> results = client.listObjects(ListObjectsArgs.builder().bucket(bucket.name()).recursive(true).build());
for (Result<Item> result : results) {
Item item = result.get();
System.out.printf("Bucket:%s Item:%s Size:%s\n", bucket.name(), item.objectName(), item.size());
}
}
} catch (Exception e) {
LOGGER.info("Failed listing bucket");
}
LOGGER.info("-----------------------------------------------------------------");
}
public static List<Item> getIcebergDataFiles(String bucketName) {
List<Item> objects = new ArrayList<>();
try {
List<Item> results = getObjectList(bucketName);
for (Item result : results) {
if (result.objectName().contains("/data/") && result.objectName().endsWith("parquet")) {
objects.add(result);
}
}
} catch (Exception e) {
LOGGER.info("Failed listing bucket");
}
return objects;
}
public static Integer getMappedPort() {
return container.getMappedPort(MINIO_DEFAULT_PORT);
}
@Override
public void stop() {
container.stop();
}
@Override
public Map<String, String> start() {
container.start();
client = MinioClient.builder()
.endpoint("http://" + container.getHost() + ":" + container.getMappedPort(MINIO_DEFAULT_PORT))
.credentials(MINIO_ACCESS_KEY, MINIO_SECRET_KEY)
.build();
try {
client.ignoreCertCheck();
client.makeBucket(MakeBucketArgs.builder()
.region(TestConfigSource.S3_REGION)
.bucket(S3_BUCKET_NAME)
.build());
} catch (Exception e) {
e.printStackTrace();
}
LOGGER.info("Minio Started!");
Map<String, String> config = new ConcurrentHashMap<>();
// FOR JDBC CATALOG
config.put("debezium.sink.iceberg.s3.endpoint", "http://localhost:" + S3Minio.getMappedPort().toString());
config.put("debezium.sink.iceberg.s3.path-style-access", "true");
config.put("debezium.sink.iceberg.s3.access-key-id", S3Minio.MINIO_ACCESS_KEY);
config.put("debezium.sink.iceberg.s3.secret-access-key", S3Minio.MINIO_SECRET_KEY);
config.put("debezium.sink.iceberg.client.region", TestConfigSource.S3_REGION);
config.put("debezium.sink.iceberg.io-impl", "org.apache.iceberg.aws.s3.S3FileIO");
config.put("debezium.sink.iceberg.warehouse", S3_BUCKET);
// FOR HADOOP CATALOG
config.put("debezium.sink.iceberg.fs.s3a.endpoint", "http://localhost:" + S3Minio.getMappedPort().toString());
config.put("debezium.sink.iceberg.fs.s3a.access.key", S3Minio.MINIO_ACCESS_KEY);
config.put("debezium.sink.iceberg.fs.s3a.secret.key", S3Minio.MINIO_SECRET_KEY);
config.put("debezium.sink.iceberg.fs.s3a.path.style.access", "true");
config.put("debezium.sink.iceberg.fs.defaultFS", "s3a://" + S3_BUCKET_NAME);
return config;
}
}