Skip to content

Commit

Permalink
improve test dependencies and code (#177)
Browse files Browse the repository at this point in the history
  • Loading branch information
ismailsimsek committed Feb 12, 2023
1 parent ababa3b commit 3fbd506
Show file tree
Hide file tree
Showing 8 changed files with 46 additions and 34 deletions.
15 changes: 2 additions & 13 deletions debezium-server-iceberg-sink/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -214,19 +214,6 @@
<version>4.2.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.debezium</groupId>
<artifactId>debezium-core</artifactId>
<version>${version.debezium}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.debezium</groupId>
<artifactId>debezium-server-core</artifactId>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
Expand All @@ -242,11 +229,13 @@
<dependency>
<groupId>io.debezium</groupId>
<artifactId>debezium-connector-postgres</artifactId>
<version>${version.debezium}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.debezium</groupId>
<artifactId>debezium-connector-mysql</artifactId>
<version>${version.debezium}</version>
<scope>test</scope>
</dependency>
<dependency>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,20 @@

package io.debezium.server.iceberg;

import io.debezium.server.TestConfigSource;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;

public class ConfigSource extends TestConfigSource {
import org.eclipse.microprofile.config.spi.ConfigSource;

public class TestConfigSource implements ConfigSource {

public static final String S3_REGION = "us-east-1";
public static final String S3_BUCKET = "test-bucket";
protected Map<String, String> config = new HashMap<>();


public ConfigSource() {
public TestConfigSource() {
config.put("quarkus.profile", "postgresql");
// common sink conf
config.put("debezium.sink.type", "iceberg");
Expand Down Expand Up @@ -69,9 +75,22 @@ public ConfigSource() {
}

@Override
public int getOrdinal() {
// Configuration property precedence is based on ordinal values and since we override the
// properties in TestConfigSource, we should give this a higher priority.
return super.getOrdinal() + 1;
public Map<String, String> getProperties() {
return config;
}

@Override
public String getName() {
return "test";
}

@Override
public Set<String> getPropertyNames() {
return config.keySet();
}

@Override
public String getValue(String propertyName) {
return config.get(propertyName);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,10 @@
package io.debezium.server.iceberg;

import io.debezium.serde.DebeziumSerdes;
import io.debezium.util.Testing;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;

import com.fasterxml.jackson.core.JsonProcessingException;
Expand All @@ -25,11 +26,14 @@
import static org.junit.jupiter.api.Assertions.*;

class TestIcebergUtil {
final String serdeWithSchema = Testing.Files.readResourceAsString("json/serde-with-schema.json");
final String unwrapWithSchema = Testing.Files.readResourceAsString("json/unwrap-with-schema.json");
final String unwrapWithGeomSchema = Testing.Files.readResourceAsString("json/serde-with-schema_geom.json");
final String unwrapWithArraySchema = Testing.Files.readResourceAsString("json/serde-with-array.json");
final String unwrapWithArraySchema2 = Testing.Files.readResourceAsString("json/serde-with-array2.json");
final String serdeWithSchema = Files.readString(Path.of("src/test/resources/json/serde-with-schema.json"));
final String unwrapWithSchema = Files.readString(Path.of("src/test/resources/json/unwrap-with-schema.json"));
final String unwrapWithGeomSchema = Files.readString(Path.of("src/test/resources/json/serde-with-schema_geom.json"));
final String unwrapWithArraySchema = Files.readString(Path.of("src/test/resources/json/serde-with-array.json"));
final String unwrapWithArraySchema2 = Files.readString(Path.of("src/test/resources/json/serde-with-array2.json"));

TestIcebergUtil() throws IOException {
}

@Test
public void testNestedJsonRecord() throws JsonProcessingException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
import org.eclipse.microprofile.config.ConfigProvider;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.junit.jupiter.api.BeforeAll;
import static io.debezium.server.iceberg.ConfigSource.S3_BUCKET;
import static io.debezium.server.iceberg.TestConfigSource.S3_BUCKET;

/**
* Integration test that uses spark to consumer data is consumed.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

package io.debezium.server.iceberg.testresources;

import io.debezium.server.iceberg.ConfigSource;
import io.debezium.server.iceberg.TestConfigSource;
import io.minio.ListObjectsArgs;
import io.minio.MakeBucketArgs;
import io.minio.MinioClient;
Expand Down Expand Up @@ -46,7 +46,7 @@ public class S3Minio implements QuarkusTestResourceLifecycleManager {
.withStartupTimeout(Duration.ofSeconds(30)))
.withEnv("MINIO_ACCESS_KEY", MINIO_ACCESS_KEY)
.withEnv("MINIO_SECRET_KEY", MINIO_SECRET_KEY)
.withEnv("MINIO_REGION_NAME", ConfigSource.S3_REGION)
.withEnv("MINIO_REGION_NAME", TestConfigSource.S3_REGION)
.withCommand("server " + DEFAULT_STORAGE_DIRECTORY)
.withExposedPorts(MINIO_DEFAULT_PORT);
public static MinioClient client;
Expand Down Expand Up @@ -120,8 +120,8 @@ public Map<String, String> start() {
try {
client.ignoreCertCheck();
client.makeBucket(MakeBucketArgs.builder()
.region(ConfigSource.S3_REGION)
.bucket(ConfigSource.S3_BUCKET)
.region(TestConfigSource.S3_REGION)
.bucket(TestConfigSource.S3_BUCKET)
.build());
} catch (Exception e) {
e.printStackTrace();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ public class SourceMysqlDB implements QuarkusTestResourceLifecycleManager {
public static final String MYSQL_PASSWORD = "mysqlpw";
public static final String MYSQL_DEBEZIUM_USER = "debezium";
public static final String MYSQL_DEBEZIUM_PASSWORD = "dbz";
public static final String MYSQL_IMAGE = "debezium/example-mysql:1.9.2.Final";
public static final String MYSQL_IMAGE = "debezium/example-mysql:2.1.2.Final";
public static final String MYSQL_HOST = "127.0.0.1";
public static final String MYSQL_DATABASE = "inventory";
public static final Integer MYSQL_PORT_DEFAULT = 3306;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ public class SourcePostgresqlDB implements QuarkusTestResourceLifecycleManager {
public static final String POSTGRES_USER = "postgres";
public static final String POSTGRES_PASSWORD = "postgres";
public static final String POSTGRES_DBNAME = "postgres";
public static final String POSTGRES_IMAGE = "debezium/example-postgres:1.9.2.Final";
public static final String POSTGRES_IMAGE = "debezium/example-postgres:2.1.2.Final";
public static final String POSTGRES_HOST = "localhost";
public static final Integer POSTGRES_PORT_DEFAULT = 5432;
private static final Logger LOGGER = LoggerFactory.getLogger(SourcePostgresqlDB.class);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,4 @@
# */
#

io.debezium.server.iceberg.ConfigSource
io.debezium.server.iceberg.TestConfigSource

0 comments on commit 3fbd506

Please sign in to comment.