Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve test dependencies and code #177

Merged
merged 1 commit into from
Feb 12, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 2 additions & 13 deletions debezium-server-iceberg-sink/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -214,19 +214,6 @@
<version>4.2.0</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.debezium</groupId>
<artifactId>debezium-core</artifactId>
<version>${version.debezium}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.debezium</groupId>
<artifactId>debezium-server-core</artifactId>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>testcontainers</artifactId>
Expand All @@ -242,11 +229,13 @@
<dependency>
<groupId>io.debezium</groupId>
<artifactId>debezium-connector-postgres</artifactId>
<version>${version.debezium}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>io.debezium</groupId>
<artifactId>debezium-connector-mysql</artifactId>
<version>${version.debezium}</version>
<scope>test</scope>
</dependency>
<dependency>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,20 @@

package io.debezium.server.iceberg;

import io.debezium.server.TestConfigSource;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;

public class ConfigSource extends TestConfigSource {
import org.eclipse.microprofile.config.spi.ConfigSource;

public class TestConfigSource implements ConfigSource {

public static final String S3_REGION = "us-east-1";
public static final String S3_BUCKET = "test-bucket";
protected Map<String, String> config = new HashMap<>();


public ConfigSource() {
public TestConfigSource() {
config.put("quarkus.profile", "postgresql");
// common sink conf
config.put("debezium.sink.type", "iceberg");
Expand Down Expand Up @@ -69,9 +75,22 @@ public ConfigSource() {
}

@Override
public int getOrdinal() {
// Configuration property precedence is based on ordinal values and since we override the
// properties in TestConfigSource, we should give this a higher priority.
return super.getOrdinal() + 1;
public Map<String, String> getProperties() {
return config;
}

@Override
public String getName() {
return "test";
}

@Override
public Set<String> getPropertyNames() {
return config.keySet();
}

@Override
public String getValue(String propertyName) {
return config.get(propertyName);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,10 @@
package io.debezium.server.iceberg;

import io.debezium.serde.DebeziumSerdes;
import io.debezium.util.Testing;

import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Collections;

import com.fasterxml.jackson.core.JsonProcessingException;
Expand All @@ -25,11 +26,14 @@
import static org.junit.jupiter.api.Assertions.*;

class TestIcebergUtil {
final String serdeWithSchema = Testing.Files.readResourceAsString("json/serde-with-schema.json");
final String unwrapWithSchema = Testing.Files.readResourceAsString("json/unwrap-with-schema.json");
final String unwrapWithGeomSchema = Testing.Files.readResourceAsString("json/serde-with-schema_geom.json");
final String unwrapWithArraySchema = Testing.Files.readResourceAsString("json/serde-with-array.json");
final String unwrapWithArraySchema2 = Testing.Files.readResourceAsString("json/serde-with-array2.json");
final String serdeWithSchema = Files.readString(Path.of("src/test/resources/json/serde-with-schema.json"));
final String unwrapWithSchema = Files.readString(Path.of("src/test/resources/json/unwrap-with-schema.json"));
final String unwrapWithGeomSchema = Files.readString(Path.of("src/test/resources/json/serde-with-schema_geom.json"));
final String unwrapWithArraySchema = Files.readString(Path.of("src/test/resources/json/serde-with-array.json"));
final String unwrapWithArraySchema2 = Files.readString(Path.of("src/test/resources/json/serde-with-array2.json"));

TestIcebergUtil() throws IOException {
}

@Test
public void testNestedJsonRecord() throws JsonProcessingException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
import org.eclipse.microprofile.config.ConfigProvider;
import org.eclipse.microprofile.config.inject.ConfigProperty;
import org.junit.jupiter.api.BeforeAll;
import static io.debezium.server.iceberg.ConfigSource.S3_BUCKET;
import static io.debezium.server.iceberg.TestConfigSource.S3_BUCKET;

/**
* Integration test that uses spark to consumer data is consumed.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

package io.debezium.server.iceberg.testresources;

import io.debezium.server.iceberg.ConfigSource;
import io.debezium.server.iceberg.TestConfigSource;
import io.minio.ListObjectsArgs;
import io.minio.MakeBucketArgs;
import io.minio.MinioClient;
Expand Down Expand Up @@ -46,7 +46,7 @@ public class S3Minio implements QuarkusTestResourceLifecycleManager {
.withStartupTimeout(Duration.ofSeconds(30)))
.withEnv("MINIO_ACCESS_KEY", MINIO_ACCESS_KEY)
.withEnv("MINIO_SECRET_KEY", MINIO_SECRET_KEY)
.withEnv("MINIO_REGION_NAME", ConfigSource.S3_REGION)
.withEnv("MINIO_REGION_NAME", TestConfigSource.S3_REGION)
.withCommand("server " + DEFAULT_STORAGE_DIRECTORY)
.withExposedPorts(MINIO_DEFAULT_PORT);
public static MinioClient client;
Expand Down Expand Up @@ -120,8 +120,8 @@ public Map<String, String> start() {
try {
client.ignoreCertCheck();
client.makeBucket(MakeBucketArgs.builder()
.region(ConfigSource.S3_REGION)
.bucket(ConfigSource.S3_BUCKET)
.region(TestConfigSource.S3_REGION)
.bucket(TestConfigSource.S3_BUCKET)
.build());
} catch (Exception e) {
e.printStackTrace();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ public class SourceMysqlDB implements QuarkusTestResourceLifecycleManager {
public static final String MYSQL_PASSWORD = "mysqlpw";
public static final String MYSQL_DEBEZIUM_USER = "debezium";
public static final String MYSQL_DEBEZIUM_PASSWORD = "dbz";
public static final String MYSQL_IMAGE = "debezium/example-mysql:1.9.2.Final";
public static final String MYSQL_IMAGE = "debezium/example-mysql:2.1.2.Final";
public static final String MYSQL_HOST = "127.0.0.1";
public static final String MYSQL_DATABASE = "inventory";
public static final Integer MYSQL_PORT_DEFAULT = 3306;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ public class SourcePostgresqlDB implements QuarkusTestResourceLifecycleManager {
public static final String POSTGRES_USER = "postgres";
public static final String POSTGRES_PASSWORD = "postgres";
public static final String POSTGRES_DBNAME = "postgres";
public static final String POSTGRES_IMAGE = "debezium/example-postgres:1.9.2.Final";
public static final String POSTGRES_IMAGE = "debezium/example-postgres:2.1.2.Final";
public static final String POSTGRES_HOST = "localhost";
public static final Integer POSTGRES_PORT_DEFAULT = 5432;
private static final Logger LOGGER = LoggerFactory.getLogger(SourcePostgresqlDB.class);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,4 +30,4 @@
# */
#

io.debezium.server.iceberg.ConfigSource
io.debezium.server.iceberg.TestConfigSource