update test containers

Signed-off-by: Heinz N. Gies <[email protected]>

tremor-connectors-aws/Cargo.toml

@@ -38,10 +38,11 @@ aws-credential-types = { version = "1", default-features = false }
 
 [dev-dependencies]
 rand = { version = "0.8", default-features = false }
-testcontainers = { version = "0.17", default-features = false }
+testcontainers = { version = "0.23", default-features = false }
 serial_test = { version = "3.0", default-features = false }
 bytes = { version = "1.6", default-features = true }
 tremor-connectors-test-helpers = { path = "../tremor-connectors-test-helpers", version = "0.13.0-rc.29" }
+testcontainers-modules = { version = "0.11.1", features = ["localstack"] }
 
 [features]
 integration-harness = []

tremor-connectors-aws/src/lib.rs

@@ -13,7 +13,6 @@
 // limitations under the License.
 
 //! Tremor AWS connectors
-
 #![deny(warnings)]
 #![deny(missing_docs)]
 #![deny(

tremor-connectors-aws/src/s3.rs

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-#![allow(clippy::doc_markdown)]
+#![allow(clippy::doc_markdown, rustdoc::invalid_codeblock_attributes)]
 //! The Amazon Web Services `s3` provides integration with the AWS Simple Storage Service or
 //! drop-in compatible replacements such as [MinIO](https://www.min.io).
 //!

tremor-connectors-aws/tests/aws.rs

@@ -27,8 +27,8 @@ use std::{
     time::{Duration, Instant},
 };
 use testcontainers::runners::AsyncRunner;
-use testcontainers::ContainerAsync;
-use testcontainers::{GenericImage, RunnableImage};
+use testcontainers::{core::IntoContainerPort, GenericImage};
+use testcontainers::{ContainerAsync, ImageExt};
 use tremor_connectors_test_helpers::free_port::find_free_tcp_port;
 
 const IMAGE: &str = "minio/minio";
@@ -76,23 +76,21 @@ async fn create_bucket(bucket: &str, http_port: u16) -> anyhow::Result<()> {
 }
 
 async fn spawn_docker() -> (ContainerAsync<GenericImage>, u16) {
+    let http_port = find_free_tcp_port().await.unwrap_or(10080);
+    let http_tls_port = find_free_tcp_port().await.unwrap_or(10443);
+
     let image = GenericImage::new(IMAGE, VERSION)
         .with_env_var("MINIO_ROOT_USER", MINIO_ROOT_USER)
         .with_env_var("MINIO_ROOT_PASSWORD", MINIO_ROOT_PASSWORD)
-        .with_env_var("MINIO_REGION", MINIO_REGION);
-    let http_port = find_free_tcp_port().await.unwrap_or(10080);
-    let http_tls_port = find_free_tcp_port().await.unwrap_or(10443);
-    let image = RunnableImage::from((
-        image,
-        vec![
+        .with_env_var("MINIO_REGION", MINIO_REGION)
+        .with_cmd([
             String::from("server"),
             String::from("/data"),
             String::from("--console-address"),
             String::from(":9001"),
-        ],
-    ))
-    .with_mapped_port((http_port, 9000_u16))
-    .with_mapped_port((http_tls_port, 9001_u16));
+        ])
+        .with_mapped_port(http_port, 9000_u16.tcp())
+        .with_mapped_port(http_tls_port, 9001_u16.tcp());
     let container = image.start().await;
 
     (container.unwrap(), http_port)

tremor-connectors-azure/Cargo.toml

@@ -1,7 +1,3 @@
-bin = []
-bench = []
-test = []
-
 [package]
 name = "tremor-connectors-azure"
 edition = "2021"
@@ -44,12 +40,3 @@ mockito = "1.4.0"
 
 [features]
 integration-harness = []
-
-[lib]
-path = "src/lib.rs"
-name = "tremor_connectors_azure"
-plugin = false
-proc-macro = false
-edition = "2021"
-required-features = []
-crate-type = ["rlib"]

tremor-connectors-gcp/Cargo.toml

@@ -59,7 +59,7 @@ simd-json-derive = { version = "0.13", default-features = true }
 simd-json = { version = "0.13", default-features = true }
 
 [dev-dependencies]
-testcontainers = { version = "0.18", default-features = true }
+testcontainers = { version = "0.23", default-features = true }
 hyper = { version = "0.14", default-features = true, features = [
     "server",
     "http1",
@@ -74,6 +74,10 @@ tremor-codec = { path = "../tremor-codec", version = "0.13.0-rc.29" }
 tempfile = { version = "3", default-features = true }
 env_logger = { version = "0.11", default-features = true }
 tremor-connectors-test-helpers = { path = "../tremor-connectors-test-helpers", version = "0.13.0-rc.29" }
+serial_test = "3.1.1"
+testcontainers-modules = { version = "0.11.1", features = [
+    "google_cloud_sdk_emulators",
+] }
 
 [features]
 integration-harness = []

tremor-connectors-gcp/tests/gcp/gpub.rs

@@ -20,11 +20,12 @@ use googapis::google::pubsub::v1::{
 use serial_test::serial;
 use std::collections::{HashMap, HashSet};
 use std::time::Duration;
-use testcontainers::{clients::Cli, RunnableImage};
+use testcontainers::runners::AsyncRunner;
+use testcontainers_modules::google_cloud_sdk_emulators::{CloudSdk, PUBSUB_PORT};
 use tokio::time::timeout;
 use tonic::transport::Channel;
-use tremor_common::ports::IN;
-use tremor_connectors::{harness::Harness, impls::gpubsub::producer::Builder};
+use tremor_connectors::harness::Harness;
+use tremor_connectors_gcp::gpubsub::producer::Builder;
 use tremor_system::{
     event::{Event, EventId},
     instance::State,
@@ -81,16 +82,10 @@ async fn no_hostname() -> anyhow::Result<()> {
 #[serial(gpubsub)]
 async fn simple_publish() -> anyhow::Result<()> {
     let _ = env_logger::try_init();
-    let runner = Cli::docker();
 
-    let (pubsub, pubsub_args) =
-        testcontainers::images::google_cloud_sdk_emulators::CloudSdk::pubsub();
-    let runnable_image = RunnableImage::from((pubsub, pubsub_args));
-    let container = runner.run(runnable_image);
-
-    let port = container
-        .get_host_port_ipv4(testcontainers::images::google_cloud_sdk_emulators::PUBSUB_PORT);
-    let endpoint = format!("http://localhost:{port}");
+    let pubsub = CloudSdk::pubsub().start().await?;
+    let port = pubsub.get_host_port_ipv4(PUBSUB_PORT).await?;
+    let endpoint: String = format!("http://localhost:{port}");
     let endpoint_clone = endpoint.clone();
 
     // note we need to keep this around until the end of the test so we can't consume it
@@ -188,19 +183,12 @@ async fn simple_publish() -> anyhow::Result<()> {
 #[serial(gpubsub)]
 async fn simple_publish_with_timeout() -> anyhow::Result<()> {
     let _ = env_logger::try_init();
-    let runner = Cli::docker();
-
-    let (pubsub, pubsub_args) =
-        testcontainers::images::google_cloud_sdk_emulators::CloudSdk::pubsub();
-    let runnable_image = RunnableImage::from((pubsub, pubsub_args));
-    let container = runner.run(runnable_image);
-
-    let port = container
-        .get_host_port_ipv4(testcontainers::images::google_cloud_sdk_emulators::PUBSUB_PORT);
+    let pubsub = CloudSdk::pubsub().start().await?;
+    let port = pubsub.get_host_port_ipv4(PUBSUB_PORT).await?;
+    let endpoint: String = format!("http://localhost:{port}");
 
     // note we need to keep this around until the end of the test so we can't consume it
     let token_file = crate::gouth_token().await?;
-    let endpoint = format!("http://localhost:{port}");
 
     let connector_yaml: Value = literal!({
         "codec": "binary",
@@ -250,7 +238,7 @@ async fn simple_publish_with_timeout() -> anyhow::Result<()> {
     harness.wait_for_connected().await?;
     harness.consume_initial_sink_contraflow().await?;
 
-    drop(container);
+    drop(pubsub);
 
     let event = Event {
         id: EventId::default(),

tremor-connectors-gcp/tests/gcp/gsub.rs

@@ -20,11 +20,11 @@ use googapis::google::pubsub::v1::{
 };
 use serial_test::serial;
 use std::collections::HashMap;
-use testcontainers::clients::Cli;
-use testcontainers::RunnableImage;
+use testcontainers::runners::AsyncRunner;
+use testcontainers_modules::google_cloud_sdk_emulators::{CloudSdk, PUBSUB_PORT};
 use tonic::transport::Channel;
 use tremor_connectors::harness::Harness;
-use tremor_connectors::impls::gpubsub::consumer::Builder;
+use tremor_connectors_gcp::gpubsub::consumer::Builder;
 use tremor_system::controlplane::CbAction;
 use tremor_value::{literal, Value};
 use value_trait::prelude::*;
@@ -104,14 +104,9 @@ async fn create_subscription(
 #[serial(gpubsub)]
 async fn simple_subscribe() -> anyhow::Result<()> {
     let _ = env_logger::try_init();
-    let runner = Cli::docker();
 
-    let (pubsub, pubsub_args) =
-        testcontainers::images::google_cloud_sdk_emulators::CloudSdk::pubsub();
-    let runnable_image = RunnableImage::from((pubsub, pubsub_args));
-    let container = runner.run(runnable_image);
-    let port = container
-        .get_host_port_ipv4(testcontainers::images::google_cloud_sdk_emulators::PUBSUB_PORT);
+    let pubsub = CloudSdk::pubsub().start().await?;
+    let port = pubsub.get_host_port_ipv4(PUBSUB_PORT).await?;
     let endpoint = format!("http://localhost:{port}");
     let topic = "projects/test/topics/test";
     let subscription = "projects/test/subscriptions/test-subscription-a";

tremor-connectors/Cargo.toml

@@ -157,7 +157,7 @@ socket2 = { version = "0.5", optional = true, default-features = false }
 serde_yaml = { version = "0.9", default-features = false }
 test-case = "3.3"
 proptest = "1.5"
-testcontainers = "0.16.7"
+testcontainers = "0.23"
 serial_test = "3.0"
 http-types = "2.0"
 bytes = "1.6"

tremor-connectors/tests/clickhouse/more_complex_test.rs

@@ -55,8 +55,10 @@ use chrono::DateTime;
 use clickhouse_chrono_tz::Tz;
 use clickhouse_rs::Pool;
 use log::error;
+use testcontainers::core::IntoContainerPort;
 use testcontainers::runners::AsyncRunner;
-use testcontainers::{GenericImage, RunnableImage};
+use testcontainers::GenericImage;
+use testcontainers::ImageExt;
 use tremor_common::ports::IN;
 use tremor_connectors::{harness::Harness, impls::clickhouse};
 use tremor_connectors_test_helpers::free_port;
@@ -86,15 +88,14 @@ async fn test() -> Result<()> {
     let _ = env_logger::try_init();
     // The following lines spin up a regular ClickHouse container and wait for
     // the database to be up and running.
+    let local = free_port::find_free_tcp_port().await?;
 
-    let image = GenericImage::new(utils::CONTAINER_NAME, utils::CONTAINER_VERSION);
+    let image = GenericImage::new(utils::CONTAINER_NAME, utils::CONTAINER_VERSION)
+        .with_mapped_port(local, utils::SERVER_PORT.tcp());
     // We want to access the container from the host, so we need to make the
     // corresponding port available.
-    let local = free_port::find_free_tcp_port().await?;
-    let port_to_expose = (local, utils::SERVER_PORT);
-    let image = RunnableImage::from(image).with_mapped_port(port_to_expose);
-    let container = image.start().await;
-    let port = container.get_host_port_ipv4(9000).await;
+    let container = image.start().await?;
+    let port = container.get_host_port_ipv4(9000).await?;
     utils::wait_for_ok(port).await?;
 
     // Once the database is available, we use the regular client to create the
@@ -356,7 +357,7 @@ async fn test() -> Result<()> {
         );
     }
 
-    container.stop().await;
+    container.stop().await?;
     Ok(())
 }
 

tremor-connectors/tests/clickhouse/simple_test.rs

@@ -19,8 +19,10 @@ use super::utils;
 use clickhouse_rs::Pool;
 use log::error;
 use std::time::{Duration, Instant};
+use testcontainers::core::IntoContainerPort;
 use testcontainers::runners::AsyncRunner;
-use testcontainers::{GenericImage, RunnableImage};
+use testcontainers::GenericImage;
+use testcontainers::ImageExt;
 use tremor_common::ports::IN;
 use tremor_connectors::{harness::Harness, impls::clickhouse};
 use tremor_connectors_test_helpers::free_port;
@@ -38,15 +40,14 @@ use tremor_value::literal;
 async fn simple_insertion() -> anyhow::Result<()> {
     // The following lines spin up a regular ClickHouse container and wait for
     // the database to be up and running.
+    let local = free_port::find_free_tcp_port().await?;
 
-    let image = GenericImage::new(utils::CONTAINER_NAME, utils::CONTAINER_VERSION);
+    let image = GenericImage::new(utils::CONTAINER_NAME, utils::CONTAINER_VERSION)
+        .with_mapped_port(local, utils::SERVER_PORT.tcp());
     // We want to access the container from the host, so we need to make the
     // corresponding port available.
-    let local = free_port::find_free_tcp_port().await?;
-    let port_to_expose = (local, utils::SERVER_PORT);
-    let image = RunnableImage::from(image).with_mapped_port(port_to_expose);
-    let container = image.start().await;
-    let port = container.get_host_port_ipv4(9000).await;
+    let container = image.start().await?;
+    let port = container.get_host_port_ipv4(9000).await?;
     utils::wait_for_ok(port).await?;
 
     // Once the database is available, we use the regular client to create the
@@ -170,7 +171,7 @@ async fn simple_insertion() -> anyhow::Result<()> {
 
     assert_eq!(ages, [42, 101]);
 
-    container.stop().await;
+    container.stop().await?;
 
     Ok(())
 }