diff --git a/external/docker-integration-tests/pom.xml b/external/docker-integration-tests/pom.xml index 298e3d36c145a..b240dd281823a 100644 --- a/external/docker-integration-tests/pom.xml +++ b/external/docker-integration-tests/pom.xml @@ -130,15 +130,9 @@ postgresql test - - - com.oracle - ojdbc6 - 11.2.0.1.0 + + com.oracle.database.jdbc + ojdbc8 test diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala index d15b366bfc9b0..6d1a22dd22b65 100644 --- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala +++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/DockerJDBCIntegrationSuite.scala @@ -95,6 +95,7 @@ abstract class DockerJDBCIntegrationSuite extends SharedSparkSession with Eventu protected val dockerIp = DockerUtils.getDockerIp() val db: DatabaseOnDocker + val connectionTimeout = timeout(2.minutes) private var docker: DockerClient = _ protected var externalPort: Int = _ @@ -155,7 +156,7 @@ abstract class DockerJDBCIntegrationSuite extends SharedSparkSession with Eventu docker.startContainer(containerId) jdbcUrl = db.getJdbcUrl(dockerIp, externalPort) var conn: Connection = null - eventually(timeout(2.minutes), interval(1.second)) { + eventually(connectionTimeout, interval(1.second)) { conn = getConnection() } // Run any setup queries: diff --git a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala index 24c3adb9c0153..9c59023cd8766 100644 --- a/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala +++ b/external/docker-integration-tests/src/test/scala/org/apache/spark/sql/jdbc/OracleIntegrationSuite.scala @@ -21,6 +21,8 @@ import java.math.BigDecimal import java.sql.{Connection, Date, Timestamp} import java.util.{Properties, TimeZone} +import org.scalatest.time.SpanSugar._ + import org.apache.spark.sql.{Row, SaveMode} import org.apache.spark.sql.execution.{RowDataSourceScanExec, WholeStageCodegenExec} import org.apache.spark.sql.execution.datasources.LogicalRelation @@ -31,27 +33,27 @@ import org.apache.spark.sql.types._ import org.apache.spark.tags.DockerTest /** - * This patch was tested using the Oracle docker. Created this integration suite for the same. - * The ojdbc6-11.2.0.2.0.jar was to be downloaded from the maven repository. Since there was - * no jdbc jar available in the maven repository, the jar was downloaded from oracle site - * manually and installed in the local; thus tested. So, for SparkQA test case run, the - * ojdbc jar might be manually placed in the local maven repository(com/oracle/ojdbc6/11.2.0.2.0) - * while Spark QA test run. - * * The following would be the steps to test this * 1. Build Oracle database in Docker, please refer below link about how to. * https://github.com/oracle/docker-images/blob/master/OracleDatabase/SingleInstance/README.md * 2. export ORACLE_DOCKER_IMAGE_NAME=$ORACLE_DOCKER_IMAGE_NAME * Pull oracle $ORACLE_DOCKER_IMAGE_NAME image - docker pull $ORACLE_DOCKER_IMAGE_NAME * 3. Start docker - sudo service docker start - * 4. Download oracle 11g driver jar and put it in maven local repo: - * (com/oracle/ojdbc6/11.2.0.2.0/ojdbc6-11.2.0.2.0.jar) - * 5. The timeout and interval parameter to be increased from 60,1 to a high value for oracle test - * in DockerJDBCIntegrationSuite.scala (Locally tested with 200,200 and executed successfully). - * 6. Run spark test - ./build/sbt "test-only org.apache.spark.sql.jdbc.OracleIntegrationSuite" + * 4. Run spark test - ./build/sbt -Pdocker-integration-tests + * "test-only org.apache.spark.sql.jdbc.OracleIntegrationSuite" + * + * An actual sequence of commands to run the test is as follows * - * All tests in this suite are ignored because of the dependency with the oracle jar from maven - * repository. + * $ git clone https://github.com/oracle/docker-images.git + * // Head SHA: 3e352a22618070595f823977a0fd1a3a8071a83c + * $ cd docker-images/OracleDatabase/SingleInstance/dockerfiles + * $ ./buildDockerImage.sh -v 18.4.0 -x + * $ export ORACLE_DOCKER_IMAGE_NAME=oracle/database:18.4.0-xe + * $ cd $SPARK_HOME + * $ ./build/sbt -Pdocker-integration-tests + * "test-only org.apache.spark.sql.jdbc.OracleIntegrationSuite" + * + * It has been validated with 18.4.0 Express Edition. */ @DockerTest class OracleIntegrationSuite extends DockerJDBCIntegrationSuite with SharedSparkSession { @@ -60,7 +62,7 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationSuite with SharedSpark override val db = new DatabaseOnDocker { override val imageName = sys.env("ORACLE_DOCKER_IMAGE_NAME") override val env = Map( - "ORACLE_ROOT_PASSWORD" -> "oracle" + "ORACLE_PWD" -> "oracle" ) override val usesIpc = false override val jdbcPort: Int = 1521 @@ -68,7 +70,11 @@ class OracleIntegrationSuite extends DockerJDBCIntegrationSuite with SharedSpark s"jdbc:oracle:thin:system/oracle@//$ip:$port/xe" } + override val connectionTimeout = timeout(7.minutes) + override def dataPreparation(conn: Connection): Unit = { + // In 18.4.0 Express Edition auto commit is enabled by default. + conn.setAutoCommit(false) conn.prepareStatement("CREATE TABLE datetime (id NUMBER(10), d DATE, t TIMESTAMP)") .executeUpdate() conn.prepareStatement( diff --git a/pom.xml b/pom.xml index 3abbf871ad2de..3e774711b16ca 100644 --- a/pom.xml +++ b/pom.xml @@ -984,6 +984,12 @@ 8.2.2.jre8 test + + com.oracle.database.jdbc + ojdbc8 + 19.6.0.0 + test + org.apache.curator curator-recipes