+ * http://www.apache.org/licenses/LICENSE-2.0 + *
+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.hadoop.ozone.recon.codegen; + +import static java.util.concurrent.TimeUnit.SECONDS; + +import org.apache.hadoop.hdds.conf.Config; +import org.apache.hadoop.hdds.conf.ConfigGroup; +import org.apache.hadoop.hdds.conf.ConfigTag; +import org.apache.hadoop.hdds.conf.ConfigType; + +/** + * The configuration class for the Recon SQL DB. + */ +@ConfigGroup(prefix = "ozone.recon.sql.db") +public class ReconSqlDbConfig { + + @Config(key = "driver", + type = ConfigType.STRING, + defaultValue = "org.apache.derby.jdbc.EmbeddedDriver", + tags = {ConfigTag.STORAGE, ConfigTag.RECON, ConfigTag.OZONE}, + description = "Recon SQL DB driver class. Defaults to Derby." + ) + private String driverClass; + + public String getDriverClass() { + return driverClass; + } + + public void setDriverClass(String driverClass) { + this.driverClass = driverClass; + } + + @Config(key = "jdbc.url", + type = ConfigType.STRING, + defaultValue = "jdbc:derby:${ozone.recon.db.dir}/ozone_recon_derby.db", + tags = {ConfigTag.STORAGE, ConfigTag.RECON, ConfigTag.OZONE}, + description = "Ozone Recon SQL database jdbc url." + ) + private String jdbcUrl; + + public String getJdbcUrl() { + return jdbcUrl; + } + + public void setJdbcUrl(String jdbcUrl) { + this.jdbcUrl = jdbcUrl; + } + + @Config(key = "username", + type = ConfigType.STRING, + defaultValue = "", + tags = {ConfigTag.STORAGE, ConfigTag.RECON, ConfigTag.OZONE}, + description = "Ozone Recon SQL database username." + ) + private String username; + + public String getUsername() { + return username; + } + + public void setUsername(String username) { + this.username = username; + } + + @Config(key = "password", + type = ConfigType.STRING, + defaultValue = "", + tags = {ConfigTag.STORAGE, ConfigTag.RECON, ConfigTag.OZONE}, + description = "Ozone Recon SQL database password." + ) + private String password; + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + + @Config(key = "auto.commit", + type = ConfigType.BOOLEAN, + defaultValue = "false", + tags = {ConfigTag.STORAGE, ConfigTag.RECON, ConfigTag.OZONE}, + description = "Sets the Ozone Recon database connection property of " + + "auto-commit to true/false." + ) + private boolean autoCommit; + + public boolean isAutoCommit() { + return autoCommit; + } + + public void setAutoCommit(boolean autoCommit) { + this.autoCommit = autoCommit; + } + + @Config(key = "conn.timeout", + type = ConfigType.TIME, + defaultValue = "30000ms", + tags = {ConfigTag.STORAGE, ConfigTag.RECON, ConfigTag.OZONE}, + description = "Sets time in milliseconds before call to getConnection " + + "is timed out." + ) + private long connectionTimeout; + + public long getConnectionTimeout() { + return connectionTimeout; + } + + public void setConnectionTimeout(long connectionTimeout) { + this.connectionTimeout = connectionTimeout; + } + + @Config(key = "conn.max.active", + type = ConfigType.INT, + defaultValue = "5", + tags = {ConfigTag.STORAGE, ConfigTag.RECON, ConfigTag.OZONE}, + description = "The max active connections to the SQL database." + ) + private int maxActiveConnections; + + public int getMaxActiveConnections() { + return maxActiveConnections; + } + + public void setMaxActiveConnections(int maxActiveConnections) { + this.maxActiveConnections = maxActiveConnections; + } + + @Config(key = "conn.max.age", + type = ConfigType.TIME, timeUnit = SECONDS, + defaultValue = "1800s", + tags = {ConfigTag.STORAGE, ConfigTag.RECON, ConfigTag.OZONE}, + description = "Sets maximum time a connection can be active in seconds." + ) + private long connectionMaxAge; + + public long getConnectionMaxAge() { + return connectionMaxAge; + } + + public void setConnectionMaxAge(long connectionMaxAge) { + this.connectionMaxAge = connectionMaxAge; + } + + @Config(key = "conn.idle.max.age", + type = ConfigType.TIME, timeUnit = SECONDS, + defaultValue = "3600s", + tags = {ConfigTag.STORAGE, ConfigTag.RECON, ConfigTag.OZONE}, + description = "Sets maximum time to live for idle connection in seconds." + ) + private long connectionIdleMaxAge; + + public long getConnectionIdleMaxAge() { + return connectionIdleMaxAge; + } + + public void setConnectionIdleMaxAge(long connectionIdleMaxAge) { + this.connectionIdleMaxAge = connectionIdleMaxAge; + } + + @Config(key = "conn.idle.test.period", + type = ConfigType.TIME, timeUnit = SECONDS, + defaultValue = "60s", + tags = {ConfigTag.STORAGE, ConfigTag.RECON, ConfigTag.OZONE}, + description = "Sets maximum time to live for idle connection in seconds." + ) + private long connectionIdleTestPeriod; + + public long getConnectionIdleTestPeriod() { + return connectionIdleTestPeriod; + } + + public void setConnectionIdleTestPeriod(long connectionIdleTestPeriod) { + this.connectionIdleTestPeriod = connectionIdleTestPeriod; + } + + @Config(key = "conn.idle.test", + type = ConfigType.STRING, + defaultValue = "SELECT 1", + tags = {ConfigTag.STORAGE, ConfigTag.RECON, ConfigTag.OZONE}, + description = "The query to send to the DB to maintain keep-alives and " + + "test for dead connections." + ) + private String idleTestQuery; + + public String getIdleTestQuery() { + return idleTestQuery; + } + + public void setIdleTestQuery(String idleTestQuery) { + this.idleTestQuery = idleTestQuery; + } + + @Config(key = "jooq.dialect", + type = ConfigType.STRING, + defaultValue = "DERBY", + tags = {ConfigTag.STORAGE, ConfigTag.RECON, ConfigTag.OZONE}, + description = "Recon internally uses Jooq to talk to its SQL DB. By " + + "default, we support Derby and Sqlite out of the box. Please refer " + + "to https://www.jooq.org/javadoc/latest/org" + + ".jooq/org/jooq/SQLDialect.html to specify different dialect." + ) + private String sqlDbDialect; + + public String getSqlDbDialect() { + return sqlDbDialect; + } + + public void setSqlDbDialect(String sqlDbDialect) { + this.sqlDbDialect = sqlDbDialect; + } + + /** + * Class to hold config keys related to Recon SQL DB. + */ + public static class ConfigKeys { + public static final String OZONE_RECON_SQL_DB_JDBC_URL = + "ozone.recon.sql.db.jdbc.url"; + } +} diff --git a/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/SqlDbUtils.java b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/SqlDbUtils.java new file mode 100644 index 000000000000..7e68541cf840 --- /dev/null +++ b/hadoop-ozone/recon-codegen/src/main/java/org/hadoop/ozone/recon/codegen/SqlDbUtils.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *
+ * http://www.apache.org/licenses/LICENSE-2.0 + *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.hadoop.ozone.recon.codegen;
+
+import static org.jooq.impl.DSL.count;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+import java.util.function.BiPredicate;
+
+import org.jooq.exception.DataAccessException;
+import org.jooq.impl.DSL;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Constants and Helper functions for Recon SQL related stuff.
+ */
+public final class SqlDbUtils {
+
+ public final static String DERBY_DRIVER_CLASS =
+ "org.apache.derby.jdbc.EmbeddedDriver";
+ public final static String SQLITE_DRIVER_CLASS = "org.sqlite.JDBC";
+ public final static String DERBY_DISABLE_LOG_METHOD =
+ SqlDbUtils.class.getName() + ".disableDerbyLogFile";
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SqlDbUtils.class);
+
+ private SqlDbUtils() {
+ }
+
+ /**
+ * Create new Derby Database with URL and schema name.
+ * @param jdbcUrl JDBC url.
+ * @param schemaName Schema name
+ * @throws ClassNotFoundException on not finding driver class.
+ * @throws SQLException on SQL exception.
+ */
+ public static void createNewDerbyDatabase(String jdbcUrl, String schemaName)
+ throws ClassNotFoundException, SQLException {
+ System.setProperty("derby.stream.error.method",
+ DERBY_DISABLE_LOG_METHOD);
+ Class.forName(DERBY_DRIVER_CLASS);
+ try(Connection connection = DriverManager.getConnection(jdbcUrl
+ + ";user=" + schemaName
+ + ";create=true")) {
+ LOG.info("Created derby database at {}.", jdbcUrl);
+ }
+ }
+
+ /**
+ * Used to suppress embedded derby database logging.
+ * @return No-Op output stream.
+ */
+ public static OutputStream disableDerbyLogFile(){
+ return new OutputStream() {
+ public void write(int b) throws IOException {
+ // Ignore all log messages
+ }
+ };
+ }
+
+ /**
+ * Helper function to check if table exists through JOOQ.
+ */
+ public static final BiPredicate
+ * http://www.apache.org/licenses/LICENSE-2.0 + *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.recon.persistence;
+
+import static org.hadoop.ozone.recon.codegen.JooqCodeGenerator.RECON_SCHEMA_NAME;
+import static org.hadoop.ozone.recon.codegen.SqlDbUtils.createNewDerbyDatabase;
+
+import javax.sql.DataSource;
+
+import org.apache.derby.jdbc.EmbeddedDataSource;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+
+/**
+ * Provide a {@link javax.sql.DataSource} for the application.
+ */
+public class DerbyDataSourceProvider implements Provider
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon.persistence;
+
+import javax.sql.DataSource;
+
+import org.sqlite.SQLiteDataSource;
+
+import com.google.inject.Inject;
+import com.google.inject.Provider;
+
+/**
+ * Provide a {@link javax.sql.DataSource} for the application.
+ */
+public class SqliteDataSourceProvider implements Provider
+ * Default sqlite database does not work with a connection pool, actually
+ * most embedded databases do not, hence returning native implementation for
+ * default db.
+ */
+ @Override
+ public DataSource get() {
+ SQLiteDataSource ds = new SQLiteDataSource();
+ ds.setUrl(configuration.getJdbcUrl());
+ return ds;
+ }
+}
\ No newline at end of file
diff --git a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/AbstractReconSqlDBTest.java b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/AbstractReconSqlDBTest.java
index f8768dce0f74..664a7321d9a2 100644
--- a/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/AbstractReconSqlDBTest.java
+++ b/hadoop-ozone/recon/src/test/java/org/apache/hadoop/ozone/recon/persistence/AbstractReconSqlDBTest.java
@@ -17,6 +17,8 @@
*/
package org.apache.hadoop.ozone.recon.persistence;
+import static org.hadoop.ozone.recon.codegen.SqlDbUtils.DERBY_DRIVER_CLASS;
+
import java.io.File;
import java.io.IOException;
import java.sql.Connection;
@@ -56,10 +58,22 @@ public class AbstractReconSqlDBTest {
private Injector injector;
private DSLContext dslContext;
+ private Provider
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.ozone.recon.persistence;
+
+import static java.util.stream.Collectors.toList;
+import static org.apache.hadoop.ozone.recon.ReconControllerModule.ReconDaoBindingModule.RECON_DAO_LIST;
+import static org.hadoop.ozone.recon.codegen.SqlDbUtils.SQLITE_DRIVER_CLASS;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import java.io.File;
+import java.io.IOException;
+import java.sql.SQLException;
+import java.util.stream.Stream;
+
+import org.hadoop.ozone.recon.schema.tables.daos.ReconTaskStatusDao;
+import org.hadoop.ozone.recon.schema.tables.pojos.ReconTaskStatus;
+import org.jooq.SQLDialect;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import com.google.inject.Provider;
+
+/**
+ * Test Recon schema with different DBs.
+ */
+@RunWith(Parameterized.class)
+public class TestReconWithDifferentSqlDBs extends AbstractReconSqlDBTest {
+
+ public TestReconWithDifferentSqlDBs(
+ Provider