Align data migration tool with current version

data-migration/pom.xml

@@ -27,7 +27,10 @@
             <groupId>com.jpmorgan.quorum</groupId>
             <artifactId>shared</artifactId>
         </dependency>
-
+        <dependency>
+            <groupId>com.jpmorgan.quorum</groupId>
+            <artifactId>ddls</artifactId>
+        </dependency>
         <dependency>
             <groupId>commons-cli</groupId>
             <artifactId>commons-cli</artifactId>
@@ -81,6 +84,33 @@
     <build>
 
         <plugins>
+
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-dependency-plugin</artifactId>
+                <version>3.1.1</version>
+                <executions>
+
+
+                    <execution>
+                        <id>unpack-ddls</id>
+                        <phase>process-resources</phase>
+                        <goals>
+                            <goal>unpack</goal>
+                        </goals>
+                        <configuration>
+                            <artifactItems>
+                                <artifactItem>
+                                    <groupId>com.jpmorgan.quorum</groupId>
+                                    <artifactId>ddls</artifactId>
+                                </artifactItem>
+                            </artifactItems>
+                            <outputDirectory>${project.build.outputDirectory}/ddls</outputDirectory>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>

data-migration/src/main/java/com/quorum/tessera/data/migration/CmdLineExecutor.java

@@ -1,21 +1,14 @@
 package com.quorum.tessera.data.migration;
 
+import org.apache.commons.cli.*;
+
 import java.io.InputStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.Arrays;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Optional;
-import java.util.Properties;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.DefaultParser;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.MissingOptionException;
-import org.apache.commons.cli.Option;
-import org.apache.commons.cli.Options;
+import java.util.*;
+
+import static java.util.Collections.singletonList;
 
 public class CmdLineExecutor {
 
@@ -49,7 +42,7 @@ protected static int execute(String... args) throws Exception {
                         .required()
                         .build());
 
-        
+
 
         options.addOption(
                 Option.builder()
@@ -61,7 +54,7 @@ protected static int execute(String... args) throws Exception {
                         .argName("TYPE")
                         .required()
                         .build());
-        
+
         options.addOption(
                 Option.builder()
                         .longOpt("dbconfig")
@@ -148,14 +141,18 @@ protected static int execute(String... args) throws Exception {
 
             String insertRow = Objects.requireNonNull(properties.getProperty("insertRow",null),
                     "No insertRow value defined in config file. ");
-            
+
             String createTable = Objects.requireNonNull(properties.getProperty("createTable",null),
                     "No createTable value defined in config file. ");
-            
+
             String jdbcUrl = Objects.requireNonNull(properties.getProperty("jdbcUrl",null),
                     "No jdbcUrl value defined in config file. ");
 
-            dataExporter = new JdbcDataExporter(jdbcUrl, insertRow, createTable);
+            Path sqlFile = Files.createTempFile(UUID.randomUUID().toString(), ".txt");
+
+            Files.write(sqlFile, singletonList(createTable));
+
+            dataExporter = new JdbcDataExporter(jdbcUrl, insertRow, sqlFile.toUri().toURL());
 
         } else {
             dataExporter = DataExporterFactory.create(exportType);

data-migration/src/main/java/com/quorum/tessera/data/migration/H2DataExporter.java

@@ -1,22 +1,14 @@
 package com.quorum.tessera.data.migration;
 
+import java.net.URL;
 import java.nio.file.Path;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.PreparedStatement;
 import java.sql.SQLException;
-import java.sql.Statement;
 import java.util.Map;
-import java.util.Map.Entry;
 
 public class H2DataExporter implements DataExporter {
 
     private static final String INSERT_ROW = "INSERT INTO ENCRYPTED_TRANSACTION (HASH,ENCODED_PAYLOAD) VALUES (?,?)";
 
-    private static final String CREATE_TABLE = "CREATE TABLE ENCRYPTED_TRANSACTION "
-        + "(ENCODED_PAYLOAD LONGVARBINARY NOT NULL, "
-        + "HASH LONGVARBINARY NOT NULL UNIQUE, PRIMARY KEY (HASH))";
-
     @Override
     public void export(final Map<byte[], byte[]> data,
                        final Path output,
@@ -25,21 +17,12 @@ public void export(final Map<byte[], byte[]> data,
 
         final String connectionString = "jdbc:h2:" + output.toString();
 
-        try (Connection conn = DriverManager.getConnection(connectionString, username, password)) {
+        final URL sqlFile = getClass().getResource("/ddls/h2-ddl.sql");
 
-            try (Statement stmt = conn.createStatement()) {
-                stmt.executeUpdate(CREATE_TABLE);
-            }
+        final JdbcDataExporter jdbcDataExporter = new JdbcDataExporter(connectionString, INSERT_ROW, sqlFile);
 
-            try (PreparedStatement insertStatement = conn.prepareStatement(INSERT_ROW)) {
-                for (Entry<byte[], byte[]> values : data.entrySet()) {
-                    insertStatement.setBytes(1, values.getKey());
-                    insertStatement.setBytes(2, values.getValue());
-                    insertStatement.execute();
-                }
-            }
+        jdbcDataExporter.export(data, output, username, password);
 
-        }
     }
 
 }

data-migration/src/main/java/com/quorum/tessera/data/migration/JdbcDataExporter.java

@@ -1,11 +1,11 @@
 package com.quorum.tessera.data.migration;
 
+import java.net.URL;
+import java.nio.file.Files;
 import java.nio.file.Path;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
-import java.sql.Statement;
+import java.nio.file.Paths;
+import java.sql.*;
+import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 
@@ -15,12 +15,12 @@ public class JdbcDataExporter implements DataExporter {
 
     private final String insertRow;
 
-    private final String createTable;
+    private final List<String> createTables;
 
-    public JdbcDataExporter(String jdbcUrl, String insertRow, String createTable) {
+    public JdbcDataExporter(String jdbcUrl, String insertRow, URL ddl) {
         this.jdbcUrl = jdbcUrl;
         this.insertRow = insertRow;
-        this.createTable = createTable;
+        this.createTables = UriCallback.execute(() -> Files.readAllLines(Paths.get(ddl.toURI())));
     }
 
     @Override
@@ -29,7 +29,9 @@ public void export(Map<byte[], byte[]> data, Path output, String username, Strin
         try (Connection conn = DriverManager.getConnection(jdbcUrl, username, password)) {
 
             try (Statement stmt = conn.createStatement()) {
-                stmt.executeUpdate(createTable);
+                for (String createTable : createTables) {
+                    stmt.executeUpdate(createTable);
+                }
             }
 
             try (PreparedStatement insertStatement = conn.prepareStatement(insertRow)) {

data-migration/src/main/java/com/quorum/tessera/data/migration/SqliteDataExporter.java

@@ -1,38 +1,25 @@
 package com.quorum.tessera.data.migration;
 
+import java.net.URL;
 import java.nio.file.Path;
-import java.sql.*;
+import java.sql.SQLException;
 import java.util.Map;
 
-//FIXME: Need to address sequence generation config for sql bfeore going live with this
 public class SqliteDataExporter implements DataExporter {
 
     private static final String INSERT_ROW = "INSERT INTO ENCRYPTED_TRANSACTION (HASH,ENCODED_PAYLOAD) VALUES (?,?)";
 
-    private static final String CREATE_TABLE = "CREATE TABLE ENCRYPTED_TRANSACTION "
-        + "(ENCODED_PAYLOAD LONGVARBINARY NOT NULL, "
-        + "HASH LONGVARBINARY NOT NULL UNIQUE, PRIMARY KEY (HASH))";
-
     @Override
     public void export(Map<byte[], byte[]> data, Path output, final String username, final String password) throws SQLException {
 
         final String connectionString = "jdbc:sqlite:" + output.toString();
 
-        try (Connection conn = DriverManager.getConnection(connectionString, username, password)) {
+        final URL sqlFile = getClass().getResource("/ddls/h2-ddl.sql");
 
-            try (Statement stmt = conn.createStatement()) {
-                stmt.executeUpdate(CREATE_TABLE);
-            }
+        final JdbcDataExporter jdbcDataExporter = new JdbcDataExporter(connectionString, INSERT_ROW, sqlFile);
 
-            try (PreparedStatement insertStatement = conn.prepareStatement(INSERT_ROW)) {
-                for (Map.Entry<byte[], byte[]> values : data.entrySet()) {
-                    insertStatement.setBytes(1, values.getKey());
-                    insertStatement.setBytes(2, values.getValue());
-                    insertStatement.execute();
-                }
-            }
+        jdbcDataExporter.export(data, output, username, password);
 
-        }
     }
 
 }

data-migration/src/main/java/com/quorum/tessera/data/migration/UriCallback.java

@@ -0,0 +1,22 @@
+package com.quorum.tessera.data.migration;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.net.URISyntaxException;
+
+public interface UriCallback<T> {
+
+    T doExecute() throws IOException, URISyntaxException;
+
+    static <T> T execute(UriCallback<T> callback) {
+        try {
+            return callback.doExecute();
+        } catch (IOException e) {
+            throw new UncheckedIOException(e);
+        } catch (URISyntaxException ex) {
+            throw new UncheckedIOException(new IOException(ex.getMessage()));
+        }
+
+    }
+
+}

data-migration/src/test/java/com/quorum/tessera/data/migration/H2DataExporterTest.java

@@ -1,29 +1,27 @@
 package com.quorum.tessera.data.migration;
 
-import java.io.File;
-
 import org.h2.jdbc.JdbcSQLException;
 import org.junit.After;
 import org.junit.Before;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TestName;
 
+import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.nio.file.Path;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.ResultSet;
-import java.sql.SQLException;
+import java.sql.*;
 import java.util.Comparator;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 
 import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.catchThrowable;
 
-import org.junit.Rule;
-import org.junit.rules.TestName;
-
 public class H2DataExporterTest {
 
     private H2DataExporter exporter;
@@ -63,6 +61,14 @@ public void exportSingleLine() throws SQLException, IOException {
 
         try (Connection conn = DriverManager.getConnection(connectionString)) {
             try (ResultSet rs = conn.prepareStatement("SELECT * FROM ENCRYPTED_TRANSACTION").executeQuery()) {
+
+                ResultSetMetaData metaData = rs.getMetaData();
+                List<String> columnNames = IntStream.range(1,metaData.getColumnCount() + 1)
+                    .mapToObj(i -> JdbcCallback.execute(() -> metaData.getColumnName(i)))
+                    .collect(Collectors.toList());
+
+                assertThat(columnNames).containsExactlyInAnyOrder("HASH","ENCODED_PAYLOAD","TIMESTAMP");
+
                 while (rs.next()) {
                     assertThat(rs.getBytes("HASH")).isEqualTo("HASH".getBytes());
                     assertThat(rs.getBytes("ENCODED_PAYLOAD")).isEqualTo("VALUE".getBytes());
@@ -91,6 +97,14 @@ public void exportSingleLineWithUsernameAndPassword() throws SQLException, IOExc
 
         try (Connection conn = DriverManager.getConnection(connectionString, username, password)) {
             try (ResultSet rs = conn.prepareStatement("SELECT * FROM ENCRYPTED_TRANSACTION").executeQuery()) {
+
+                ResultSetMetaData metaData = rs.getMetaData();
+                List<String> columnNames = IntStream.range(1,metaData.getColumnCount() + 1)
+                    .mapToObj(i -> JdbcCallback.execute(() -> metaData.getColumnName(i)))
+                    .collect(Collectors.toList());
+
+                assertThat(columnNames).containsExactlyInAnyOrder("HASH","ENCODED_PAYLOAD","TIMESTAMP");
+
                 while (rs.next()) {
                     assertThat(rs.getBytes("HASH")).isEqualTo("HASH".getBytes());
                     assertThat(rs.getBytes("ENCODED_PAYLOAD")).isEqualTo("VALUE".getBytes());

data-migration/src/test/java/com/quorum/tessera/data/migration/JdbcDataExporterTest.java

@@ -1,19 +1,21 @@
-
 package com.quorum.tessera.data.migration;
 
 import com.mockrunner.jdbc.BasicJDBCTestCaseAdapter;
 import com.mockrunner.mock.jdbc.JDBCMockObjectFactory;
+import org.junit.After;
+import org.junit.Test;
+
+import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.Arrays;
 import java.util.HashMap;
 import java.util.Map;
-import org.junit.After;
-import org.junit.Test;
-import static org.mockito.Mockito.mock;
+import java.util.UUID;
 
+import static org.mockito.Mockito.mock;
 
 public class JdbcDataExporterTest extends BasicJDBCTestCaseAdapter{
-
-
+
     private final JDBCMockObjectFactory mockObjectFactory = new JDBCMockObjectFactory();
 
     @Test
@@ -28,24 +30,23 @@ public void onTearDown(){
 
     @Test
     public void doStuff() throws Exception {
-
+        Path sqlFile = Files.createTempFile(UUID.randomUUID().toString(),".txt");
+
+        Files.write(sqlFile, Arrays.asList("create stuff"));
+
+        JdbcDataExporter exporter = new JdbcDataExporter("jdbc:bogus","insert stuff",sqlFile.toUri().toURL());
 
-        JdbcDataExporter exporter = new JdbcDataExporter("jdbc:bogus","insert stuff","create stuff");
-
         Map<byte[],byte[]> data = new HashMap<byte[],byte[]>() {{
             put("ONE".getBytes(),"TWO".getBytes());
         }};
-
-
-
+
         Path output = mock(Path.class);
 
         exporter.export(data, output, "someone", "pw");
 
         verifyAllStatementsClosed();
         verifyAllStatementsClosed();
 
-
     }
-    
+
 }