Close streams after they are used (#721)

* Close streams after they are used so we don't hit descriptor limits

* Run the test only in certain situations (we can fetch the descriptor
limit AND the module exists).

* Remove excess whitespace

data-migration/src/main/java/com/quorum/tessera/data/migration/JdbcDataExporter.java

@@ -1,6 +1,7 @@
 package com.quorum.tessera.data.migration;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.nio.file.Path;
 import java.sql.*;
 import java.util.List;
@@ -36,9 +37,11 @@ public void export(final StoreLoader loader,
             try (PreparedStatement insertStatement = conn.prepareStatement(insertRow)) {
                 DataEntry next;
                 while ((next = loader.nextEntry()) != null) {
-                    insertStatement.setBytes(1, next.getKey());
-                    insertStatement.setBinaryStream(2, next.getValue());
-                    insertStatement.execute();
+                    try (InputStream data = next.getValue()) {
+                        insertStatement.setBytes(1, next.getKey());
+                        insertStatement.setBinaryStream(2, data);
+                        insertStatement.execute();
+                    }
                 }
             }
 

data-migration/src/main/java/com/quorum/tessera/data/migration/SqliteDataExporter.java

@@ -5,6 +5,7 @@
 import org.apache.commons.io.IOUtils;
 
 import java.io.IOException;
+import java.io.InputStream;
 import java.net.URI;
 import java.nio.file.Files;
 import java.nio.file.Path;
@@ -39,9 +40,11 @@ public void export(final StoreLoader loader,
             try (PreparedStatement insertStatement = conn.prepareStatement(INSERT_ROW)) {
                 DataEntry next;
                 while ((next = loader.nextEntry()) != null) {
-                    insertStatement.setBytes(1, next.getKey());
-                    insertStatement.setBytes(2, IOUtils.toByteArray(next.getValue()));
-                    insertStatement.execute();
+                    try (InputStream data = next.getValue()) {
+                        insertStatement.setBytes(1, next.getKey());
+                        insertStatement.setBytes(2, IOUtils.toByteArray(data));
+                        insertStatement.execute();
+                    }
                 }
             }
 

data-migration/src/test/java/com/quorum/tessera/data/migration/CmdLineExecutorTest.java

@@ -1,12 +1,17 @@
 package com.quorum.tessera.data.migration;
 
 import com.mockrunner.mock.jdbc.JDBCMockObjectFactory;
+import com.sun.management.UnixOperatingSystemMXBean;
 import org.apache.commons.cli.MissingOptionException;
+import org.apache.commons.codec.binary.Base32;
+import org.apache.commons.io.IOUtils;
 import org.junit.Before;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.TestName;
 
+import java.io.InputStream;
+import java.lang.management.ManagementFactory;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -78,9 +83,6 @@ public void missingInputFileOption() {
         assertThat(((MissingOptionException) throwable).getMissingOptions()).containsExactlyInAnyOrder("inputpath");
     }
 
-
-
-
     @Test
     public void bdbStoreType() throws Exception {
         final Path inputFile = Paths.get(getClass().getResource("/bdb/single-entry.txt").toURI());
@@ -153,4 +155,42 @@ public void exportTypeJdbc() throws Exception {
         }
 
     }
+
+    //This tests that even with a lot of files, the file descriptor limit isn't hit
+    @Test
+    public void directoryStoreAndSqliteWithLotsOfFilesWorks() throws Exception {
+        final Path descriptorTestFolder = Files.createTempDirectory("descriptorTest");
+        final InputStream dataStream = getClass().getResourceAsStream("/dir/2JRLWGXFSDJUYUKADO7VFO3INL27WUXB2YDR5FCI3REQDTJGX6FULIDCIMYDV4H23PFUECWFYBMTIUTNY2ESAFMQADFCFUYBHBBJT4I=");
+        final byte[] data = IOUtils.toByteArray(dataStream);
+
+        //this code snippet fetches the number of file descriptors we can use
+        //some will already be used, but opening more doesn't hurt since that is what we are testing
+        if (!(ManagementFactory.getOperatingSystemMXBean() instanceof UnixOperatingSystemMXBean)) {
+            //we skip this test on Windows and other unsupported OS's
+            //the point of the test is to show we don't keep file descriptors open when not needed
+            //which is independent of the OS we are running on
+            return;
+        }
+
+        final UnixOperatingSystemMXBean osMxBean
+            = (UnixOperatingSystemMXBean)ManagementFactory.getOperatingSystemMXBean();
+        final long descriptorCount = osMxBean.getMaxFileDescriptorCount();
+
+        for(int i = 0; i < descriptorCount; i++) {
+            final String filename = new Base32().encodeToString(String.valueOf(i).getBytes());
+            final Path newFile = descriptorTestFolder.resolve(filename);
+            Files.write(newFile, data);
+        }
+
+        final String[] args = new String[]{
+            "-storetype", "dir",
+            "-inputpath", descriptorTestFolder.toString(),
+            "-outputfile", outputPath.toString(),
+            "-exporttype", "sqlite",
+            "-dbpass", "-dbuser"
+        };
+
+        executor.execute(args);
+    }
+
 }