use intelDeflaterFactory

build.gradle

@@ -26,7 +26,7 @@ import de.undercouch.gradle.tasks.download.Download
 mainClassName = "org.broadinstitute.hellbender.Main"
 
 //Note: the test suite must use the same defaults. If you change system properties in this list you must also update the one in the test task
-applicationDefaultJvmArgs = ["-Dsamjdk.use_async_io_samtools=true", "-Dsamjdk.use_async_io_tribble=false", "-Dsamjdk.intel_deflater_so_path=build/libIntelDeflater.so", "-Dsamjdk.compression_level=1"]
+applicationDefaultJvmArgs = ["-Dsamjdk.use_async_io_read_samtools=false","-Dsamjdk.use_async_io_write_samtools=true", "-Dsamjdk.use_async_io_write_tribble=false", "-Dsamjdk.compression_level=1"]
 
 //Delete the windows script - we never test on Windows so let's not pretend it works
 startScripts {
@@ -235,9 +235,9 @@ test {
         }
     }
 
-    systemProperty "samjdk.use_async_io_samtools", "true"
-    systemProperty "samjdk.use_async_io_tribble", "false"
-    systemProperty "samjdk.intel_deflater_so_path", "build/libIntelDeflater.so"
+    systemProperty "samjdk.use_async_io_read_samtools", "false"
+    systemProperty "samjdk.use_async_io_write_samtools", "true"
+    systemProperty "samjdk.use_async_io_write_tribble", "false"
     systemProperty "samjdk.compression_level", "1"
     systemProperty "gatk.spark.debug", System.getProperty("gatk.spark.debug")
 

gatk-launch

@@ -23,14 +23,13 @@ BUILD_LOCATION = script +"/build/install/" + projectName + "/bin/"
 GATK_RUN_SCRIPT = BUILD_LOCATION + projectName
 BIN_PATH = script + "/build/libs"
 
-EXTRA_JAVA_OPTIONS="-Dsamjdk.intel_deflater_so_path=libIntelDeflater.so -Dsamjdk.compression_level=1 -DGATK_STACKTRACE_ON_USER_EXCEPTION=true "
+EXTRA_JAVA_OPTIONS="-Dsamjdk.compression_level=1 -DGATK_STACKTRACE_ON_USER_EXCEPTION=true "
 
 DEFAULT_SPARK_ARGS = ["--conf", "spark.kryoserializer.buffer.max=512m",
 "--conf", "spark.driver.maxResultSize=0",
 "--conf", "spark.driver.userClassPathFirst=true",
 "--conf", "spark.io.compression.codec=lzf",
 "--conf", "spark.yarn.executor.memoryOverhead=600",
-"--conf", "spark.yarn.dist.files=" + script + "/build/libIntelDeflater.so",
 "--conf", "spark.driver.extraJavaOptions=" + EXTRA_JAVA_OPTIONS,
 "--conf", "spark.executor.extraJavaOptions=" + EXTRA_JAVA_OPTIONS]
 

src/main/java/org/broadinstitute/hellbender/cmdline/CommandLineProgram.java

@@ -1,13 +1,14 @@
 package org.broadinstitute.hellbender.cmdline;
 
+import com.intel.gkl.compression.IntelDeflaterFactory;
 import htsjdk.samtools.Defaults;
 import htsjdk.samtools.metrics.Header;
 import htsjdk.samtools.metrics.MetricBase;
 import htsjdk.samtools.metrics.MetricsFile;
 import htsjdk.samtools.metrics.StringHeader;
+import htsjdk.samtools.util.BlockCompressedOutputStream;
 import htsjdk.samtools.util.IOUtil;
 import htsjdk.samtools.util.Log;
-import htsjdk.samtools.util.zip.DeflaterFactory;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.broadinstitute.hellbender.utils.LoggingUtils;
@@ -56,6 +57,9 @@ public abstract class CommandLineProgram {
     public Boolean QUIET = false;
     private final String standardUsagePreamble = CommandLineParser.getStandardUsagePreamble(getClass());
 
+    @Argument(fullName = "use_jdk_deflater", shortName = "jdk_deflater", doc = "Whether to use the JdkDeflater (as opposed to IntelDeflater)", common=true)
+    public boolean useJdkDeflater = false;
+
     /**
     * Initialized in parseArgs.  Subclasses may want to access this to do their
     * own validation, and then print usage using commandLineParser.
@@ -128,6 +132,15 @@ public Object instanceMainPostParseArgs() {
             System.setProperty("java.io.tmpdir", f.getAbsolutePath()); // in loop so that last one takes effect
         }
 
+        //Set defaults (note: setting them here means they are not controllable by the user)
+        if (! useJdkDeflater) {
+            BlockCompressedOutputStream.setDefaultDeflaterFactory(new IntelDeflaterFactory());
+        }
+        //Note: IntelDeflaterFactory (incorrectly) only loads the library and sets the usingIntelDeflater if some deflaters are actually created
+        //That means that the reported Deflater will be JdkDeflater even when IntelDeflater will actually be used.
+        //The workaround is to forcefully make 1 deflater from the factory
+        BlockCompressedOutputStream.getDefaultDeflaterFactory().makeDeflater(1, true);
+
         if (!QUIET) {
             System.err.println("[" + ZonedDateTime.now().format(DateTimeFormatter.ofLocalizedDateTime(FormatStyle.LONG)) +
                                 "] " + commandLine);
@@ -145,8 +158,8 @@ public Object instanceMainPostParseArgs() {
                 Defaults.allDefaults().entrySet().stream().forEach(e->
                         logger.info(Defaults.class.getSimpleName() + "." + e.getKey() + " : " + e.getValue())
                 );
-// TODO: remove comment when this is working in HaplotypeCaller
-//                logger.info("Deflater " + (DeflaterFactory.usingIntelDeflater()? "IntelDeflater": "JdkDeflater"));
+                final boolean usingIntelDeflater = (BlockCompressedOutputStream.getDefaultDeflaterFactory() instanceof IntelDeflaterFactory && ((IntelDeflaterFactory)BlockCompressedOutputStream.getDefaultDeflaterFactory()).usingIntelDeflater());
+                logger.info("Deflater " + (usingIntelDeflater ? "IntelDeflater": "JdkDeflater"));
             }
             catch (final Exception e) { /* Unpossible! */ }
         }