diff --git a/build.gradle b/build.gradle index ee769b9f58e..c95bd4519c7 100644 --- a/build.gradle +++ b/build.gradle @@ -26,7 +26,7 @@ import de.undercouch.gradle.tasks.download.Download mainClassName = "org.broadinstitute.hellbender.Main" //Note: the test suite must use the same defaults. If you change system properties in this list you must also update the one in the test task -applicationDefaultJvmArgs = ["-Dsamjdk.use_async_io_samtools=true", "-Dsamjdk.use_async_io_tribble=false", "-Dsamjdk.intel_deflater_so_path=build/libIntelDeflater.so", "-Dsamjdk.compression_level=1"] +applicationDefaultJvmArgs = ["-Dsamjdk.use_async_io_read_samtools=false","-Dsamjdk.use_async_io_write_samtools=true", "-Dsamjdk.use_async_io_write_tribble=false", "-Dsamjdk.compression_level=1"] //Delete the windows script - we never test on Windows so let's not pretend it works startScripts { @@ -171,7 +171,7 @@ dependencies { // Dependency change for including MLLib compile('com.esotericsoftware:reflectasm:1.10.0:shaded') - compile('com.intel.gkl:gkl:0.1.0') { + compile('com.intel.gkl:gkl:0.1.1') { exclude module: 'htsjdk' } compile 'com.github.lindenb:jbwa:bd72b489f9697a857671533c86ad5906967f8672-20160521.042635-1' @@ -235,9 +235,9 @@ test { } } - systemProperty "samjdk.use_async_io_samtools", "true" - systemProperty "samjdk.use_async_io_tribble", "false" - systemProperty "samjdk.intel_deflater_so_path", "build/libIntelDeflater.so" + systemProperty "samjdk.use_async_io_read_samtools", "false" + systemProperty "samjdk.use_async_io_write_samtools", "true" + systemProperty "samjdk.use_async_io_write_tribble", "false" systemProperty "samjdk.compression_level", "1" systemProperty "gatk.spark.debug", System.getProperty("gatk.spark.debug") diff --git a/gatk-launch b/gatk-launch index 27028c2a38d..39e19d38cd2 100755 --- a/gatk-launch +++ b/gatk-launch @@ -23,14 +23,13 @@ BUILD_LOCATION = script +"/build/install/" + projectName + "/bin/" GATK_RUN_SCRIPT = BUILD_LOCATION + projectName BIN_PATH = script + "/build/libs" -EXTRA_JAVA_OPTIONS="-Dsamjdk.intel_deflater_so_path=libIntelDeflater.so -Dsamjdk.compression_level=1 -DGATK_STACKTRACE_ON_USER_EXCEPTION=true " +EXTRA_JAVA_OPTIONS="-Dsamjdk.compression_level=1 -DGATK_STACKTRACE_ON_USER_EXCEPTION=true " DEFAULT_SPARK_ARGS = ["--conf", "spark.kryoserializer.buffer.max=512m", "--conf", "spark.driver.maxResultSize=0", "--conf", "spark.driver.userClassPathFirst=true", "--conf", "spark.io.compression.codec=lzf", "--conf", "spark.yarn.executor.memoryOverhead=600", -"--conf", "spark.yarn.dist.files=" + script + "/build/libIntelDeflater.so", "--conf", "spark.driver.extraJavaOptions=" + EXTRA_JAVA_OPTIONS, "--conf", "spark.executor.extraJavaOptions=" + EXTRA_JAVA_OPTIONS] diff --git a/src/main/java/org/broadinstitute/hellbender/cmdline/CommandLineProgram.java b/src/main/java/org/broadinstitute/hellbender/cmdline/CommandLineProgram.java index b6fdc9c6669..2019a6161b7 100644 --- a/src/main/java/org/broadinstitute/hellbender/cmdline/CommandLineProgram.java +++ b/src/main/java/org/broadinstitute/hellbender/cmdline/CommandLineProgram.java @@ -1,13 +1,14 @@ package org.broadinstitute.hellbender.cmdline; +import com.intel.gkl.compression.IntelDeflaterFactory; import htsjdk.samtools.Defaults; import htsjdk.samtools.metrics.Header; import htsjdk.samtools.metrics.MetricBase; import htsjdk.samtools.metrics.MetricsFile; import htsjdk.samtools.metrics.StringHeader; +import htsjdk.samtools.util.BlockCompressedOutputStream; import htsjdk.samtools.util.IOUtil; import htsjdk.samtools.util.Log; -import htsjdk.samtools.util.zip.DeflaterFactory; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.broadinstitute.hellbender.utils.LoggingUtils; @@ -56,6 +57,9 @@ public abstract class CommandLineProgram { public Boolean QUIET = false; private final String standardUsagePreamble = CommandLineParser.getStandardUsagePreamble(getClass()); + @Argument(fullName = "use_jdk_deflater", shortName = "jdk_deflater", doc = "Whether to use the JdkDeflater (as opposed to IntelDeflater)", common=true) + public boolean useJdkDeflater = false; + /** * Initialized in parseArgs. Subclasses may want to access this to do their * own validation, and then print usage using commandLineParser. @@ -128,6 +132,11 @@ public Object instanceMainPostParseArgs() { System.setProperty("java.io.tmpdir", f.getAbsolutePath()); // in loop so that last one takes effect } + //Set defaults (note: setting them here means they are not controllable by the user) + if (! useJdkDeflater) { + BlockCompressedOutputStream.setDefaultDeflaterFactory(new IntelDeflaterFactory()); + } + if (!QUIET) { System.err.println("[" + ZonedDateTime.now().format(DateTimeFormatter.ofLocalizedDateTime(FormatStyle.LONG)) + "] " + commandLine); @@ -145,8 +154,8 @@ public Object instanceMainPostParseArgs() { Defaults.allDefaults().entrySet().stream().forEach(e-> logger.info(Defaults.class.getSimpleName() + "." + e.getKey() + " : " + e.getValue()) ); -// TODO: remove comment when this is working in HaplotypeCaller -// logger.info("Deflater " + (DeflaterFactory.usingIntelDeflater()? "IntelDeflater": "JdkDeflater")); + final boolean usingIntelDeflater = (BlockCompressedOutputStream.getDefaultDeflaterFactory() instanceof IntelDeflaterFactory && ((IntelDeflaterFactory)BlockCompressedOutputStream.getDefaultDeflaterFactory()).usingIntelDeflater()); + logger.info("Deflater " + (usingIntelDeflater ? "IntelDeflater": "JdkDeflater")); } catch (final Exception e) { /* Unpossible! */ } }