diff --git a/build.gradle b/build.gradle index ec5ace94de..a05342a76a 100644 --- a/build.gradle +++ b/build.gradle @@ -16,6 +16,12 @@ plugins { repositories { mavenCentral() + + // Include the Broad artifactory for intel-gkl snapshots. This is temporary + // until intel-gkl is available on maven central. + maven { + url "https://artifactory.broadinstitute.org/artifactory/libs-snapshot/" + } } jacocoTestReport { @@ -41,6 +47,7 @@ dependencies { compile "org.apache.commons:commons-compress:1.4.1" compile "org.tukaani:xz:1.5" compile "gov.nih.nlm.ncbi:ngs-java:1.2.2" + compile "com.intel:intel-gkl:0.0.1-20160525.160915-2" testCompile "org.testng:testng:6.9.9" } @@ -65,12 +72,6 @@ jar { } } -processResources { - into "lib/jni/", { - from ("lib/jni/libIntelDeflater.so") - } -} - import org.gradle.internal.os.OperatingSystem; tasks.withType(Test) { @@ -83,7 +84,6 @@ tasks.withType(Test) { jvmArgs '-Djava.awt.headless=true' //this prevents awt from displaying a java icon while the tests are running - if (System.env.CI == "true") { //if running under a CI output less into the logs int count = 0 @@ -121,6 +121,7 @@ tasks.withType(Test) { test { description = "Runs the unit tests other than the SRA tests" + useTestNG { if( OperatingSystem.current().isUnix() ){ excludeGroups "slow", "broken", "sra", "intel" @@ -131,7 +132,7 @@ test { } task testIntelDeflater(type: Test){ - jvmArgs '-Dsamjdk.intel_deflater_so_path=lib/jni/libIntelDeflater.so' + jvmArgs '-Dsamjdk.try_use_intel_deflater=true' description "Run the Intel Deflater tests" useTestNG { diff --git a/lib/jni/libIntelDeflater.so b/lib/jni/libIntelDeflater.so deleted file mode 100755 index 5e8dd79870..0000000000 Binary files a/lib/jni/libIntelDeflater.so and /dev/null differ diff --git a/scripts/build_intel_deflater.sh b/scripts/build_intel_deflater.sh deleted file mode 100644 index f139cf0fb4..0000000000 --- a/scripts/build_intel_deflater.sh +++ /dev/null @@ -1,78 +0,0 @@ -#! /bin/bash -# -# The MIT License -# -# Copyright (c) 2013 The Broad Institute -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -# THE SOFTWARE. -# - -# Build libIntelDeflater.so, the JNI library that wraps Intel IPP compression library and igzip. -# Note that this is not built as part of standard release process. Rather, it is built manually and then -# copied to htsjdk/lib/jni. - -# Assumes OpenJDK exists at $OPENJDK. I used openjdk-7-fcs-src-b147-27_jun_2011.zip -# Assumes that Picard-public java sources have been compiled -# Assumes IPP8_CODE_SAMPLES_DIR points to Intel IPP sample code built with -fPIC -# Assumes IPP8_INSTALL_DIR points to composer_xe_2013_sp1 installation -# Assumes IGZIP_LIB points to the directory containing libigzip0c.a -source ${IPP8_INSTALL_DIR}/bin/ippvars.sh intel64 - -set -e - -if [ "$OPENJDK" = "" ] -then echo "ERROR: OPENJDK environment variable not defined." >&2 - exit 1 -fi - -if [ "$IPP8_CODE_SAMPLES_DIR" = "" ] -then echo "ERROR: IPP8_CODE_SAMPLES_DIR environment variable not defined." >&2 - exit 1 -fi - -if [ "$IPP8_INSTALL_DIR" = "" ] -then echo "ERROR: IPP8_INSTALL_DIR environment variable not defined." >&2 - exit 1 -fi -if [ "$IGZIP_LIB" = "" ] -then echo "ERROR: IGZIP_LIB environment variable not defined." >&2 - exit 1 -fi - -rootdir=$(dirname $(dirname $(dirname $0))) - - -builddir=$rootdir/lib_build -rm -rf $builddir -mkdir -p $builddir - -echo $rootdir -# Create JNI C header file -javah -jni -classpath $rootdir/classes -d $builddir htsjdk.samtools.util.zip.IntelDeflater - -# Compile source and create library. -gcc -I$builddir -I$rootdir/src/c/inteldeflater/ -I$JAVA_HOME/include/ -I$JAVA_HOME/include/linux/ -I$OPENJDK/jdk/src/share/native/common/ \ --I$OPENJDK/jdk/src/solaris/native/common/ -c -O3 -fPIC $rootdir/src/c/inteldeflater/IntelDeflater.c -gcc -z noexecstack -shared -o $builddir/libIntelDeflater.so IntelDeflater.o -L${IPP8_CODE_SAMPLES_DIR}/__cmake/data-compression.intel64.make.static.release/__lib/release \ --lzlib -lstdc++ -Wl,-Bstatic -L$IGZIP_LIB -ligzip0c -lbfp754 -ldecimal -liomp5 -liompstubs5 -lipgo -lippac -lippcc -lippch -lippcv \ --lippdc -lippdi -lippgen -lippi -lippj -lippm -lippr -lippsc -lippvc -lippvm -lirng -lmatmul -lpdbx \ --lpdbxinst -lsvml -lipps -limf -lirc -lirc_s -lippcore -Wl,-Bdynamic - - - diff --git a/src/main/c/inteldeflater/IntelDeflater.c b/src/main/c/inteldeflater/IntelDeflater.c deleted file mode 100644 index 65f5d1d9e6..0000000000 --- a/src/main/c/inteldeflater/IntelDeflater.c +++ /dev/null @@ -1,392 +0,0 @@ -/* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ - -/* - * Native method support for htsjdk.samtools.util.zip.IntelDeflater. - * This is copied from OpenJDK native support for java.util.zip.Deflater, and modified to support igzip. - */ - -#include -#include -#include -#include -#include -#include -#include "jlong.h" -#include "jni.h" -//#include "jni_util.h" - -#include "zlib.h" -#include "htsjdk_samtools_util_zip_IntelDeflater.h" -#include "igzip_lib.h" -#define DEF_MEM_LEVEL 8 -#define FAST_COMPRESSION 1 -#define IGZIP_TRUE 1 - -static jfieldID levelID; -static jfieldID strategyID; -static jfieldID setParamsID; -static jfieldID finishID; -static jfieldID finishedID; -static jfieldID bufID, offID, lenID; - -typedef struct { - z_stream zStream; - LZ_Stream2 lz2Stream; - int useIGZIP; -} Stream; - - -bool is_cpuid_ecx_bit_set(int eax, int bitidx) -{ - int ecx = 0, edx = 0, ebx = 0; - __asm__ ("cpuid" - :"=b" (ebx), - "=c" (ecx), - "=d" (edx) - :"a" (eax) - ); - return (((ecx >> bitidx)&1) == 1); -} - -bool is_sse42_supported() -{ -#ifdef __INTEL_COMPILER - return (_may_i_use_cpu_feature(_FEATURE_SSE4_2) > 0); -#else - // return __builtin_cpu_supports("sse4.2"); - return is_cpuid_ecx_bit_set(1, 20); -#endif -} -////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - -/** - * Throw a Java exception by name. Similar to SignalError. - */ -JNIEXPORT void JNICALL -JNU_ThrowByName(JNIEnv *env, const char *name, const char *msg) -{ - jclass cls = (*env)->FindClass(env, name); - - if (cls != 0) /* Otherwise an exception has already been thrown */ - (*env)->ThrowNew(env, cls, msg); -} - -/* JNU_Throw common exceptions */ - -JNIEXPORT void JNICALL -JNU_ThrowNullPointerException(JNIEnv *env, const char *msg) -{ - JNU_ThrowByName(env, "java/lang/NullPointerException", msg); -} - - -JNIEXPORT void JNICALL -JNU_ThrowOutOfMemoryError(JNIEnv *env, const char *msg) -{ - JNU_ThrowByName(env, "java/lang/OutOfMemoryError", msg); -} - -JNIEXPORT void JNICALL -JNU_ThrowIllegalArgumentException(JNIEnv *env, const char *msg) -{ - JNU_ThrowByName(env, "java/lang/IllegalArgumentException", msg); -} - -JNIEXPORT void JNICALL -JNU_ThrowInternalError(JNIEnv *env, const char *msg) -{ - JNU_ThrowByName(env, "java/lang/InternalError", msg); -} -///////////////////////////////////////////////////////////////////////////////////////////////////////////////// -JNIEXPORT void JNICALL -Java_htsjdk_samtools_util_zip_IntelDeflater_initIDs(JNIEnv *env, jclass cls) -{ - levelID = (*env)->GetFieldID(env, cls, "level", "I"); - strategyID = (*env)->GetFieldID(env, cls, "strategy", "I"); - setParamsID = (*env)->GetFieldID(env, cls, "setParams", "Z"); - finishID = (*env)->GetFieldID(env, cls, "finish", "Z"); - finishedID = (*env)->GetFieldID(env, cls, "finished", "Z"); - bufID = (*env)->GetFieldID(env, cls, "buf", "[B"); - offID = (*env)->GetFieldID(env, cls, "off", "I"); - lenID = (*env)->GetFieldID(env, cls, "len", "I"); - -} - -JNIEXPORT jlong JNICALL -Java_htsjdk_samtools_util_zip_IntelDeflater_init(JNIEnv *env, jclass cls, jint level, - jint strategy, jboolean nowrap) -{ - Stream *strm = calloc(1, sizeof(Stream)); - if (level == FAST_COMPRESSION && is_sse42_supported()) { //Use igzip - printf("Using igzip\n"); - strm->useIGZIP = IGZIP_TRUE; - if (strm == 0) { - JNU_ThrowOutOfMemoryError(env, 0); - return jlong_zero; - } else { - init_stream(&strm->lz2Stream); //CHECK RETURN VALUE - return ptr_to_jlong(strm); - } - - } else { - - if (strm == 0) { - JNU_ThrowOutOfMemoryError(env, 0); - return jlong_zero; - } else { - char *msg; - switch (deflateInit2(&strm->zStream, level, Z_DEFLATED, - nowrap ? -MAX_WBITS : MAX_WBITS, - DEF_MEM_LEVEL, strategy)) { - case Z_OK: - return ptr_to_jlong(&strm->zStream); - case Z_MEM_ERROR: - free(strm); - JNU_ThrowOutOfMemoryError(env, 0); - return jlong_zero; - case Z_STREAM_ERROR: - free(strm); - JNU_ThrowIllegalArgumentException(env, 0); - return jlong_zero; - default: - msg = strm->zStream.msg; - free(strm); - JNU_ThrowInternalError(env, msg); - return jlong_zero; - } - } - } -} - -JNIEXPORT void JNICALL -Java_htsjdk_samtools_util_zip_IntelDeflater_setDictionary(JNIEnv *env, jclass cls, jlong addr, - jarray b, jint off, jint len) -{ - Bytef *buf = (*env)->GetPrimitiveArrayCritical(env, b, 0); - int res; - if (buf == 0) {/* out of memory */ - return; - } - res = deflateSetDictionary(&((Stream *)jlong_to_ptr(addr))->zStream, buf + off, len); - (*env)->ReleasePrimitiveArrayCritical(env, b, buf, 0); - switch (res) { - case Z_OK: - break; - case Z_STREAM_ERROR: - JNU_ThrowIllegalArgumentException(env, 0); - break; - default: - JNU_ThrowInternalError(env, ((Stream *)jlong_to_ptr(addr))->zStream.msg); - break; - } -} - -JNIEXPORT jint JNICALL -Java_htsjdk_samtools_util_zip_IntelDeflater_deflateBytes(JNIEnv *env, jobject this, jlong addr, - jarray b, jint off, jint len, jint flush) -{ - jarray this_buf = (*env)->GetObjectField(env, this, bufID); - jint this_off = (*env)->GetIntField(env, this, offID); - jint this_len = (*env)->GetIntField(env, this, lenID); - jbyte *in_buf; - jbyte *out_buf; - Stream *strm = jlong_to_ptr(addr); - - //igzip only supports one compression level so setParamsID should not be set when using igzip - //igzip does not support flush - if (((Stream *)jlong_to_ptr(addr))->useIGZIP && (((*env)->GetBooleanField(env, this, setParamsID) && strm->lz2Stream.total_in != 0) || flush == 1)) { - JNU_ThrowInternalError(env, "igzip doesn't support this"); - } else if (((Stream *)jlong_to_ptr(addr))->useIGZIP) { - in_buf = (*env)->GetPrimitiveArrayCritical(env, this_buf, 0); - if (in_buf == NULL) { - // Throw OOME only when length is not zero - if (this_len != 0) { - JNU_ThrowOutOfMemoryError(env, 0); - } - return 0; - } - out_buf = (*env)->GetPrimitiveArrayCritical(env, b, 0); - if (out_buf == NULL) { - (*env)->ReleasePrimitiveArrayCritical(env, this_buf, in_buf, 0); - if (len != 0) { - JNU_ThrowOutOfMemoryError(env, 0); - } - return 0; - } - strm->lz2Stream.next_in = (Bytef *) (in_buf + this_off); - strm->lz2Stream.next_out = (Bytef *) (out_buf + off); - strm->lz2Stream.avail_in = this_len; - strm->lz2Stream.avail_out = len; - assert(strm->lz2Stream.avail_in != 0); - assert(strm->lz2Stream.avail_out != 0); - jboolean finish = (*env)->GetBooleanField(env, this, finishID); - if (finish) { - strm->lz2Stream.end_of_stream = 1; - } else { - strm->lz2Stream.end_of_stream = 0; - } - fast_lz(&strm->lz2Stream); - - (*env)->ReleasePrimitiveArrayCritical(env, b, out_buf, 0); - (*env)->ReleasePrimitiveArrayCritical(env, this_buf, in_buf, 0); - if (finish) { - (*env)->SetBooleanField(env, this, finishedID, JNI_TRUE); - } - this_off += this_len - strm->lz2Stream.avail_in; - (*env)->SetIntField(env, this, offID, this_off); - (*env)->SetIntField(env, this, lenID, strm->lz2Stream.avail_in); - return len - strm->lz2Stream.avail_out; - } else { - - int res; - if ((*env)->GetBooleanField(env, this, setParamsID)) { - int level = (*env)->GetIntField(env, this, levelID); - int strategy = (*env)->GetIntField(env, this, strategyID); - - in_buf = (*env)->GetPrimitiveArrayCritical(env, this_buf, 0); - if (in_buf == NULL) { - // Throw OOME only when length is not zero - if (this_len != 0) - JNU_ThrowOutOfMemoryError(env, 0); - return 0; - } - out_buf = (*env)->GetPrimitiveArrayCritical(env, b, 0); - if (out_buf == NULL) { - (*env)->ReleasePrimitiveArrayCritical(env, this_buf, in_buf, 0); - if (len != 0) - JNU_ThrowOutOfMemoryError(env, 0); - return 0; - } - - strm->zStream.next_in = (Bytef *) (in_buf + this_off); - strm->zStream.next_out = (Bytef *) (out_buf + off); - strm->zStream.avail_in = this_len; - strm->zStream.avail_out = len; - res = deflateParams(&strm->zStream, level, strategy); - (*env)->ReleasePrimitiveArrayCritical(env, b, out_buf, 0); - (*env)->ReleasePrimitiveArrayCritical(env, this_buf, in_buf, 0); - - switch (res) { - case Z_OK: - (*env)->SetBooleanField(env, this, setParamsID, JNI_FALSE); - this_off += this_len - strm->zStream.avail_in; - (*env)->SetIntField(env, this, offID, this_off); - (*env)->SetIntField(env, this, lenID, strm->zStream.avail_in); - return len - strm->zStream.avail_out; - case Z_BUF_ERROR: - (*env)->SetBooleanField(env, this, setParamsID, JNI_FALSE); - return 0; - default: - JNU_ThrowInternalError(env, strm->zStream.msg); - return 0; - } - } else { - jboolean finish = (*env)->GetBooleanField(env, this, finishID); - in_buf = (*env)->GetPrimitiveArrayCritical(env, this_buf, 0); - if (in_buf == NULL) { - if (this_len != 0) - JNU_ThrowOutOfMemoryError(env, 0); - return 0; - } - out_buf = (*env)->GetPrimitiveArrayCritical(env, b, 0); - if (out_buf == NULL) { - (*env)->ReleasePrimitiveArrayCritical(env, this_buf, in_buf, 0); - if (len != 0) - JNU_ThrowOutOfMemoryError(env, 0); - - return 0; - } - - strm->zStream.next_in = (Bytef *) (in_buf + this_off); - strm->zStream.next_out = (Bytef *) (out_buf + off); - strm->zStream.avail_in = this_len; - strm->zStream.avail_out = len; - res = deflate(&strm->zStream, finish ? Z_FINISH : flush); - (*env)->ReleasePrimitiveArrayCritical(env, b, out_buf, 0); - (*env)->ReleasePrimitiveArrayCritical(env, this_buf, in_buf, 0); - - switch (res) { - case Z_STREAM_END: - (*env)->SetBooleanField(env, this, finishedID, JNI_TRUE); - /* fall through */ - case Z_OK: - this_off += this_len - strm->zStream.avail_in; - (*env)->SetIntField(env, this, offID, this_off); - (*env)->SetIntField(env, this, lenID, strm->zStream.avail_in); - return len - strm->zStream.avail_out; - case Z_BUF_ERROR: - return 0; - default: - JNU_ThrowInternalError(env, strm->zStream.msg); - return 0; - } - } - } -} - -JNIEXPORT jint JNICALL -Java_htsjdk_samtools_util_zip_IntelDeflater_getAdler(JNIEnv *env, jclass cls, jlong addr) -{ - if (((Stream *)jlong_to_ptr(addr))->useIGZIP) - JNU_ThrowInternalError(env, "igzip doesn't support getAdler function"); - else - return ((Stream *)jlong_to_ptr(addr))->zStream.adler; -} - -JNIEXPORT jlong JNICALL -Java_htsjdk_samtools_util_zip_IntelDeflater_getBytesRead(JNIEnv *env, jclass cls, jlong addr) -{ - return ( ((Stream *)jlong_to_ptr(addr))->useIGZIP ? ((Stream *) jlong_to_ptr(addr))->lz2Stream.total_in : ((Stream *)jlong_to_ptr(addr))->zStream.total_in); -} - -JNIEXPORT jlong JNICALL -Java_htsjdk_samtools_util_zip_IntelDeflater_getBytesWritten(JNIEnv *env, jclass cls, jlong addr) -{ - return ( ((Stream *)jlong_to_ptr(addr))->useIGZIP ? ((Stream *) jlong_to_ptr(addr))->lz2Stream.total_out : ((Stream *)jlong_to_ptr(addr))->zStream.total_out); -} - -JNIEXPORT void JNICALL -Java_htsjdk_samtools_util_zip_IntelDeflater_reset(JNIEnv *env, jclass cls, jlong addr) -{ - if (((Stream *)jlong_to_ptr(addr))->useIGZIP) - init_stream(&(((Stream *)jlong_to_ptr(addr))->lz2Stream)); - else { - if (deflateReset(&(((Stream *)jlong_to_ptr(addr))->zStream)) != Z_OK) { - JNU_ThrowInternalError(env, 0); - } - } -} - -JNIEXPORT void JNICALL -Java_htsjdk_samtools_util_zip_IntelDeflater_end(JNIEnv *env, jclass cls, jlong addr) -{ - if (!((Stream *)jlong_to_ptr(addr))->useIGZIP) { - if (deflateEnd(&(((Stream *)jlong_to_ptr(addr))->zStream)) == Z_STREAM_ERROR) { - JNU_ThrowInternalError(env, 0); - } - } - free((Stream *)jlong_to_ptr(addr)); -} diff --git a/src/main/c/inteldeflater/README b/src/main/c/inteldeflater/README deleted file mode 100644 index 31170d4b71..0000000000 --- a/src/main/c/inteldeflater/README +++ /dev/null @@ -1,7 +0,0 @@ -- IntelDeflater.c implements JNI for the IntelDeflater in htsjdk.samtools.util.zip.IntelDeflater -- IntelDeflater uses Intel(R) Integrated Performance Primitives (Intel(R) IPP) Samples and igzip to accelerate BAM compression. -- Steps to build Intel Deflater using src/scripts/build_intel_deflater.sh: - - $OPENJDK should point to the OpenJDK directory - - $IPP8_INSTALL_DIR should point to the composer_xe_YEAR/ipp directory - - $IPP8_CODE_SAMPLES_DIR should point to ipp-samples.8.0.0.x directory - - $IGZIP_LIB should point to igzip_042/igzip directory, igzip should be built with ONLY_DEFLATE and GENOME_BAM defined. diff --git a/src/main/c/inteldeflater/igzip_lib.h b/src/main/c/inteldeflater/igzip_lib.h deleted file mode 100644 index a27b334cc5..0000000000 --- a/src/main/c/inteldeflater/igzip_lib.h +++ /dev/null @@ -1,50 +0,0 @@ -/********************************************************************** -The MIT License - -Copyright (c) 2014 Intel Corporation - - Permission is hereby granted, free of charge, to any person - obtaining a copy of this software and associated documentation - files (the "Software"), to deal in the Software without - restriction, including without limitation the rights to use, - copy, modify, merge, publish, distribute, sublicense, and/or - sell copies of the Software, and to permit persons to whom the - Software is furnished to do so, subject to the following - conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the - Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY - KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE - WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR - PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -**********************************************************************/ -#include "internal_state_size.h" -#include "types.h" - -typedef struct { - UINT8 opaque[INTERNAL_STATE_SIZE]; -} LZ_State2; - -typedef struct { - UINT8 *next_in; // Next input byte - UINT32 avail_in; // number of bytes available at next_in - UINT32 total_in; // total number of bytes read so far - - UINT8 *next_out; // Next output byte - UINT32 avail_out; // number of bytes available at next_out - UINT32 total_out; // total number of bytes written so far - UINT32 end_of_stream; // non-zero if this is the last input buffer - - LZ_State2 internal_state; -} LZ_Stream2; - - -void init_stream(LZ_Stream2 *stream); -void fast_lz(LZ_Stream2 *stream); diff --git a/src/main/c/inteldeflater/internal_state_size.h b/src/main/c/inteldeflater/internal_state_size.h deleted file mode 100644 index 1823a33e00..0000000000 --- a/src/main/c/inteldeflater/internal_state_size.h +++ /dev/null @@ -1,32 +0,0 @@ -/********************************************************************** -The MIT License - -Copyright (c) 2014 Intel Corporation - - Permission is hereby granted, free of charge, to any person - obtaining a copy of this software and associated documentation - files (the "Software"), to deal in the Software without - restriction, including without limitation the rights to use, - copy, modify, merge, publish, distribute, sublicense, and/or - sell copies of the Software, and to permit persons to whom the - Software is furnished to do so, subject to the following - conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the - Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY - KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE - WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR - PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -**********************************************************************/ -//// for 8K -#define INTERNAL_STATE_SIZE 82368+16 - -// for 32K -//#define INTERNAL_STATE_SIZE 328128+16 diff --git a/src/main/c/inteldeflater/types.h b/src/main/c/inteldeflater/types.h deleted file mode 100644 index 83f7cc2667..0000000000 --- a/src/main/c/inteldeflater/types.h +++ /dev/null @@ -1,46 +0,0 @@ -/********************************************************************** -The MIT License - -Copyright (c) 2014 Intel Corporation - - Permission is hereby granted, free of charge, to any person - obtaining a copy of this software and associated documentation - files (the "Software"), to deal in the Software without - restriction, including without limitation the rights to use, - copy, modify, merge, publish, distribute, sublicense, and/or - sell copies of the Software, and to permit persons to whom the - Software is furnished to do so, subject to the following - conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the - Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY - KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE - WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR - PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR - COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -**********************************************************************/ - -#ifndef __TYPES_H -#define __TYPES_H - -#include // For standard integer types - -typedef struct { - uint64_t low; - uint64_t high; -} uint128_t; - -typedef int64_t INT64; -typedef uint64_t UINT64; -typedef uint32_t UINT32; -typedef uint16_t UINT16; -typedef uint8_t UINT8; -typedef uint128_t UINT128; - -#endif diff --git a/src/main/java/htsjdk/samtools/Defaults.java b/src/main/java/htsjdk/samtools/Defaults.java index e6a2e1309e..e25d09aca4 100644 --- a/src/main/java/htsjdk/samtools/Defaults.java +++ b/src/main/java/htsjdk/samtools/Defaults.java @@ -57,12 +57,6 @@ public class Defaults { /** Should BlockCompressedOutputStream attempt to load libIntelDeflater? */ public static final boolean TRY_USE_INTEL_DEFLATER; - /** - * Path to libIntelDeflater.so. If this is not set, the library is looked for in the directory - * where the executable jar lives. - */ - public static final String INTEL_DEFLATER_SHARED_LIBRARY_PATH; - /** * The reference FASTA file. If this is not set, the file is null. This file may be required for reading * writing SAM files (ex. CRAM). @@ -104,8 +98,7 @@ public class Defaults { } COMPRESSION_LEVEL = getIntProperty("compression_level", 5); BUFFER_SIZE = getIntProperty("buffer_size", 1024 * 128); - TRY_USE_INTEL_DEFLATER = getBooleanProperty("try_use_intel_deflater", true); - INTEL_DEFLATER_SHARED_LIBRARY_PATH = getStringProperty("intel_deflater_so_path", null); + TRY_USE_INTEL_DEFLATER = getBooleanProperty("try_use_intel_deflater", false); if (BUFFER_SIZE == 0) { NON_ZERO_BUFFER_SIZE = 1024 * 128; } else { @@ -132,7 +125,6 @@ public static SortedMap allDefaults(){ result.put("COMPRESSION_LEVEL", COMPRESSION_LEVEL); result.put("BUFFER_SIZE", BUFFER_SIZE); result.put("TRY_USE_INTEL_DEFLATER", TRY_USE_INTEL_DEFLATER); - result.put("INTEL_DEFLATER_SHARED_LIBRARY_PATH", INTEL_DEFLATER_SHARED_LIBRARY_PATH); result.put("NON_ZERO_BUFFER_SIZE", NON_ZERO_BUFFER_SIZE); result.put("REFERENCE_FASTA", REFERENCE_FASTA); result.put("USE_CRAM_REF_DOWNLOAD", USE_CRAM_REF_DOWNLOAD); diff --git a/src/main/java/htsjdk/samtools/example/PrintReadsExample.java b/src/main/java/htsjdk/samtools/example/PrintReadsExample.java index b8b4106fc1..5b8f6b0067 100755 --- a/src/main/java/htsjdk/samtools/example/PrintReadsExample.java +++ b/src/main/java/htsjdk/samtools/example/PrintReadsExample.java @@ -96,7 +96,7 @@ private static void printConfigurationInfo() throws IOException { " on " + System.getProperty("os.name") + ' ' + System.getProperty("os.version") + ' ' + System.getProperty("os.arch") + "; " + System.getProperty("java.vm.name") + ' ' + System.getProperty("java.runtime.version") + - ' ' + (DeflaterFactory.usingIntelDeflater() ? "IntelDeflater" : "JdkDeflater")); + ' ' + (DeflaterFactory.usingIntelDeflater() ? "IntelDeflater loaded and available for Level 1 compression" : "Using JdkDeflater")); final List list = Defaults.allDefaults().entrySet().stream().map(e -> e.getKey() + ':' + e.getValue()).collect(Collectors.toList()); log.info(String.join(" ", list)); diff --git a/src/main/java/htsjdk/samtools/util/BlockCompressedOutputStream.java b/src/main/java/htsjdk/samtools/util/BlockCompressedOutputStream.java index 138a9754cf..3dde3e8e08 100644 --- a/src/main/java/htsjdk/samtools/util/BlockCompressedOutputStream.java +++ b/src/main/java/htsjdk/samtools/util/BlockCompressedOutputStream.java @@ -50,6 +50,9 @@ public class BlockCompressedOutputStream extends OutputStream implements LocationAware { + + private static final Log log = Log.getInstance(BlockCompressedOutputStream.class); + private static int defaultCompressionLevel = BlockCompressedStreamConstants.DEFAULT_COMPRESSION_LEVEL; /** @@ -127,6 +130,7 @@ public BlockCompressedOutputStream(final File file, final int compressionLevel) this.file = file; codec = new BinaryCodec(file, true); deflater = DeflaterFactory.makeDeflater(compressionLevel, true); + log.debug("Using deflater: " + deflater.getClass().getSimpleName()); } /** @@ -144,6 +148,7 @@ public BlockCompressedOutputStream(final OutputStream os, final File file, final codec.setOutputFileName(file.getAbsolutePath()); } deflater = DeflaterFactory.makeDeflater(compressionLevel, true); + log.debug("Using deflater: " + deflater.getClass().getSimpleName()); } /** diff --git a/src/main/java/htsjdk/samtools/util/zip/DeflaterFactory.java b/src/main/java/htsjdk/samtools/util/zip/DeflaterFactory.java index 64bb8aab1b..368a9fd391 100644 --- a/src/main/java/htsjdk/samtools/util/zip/DeflaterFactory.java +++ b/src/main/java/htsjdk/samtools/util/zip/DeflaterFactory.java @@ -23,49 +23,66 @@ */ package htsjdk.samtools.util.zip; +import com.intel.gkl.compression.IntelDeflater; import htsjdk.samtools.Defaults; import htsjdk.samtools.SAMException; import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; import java.util.zip.Deflater; /** - * Create zlib-based Deflater if JNI library and other require libraries are available, otherwise create standard - * JDK Deflater. - * Java 7 has its own Deflater implementation (libzip.so). This is almost as fast as a zlib-based Deflater, so in general - * there isn't a compelling reason to use zlib. However, Intel has created a hardware-assisted zlib implementation - * as part of their IPP (Integrated Performance Primitives) package that can run significantly faster on some Intel - * hardware. We have seen compression times reduced by 13% to 33% depending on particular hardware, and hope that - * newer Intel processors will be even better. + * Create a hardware-accelerated Intel deflater if the required library is available on the classpath and + * {@link Defaults#TRY_USE_INTEL_DEFLATER} is true, otherwise create a standard JDK deflater. * - * Note that this class will no longer be necessary once Java 8 is required, because JDK 8 will use zlib instead - * of libzip implementation. + * The Intel deflater has been shown to significantly (by ~30%) outperform the standard JDK deflater + * at compression level 1, and to outperform the JDK deflater by smaller but still significant margins + * at higher compression levels. + * + * We use reflection to instantiate the IntelDeflater so that DeflaterFactory can be loaded + * and used at runtime even when the IntelDeflater is not on the classpath. */ public class DeflaterFactory { - private static Constructor intelDeflaterConstructor; + public static final String INTEL_DEFLATER_CLASS_NAME = "com.intel.gkl.compression.IntelDeflater"; + + private static final boolean usingIntelDeflater; + private static Constructor intelDeflaterConstructor; static { + boolean intelDeflaterLibrarySuccessfullyLoaded = false; + try { if (Defaults.TRY_USE_INTEL_DEFLATER) { - final Class clazz = (Class) Class.forName("htsjdk.samtools.util.zip.IntelDeflater"); + @SuppressWarnings("unchecked") + final Class clazz = (Class)Class.forName(INTEL_DEFLATER_CLASS_NAME); + + // Get the constructor we'll use to create new instances of IntelDeflater in makeDeflater() intelDeflaterConstructor = clazz.getConstructor(Integer.TYPE, Boolean.TYPE); + + // We also need to call load() on an IntelDeflater instance to actually load the native library + // (.so or .dylib) from a resource on the classpath. This should only be done once at startup. + final Constructor zeroArgIntelDeflaterConstructor = clazz.getConstructor(); + intelDeflaterLibrarySuccessfullyLoaded = zeroArgIntelDeflaterConstructor != null && + zeroArgIntelDeflaterConstructor.newInstance().load(); } - } catch (ClassNotFoundException e) { - intelDeflaterConstructor = null; - } catch (NoSuchMethodException e) { - intelDeflaterConstructor = null; - } catch (UnsatisfiedLinkError e) { + } + catch ( ClassNotFoundException | NoSuchMethodException | UnsatisfiedLinkError | + IllegalAccessException | InstantiationException | InvocationTargetException e ) { intelDeflaterConstructor = null; } + + usingIntelDeflater = intelDeflaterConstructor != null && intelDeflaterLibrarySuccessfullyLoaded; } public static Deflater makeDeflater(final int compressionLevel, final boolean nowrap) { - if (intelDeflaterConstructor != null) { + // The Intel Deflater currently only supports compression level 1 + if ( usingIntelDeflater() && compressionLevel == 1 ) { try { return intelDeflaterConstructor.newInstance(compressionLevel, nowrap); - } catch (Exception e) { - throw new SAMException("Exception constructing IntelDeflater", e); + } + catch ( IllegalAccessException | InstantiationException | InvocationTargetException e ) { + throw new SAMException("Error constructing IntelDeflater", e); } } else { return new Deflater(compressionLevel, nowrap); @@ -73,6 +90,6 @@ public static Deflater makeDeflater(final int compressionLevel, final boolean no } public static boolean usingIntelDeflater() { - return intelDeflaterConstructor != null; + return usingIntelDeflater; } } diff --git a/src/main/java/htsjdk/samtools/util/zip/IntelDeflater.java b/src/main/java/htsjdk/samtools/util/zip/IntelDeflater.java deleted file mode 100644 index 30cb698ffb..0000000000 --- a/src/main/java/htsjdk/samtools/util/zip/IntelDeflater.java +++ /dev/null @@ -1,567 +0,0 @@ -/* - * Copyright (c) 1996, 2010, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package htsjdk.samtools.util.zip; - -import htsjdk.samtools.Defaults; - -import java.io.File; -import java.net.URL; -import java.util.zip.Deflater; - -/** - * This is a copy of java.util.zip.Deflater from OpenJDK 7, with the following changes: - * - package and class name changed - * - static block to load libIntelDeflater library - * - extends java.util.zip.Deflater so that IntelDeflater object can be used as regular Deflater object. - * Note however that all methods of Deflater are overridden. - * - * The shared library is found via one of the following mechanisms: - * 1. if samjdk.intel_deflater_so_path system property is set, this is assumed to be the path of libIntelDeflater.so - * 2. If system property is not set, directory where the jarfile that this class came from lives is tried as location - * of libIntelDeflater.so - * 3. If either of the above fails to find the library, regular LD_LIBRARY_PATH is used to find the library. - * 4. If that doesn't work, class fails to load and code falls back to regular Java Deflater class. - * - * - * The rest of this document is copied verbatim from the original OpenJDK file. - * - * This class provides support for general purpose compression using the - * popular ZLIB compression library. The ZLIB compression library was - * initially developed as part of the PNG graphics standard and is not - * protected by patents. It is fully described in the specifications at - * the java.util.zip - * package description. - * - *

The following code fragment demonstrates a trivial compression - * and decompression of a string using IntelDeflater and - * Inflater. - * - *

- * try {
- *     // Encode a String into bytes
- *     String inputString = "blahblahblah";
- *     byte[] input = inputString.getBytes("UTF-8");
- *
- *     // Compress the bytes
- *     byte[] output = new byte[100];
- *     IntelDeflater compresser = new IntelDeflater();
- *     compresser.setInput(input);
- *     compresser.finish();
- *     int compressedDataLength = compresser.deflate(output);
- *     compresser.end();
- *
- *     // Decompress the bytes
- *     Inflater decompresser = new Inflater();
- *     decompresser.setInput(output, 0, compressedDataLength);
- *     byte[] result = new byte[100];
- *     int resultLength = decompresser.inflate(result);
- *     decompresser.end();
- *
- *     // Decode the bytes into a String
- *     String outputString = new String(result, 0, resultLength, "UTF-8");
- * } catch(java.io.UnsupportedEncodingException ex) {
- *     // handle
- * } catch (java.util.zip.DataFormatException ex) {
- *     // handle
- * }
- * 
- * - * @see java.util.zip.Inflater - * @author David Connelly - */ -public -class IntelDeflater extends Deflater { - - private final ZStreamRef zsRef; - private byte[] buf = new byte[0]; - private int off, len; - private int level, strategy; - private boolean setParams; - private boolean finish, finished; - - /** - * Compression flush mode used to achieve best compression result. - * - * @see IntelDeflater#deflate(byte[], int, int, int) - * @since 1.7 - */ - public static final int NO_FLUSH = 0; - - /** - * Compression flush mode used to flush out all pending output; may - * degrade compression for some compression algorithms. - * - * @see IntelDeflater#deflate(byte[], int, int, int) - * @since 1.7 - */ - public static final int SYNC_FLUSH = 2; - - /** - * Compression flush mode used to flush out all pending output and - * reset the deflater. Using this mode too often can seriously degrade - * compression. - * - * @see IntelDeflater#deflate(byte[], int, int, int) - * @since 1.7 - */ - public static final int FULL_FLUSH = 3; - - static { - try { - final File sharedLibrary; - if (Defaults.INTEL_DEFLATER_SHARED_LIBRARY_PATH != null) { - // Load via path set by -Dsamjdk.intel_deflater_so_path= - sharedLibrary = new File(Defaults.INTEL_DEFLATER_SHARED_LIBRARY_PATH); - } else { - // Look in directory containing this class for the library - URL jarUrl = IntelDeflater.class.getProtectionDomain().getCodeSource().getLocation(); - sharedLibrary = new File(new File(jarUrl.getPath()).getParentFile(), "libIntelDeflater.so"); - } - System.load(sharedLibrary.getAbsolutePath()); - } catch (Throwable e) { - // Possible exceptions: - // System.load: UnsatisfiedLinkError - // getProtectionDomain: SecurityException - // NullPointerException due to getCodeSource returning null - - // Try to find via LD_LIBRARY_PATH - System.loadLibrary("IntelDeflater"); - } - initIDs(); - } - - /** - * Creates a new compressor using the specified compression level. - * If 'nowrap' is true then the ZLIB header and checksum fields will - * not be used in order to support the compression format used in - * both GZIP and PKZIP. - * @param level the compression level (0-9) - * @param nowrap if true then use GZIP compatible compression - */ - public IntelDeflater(int level, boolean nowrap) { - this.level = level; - this.strategy = DEFAULT_STRATEGY; - this.zsRef = new ZStreamRef(init(level, DEFAULT_STRATEGY, nowrap)); - } - - /** - * Creates a new compressor using the specified compression level. - * Compressed data will be generated in ZLIB format. - * @param level the compression level (0-9) - */ - public IntelDeflater(int level) { - this(level, false); - } - - /** - * Creates a new compressor with the default compression level. - * Compressed data will be generated in ZLIB format. - */ - public IntelDeflater() { - this(DEFAULT_COMPRESSION, false); - } - - /** - * Sets input data for compression. This should be called whenever - * needsInput() returns true indicating that more input data is required. - * @param b the input data bytes - * @param off the start offset of the data - * @param len the length of the data - * @see IntelDeflater#needsInput - */ - @Override - public void setInput(byte[] b, int off, int len) { - if (b== null) { - throw new NullPointerException(); - } - if (off < 0 || len < 0 || off > b.length - len) { - throw new ArrayIndexOutOfBoundsException(); - } - synchronized (zsRef) { - this.buf = b; - this.off = off; - this.len = len; - } - } - - /** - * Sets input data for compression. This should be called whenever - * needsInput() returns true indicating that more input data is required. - * @param b the input data bytes - * @see IntelDeflater#needsInput - */ - @Override - public void setInput(byte[] b) { - setInput(b, 0, b.length); - } - - /** - * Sets preset dictionary for compression. A preset dictionary is used - * when the history buffer can be predetermined. When the data is later - * uncompressed with Inflater.inflate(), Inflater.getAdler() can be called - * in order to get the Adler-32 value of the dictionary required for - * decompression. - * @param b the dictionary data bytes - * @param off the start offset of the data - * @param len the length of the data - * @see java.util.zip.Inflater#inflate - * @see java.util.zip.Inflater#getAdler - */ - @Override - public void setDictionary(byte[] b, int off, int len) { - if (b == null) { - throw new NullPointerException(); - } - if (off < 0 || len < 0 || off > b.length - len) { - throw new ArrayIndexOutOfBoundsException(); - } - synchronized (zsRef) { - ensureOpen(); - setDictionary(zsRef.address(), b, off, len); - } - } - - /** - * Sets preset dictionary for compression. A preset dictionary is used - * when the history buffer can be predetermined. When the data is later - * uncompressed with Inflater.inflate(), Inflater.getAdler() can be called - * in order to get the Adler-32 value of the dictionary required for - * decompression. - * @param b the dictionary data bytes - * @see java.util.zip.Inflater#inflate - * @see java.util.zip.Inflater#getAdler - */ - @Override - public void setDictionary(byte[] b) { - setDictionary(b, 0, b.length); - } - - /** - * Sets the compression strategy to the specified value. - * @param strategy the new compression strategy - * @exception IllegalArgumentException if the compression strategy is - * invalid - */ - @Override - public void setStrategy(int strategy) { - switch (strategy) { - case DEFAULT_STRATEGY: - case FILTERED: - case HUFFMAN_ONLY: - break; - default: - throw new IllegalArgumentException(); - } - synchronized (zsRef) { - if (this.strategy != strategy) { - this.strategy = strategy; - setParams = true; - } - } - } - - /** - * Sets the current compression level to the specified value. - * @param level the new compression level (0-9) - * @exception IllegalArgumentException if the compression level is invalid - */ - @Override - public void setLevel(int level) { - if ((level < 0 || level > 9) && level != DEFAULT_COMPRESSION) { - throw new IllegalArgumentException("invalid compression level"); - } - synchronized (zsRef) { - if (this.level != level) { - this.level = level; - setParams = true; - } - } - } - - /** - * Returns true if the input data buffer is empty and setInput() - * should be called in order to provide more input. - * @return true if the input data buffer is empty and setInput() - * should be called in order to provide more input - */ - @Override - public boolean needsInput() { - return len <= 0; - } - - /** - * When called, indicates that compression should end with the current - * contents of the input buffer. - */ - @Override - public void finish() { - synchronized (zsRef) { - finish = true; - } - } - - /** - * Returns true if the end of the compressed data output stream has - * been reached. - * @return true if the end of the compressed data output stream has - * been reached - */ - @Override - public boolean finished() { - synchronized (zsRef) { - return finished; - } - } - - /** - * Compresses the input data and fills specified buffer with compressed - * data. Returns actual number of bytes of compressed data. A return value - * of 0 indicates that {@link #needsInput() needsInput} should be called - * in order to determine if more input data is required. - * - *

This method uses {@link #NO_FLUSH} as its compression flush mode. - * An invocation of this method of the form {@code deflater.deflate(b, off, len)} - * yields the same result as the invocation of - * {@code deflater.deflate(b, off, len, IntelDeflater.NO_FLUSH)}. - * - * @param b the buffer for the compressed data - * @param off the start offset of the data - * @param len the maximum number of bytes of compressed data - * @return the actual number of bytes of compressed data written to the - * output buffer - */ - @Override - public int deflate(byte[] b, int off, int len) { - return deflate(b, off, len, NO_FLUSH); - } - - /** - * Compresses the input data and fills specified buffer with compressed - * data. Returns actual number of bytes of compressed data. A return value - * of 0 indicates that {@link #needsInput() needsInput} should be called - * in order to determine if more input data is required. - * - *

This method uses {@link #NO_FLUSH} as its compression flush mode. - * An invocation of this method of the form {@code deflater.deflate(b)} - * yields the same result as the invocation of - * {@code deflater.deflate(b, 0, b.length, IntelDeflater.NO_FLUSH)}. - * - * @param b the buffer for the compressed data - * @return the actual number of bytes of compressed data written to the - * output buffer - */ - @Override - public int deflate(byte[] b) { - return deflate(b, 0, b.length, NO_FLUSH); - } - - /** - * Compresses the input data and fills the specified buffer with compressed - * data. Returns actual number of bytes of data compressed. - * - *

Compression flush mode is one of the following three modes: - * - *

    - *
  • {@link #NO_FLUSH}: allows the deflater to decide how much data - * to accumulate, before producing output, in order to achieve the best - * compression (should be used in normal use scenario). A return value - * of 0 in this flush mode indicates that {@link #needsInput()} should - * be called in order to determine if more input data is required. - * - *
  • {@link #SYNC_FLUSH}: all pending output in the deflater is flushed, - * to the specified output buffer, so that an inflater that works on - * compressed data can get all input data available so far (In particular - * the {@link #needsInput()} returns {@code true} after this invocation - * if enough output space is provided). Flushing with {@link #SYNC_FLUSH} - * may degrade compression for some compression algorithms and so it - * should be used only when necessary. - * - *
  • {@link #FULL_FLUSH}: all pending output is flushed out as with - * {@link #SYNC_FLUSH}. The compression state is reset so that the inflater - * that works on the compressed output data can restart from this point - * if previous compressed data has been damaged or if random access is - * desired. Using {@link #FULL_FLUSH} too often can seriously degrade - * compression. - *
- * - *

In the case of {@link #FULL_FLUSH} or {@link #SYNC_FLUSH}, if - * the return value is {@code len}, the space available in output - * buffer {@code b}, this method should be invoked again with the same - * {@code flush} parameter and more output space. - * - * @param b the buffer for the compressed data - * @param off the start offset of the data - * @param len the maximum number of bytes of compressed data - * @param flush the compression flush mode - * @return the actual number of bytes of compressed data written to - * the output buffer - * - * @throws IllegalArgumentException if the flush mode is invalid - * @since 1.7 - */ - public int deflate(byte[] b, int off, int len, int flush) { - //System.out.println("Inside IntelDeflater\n"); - if (b == null) { - throw new NullPointerException(); - } - if (off < 0 || len < 0 || off > b.length - len) { - throw new ArrayIndexOutOfBoundsException(); - } - synchronized (zsRef) { - ensureOpen(); - if (flush == NO_FLUSH || flush == SYNC_FLUSH || - flush == FULL_FLUSH) - return deflateBytes(zsRef.address(), b, off, len, flush); - throw new IllegalArgumentException(); - } - } - - /** - * Returns the ADLER-32 value of the uncompressed data. - * @return the ADLER-32 value of the uncompressed data - */ - @Override - public int getAdler() { - synchronized (zsRef) { - ensureOpen(); - return getAdler(zsRef.address()); - } - } - - /** - * Returns the total number of uncompressed bytes input so far. - * - *

Since the number of bytes may be greater than - * Integer.MAX_VALUE, the {@link #getBytesRead()} method is now - * the preferred means of obtaining this information.

- * - * @return the total number of uncompressed bytes input so far - */ - @Override - public int getTotalIn() { - return (int) getBytesRead(); - } - - /** - * Returns the total number of uncompressed bytes input so far.

- * - * @return the total (non-negative) number of uncompressed bytes input so far - * @since 1.5 - */ - @Override - public long getBytesRead() { - synchronized (zsRef) { - ensureOpen(); - return getBytesRead(zsRef.address()); - } - } - - /** - * Returns the total number of compressed bytes output so far. - * - *

Since the number of bytes may be greater than - * Integer.MAX_VALUE, the {@link #getBytesWritten()} method is now - * the preferred means of obtaining this information.

- * - * @return the total number of compressed bytes output so far - */ - @Override - public int getTotalOut() { - return (int) getBytesWritten(); - } - - /** - * Returns the total number of compressed bytes output so far.

- * - * @return the total (non-negative) number of compressed bytes output so far - * @since 1.5 - */ - @Override - public long getBytesWritten() { - synchronized (zsRef) { - ensureOpen(); - return getBytesWritten(zsRef.address()); - } - } - - /** - * Resets deflater so that a new set of input data can be processed. - * Keeps current compression level and strategy settings. - */ - @Override - public void reset() { - synchronized (zsRef) { - ensureOpen(); - reset(zsRef.address()); - finish = false; - finished = false; - off = len = 0; - } - } - - /** - * Closes the compressor and discards any unprocessed input. - * This method should be called when the compressor is no longer - * being used, but will also be called automatically by the - * finalize() method. Once this method is called, the behavior - * of the IntelDeflater object is undefined. - */ - @Override - public void end() { - synchronized (zsRef) { - long addr = zsRef.address(); - zsRef.clear(); - if (addr != 0) { - end(addr); - buf = null; - } - } - } - - /** - * Closes the compressor when garbage is collected. - */ - protected void finalize() { - end(); - } - - private void ensureOpen() { - assert Thread.holdsLock(zsRef); - if (zsRef.address() == 0) - throw new NullPointerException("IntelDeflater has been closed"); - } - - private static native void initIDs(); - private native static long init(int level, int strategy, boolean nowrap); - private native static void setDictionary(long addr, byte[] b, int off, int len); - private native int deflateBytes(long addr, byte[] b, int off, int len, - int flush); - private native static int getAdler(long addr); - private native static long getBytesRead(long addr); - private native static long getBytesWritten(long addr); - private native static void reset(long addr); - private native static void end(long addr); -} - diff --git a/src/main/java/htsjdk/samtools/util/zip/ZStreamRef.java b/src/main/java/htsjdk/samtools/util/zip/ZStreamRef.java deleted file mode 100644 index 8fa5f78e55..0000000000 --- a/src/main/java/htsjdk/samtools/util/zip/ZStreamRef.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (c) 2009, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. Oracle designates this - * particular file as subject to the "Classpath" exception as provided - * by Oracle in the LICENSE file that accompanied this code. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package htsjdk.samtools.util.zip; - -/** - * Copied from OpenJDK 7. Only change is package name, because IntelDeflater needs this and - * the original class is private to java.util.zip package. - * - * A reference to the native zlib's z_stream structure. - */ - -class ZStreamRef { - - private long address; - ZStreamRef (long address) { - this.address = address; - } - - long address() { - return address; - } - - void clear() { - address = 0; - } -} diff --git a/src/main/java/htsjdk/variant/example/PrintVariantsExample.java b/src/main/java/htsjdk/variant/example/PrintVariantsExample.java index 997f0ee6b7..7810ad731c 100755 --- a/src/main/java/htsjdk/variant/example/PrintVariantsExample.java +++ b/src/main/java/htsjdk/variant/example/PrintVariantsExample.java @@ -98,7 +98,7 @@ private static void printConfigurationInfo() throws IOException { " on " + System.getProperty("os.name") + ' ' + System.getProperty("os.version") + ' ' + System.getProperty("os.arch") + "; " + System.getProperty("java.vm.name") + ' ' + System.getProperty("java.runtime.version") + - ' ' + (DeflaterFactory.usingIntelDeflater() ? "IntelDeflater" : "JdkDeflater")); + ' ' + (DeflaterFactory.usingIntelDeflater() ? "IntelDeflater loaded and available for Level 1 compression" : "Using JdkDeflater")); log.info(Defaults.allDefaults().entrySet().stream().map(e -> e.getKey() + ':' + e.getValue()).collect(Collectors.joining(" "))); } diff --git a/src/test/java/htsjdk/samtools/util/IntelDeflaterTest.java b/src/test/java/htsjdk/samtools/util/IntelDeflaterTest.java index 00fb0ba4b0..747597e186 100755 --- a/src/test/java/htsjdk/samtools/util/IntelDeflaterTest.java +++ b/src/test/java/htsjdk/samtools/util/IntelDeflaterTest.java @@ -72,7 +72,7 @@ Iterator TestIntelDeflaterIsLoadedData(){ return retVal.iterator(); } - @Test(dataProvider = "TestIntelDeflaterIsLoadedData", groups={"unix", "intel"},expectedExceptions = IllegalAccessError.class) + @Test(dataProvider = "TestIntelDeflaterIsLoadedData", groups={"intel"}, expectedExceptions = IllegalAccessError.class) public void TestIntelDeflatorIsLoaded(final File inputFile, final Boolean eagerlyDecode,final Integer compressionLevel) throws IOException,IllegalAccessError { Log log = Log.getInstance(IntelDeflaterTest.class); Log.setGlobalLogLevel(Log.LogLevel.INFO);