From 9c1ce50504ae679d33048330249c144e600eab4c Mon Sep 17 00:00:00 2001
From: Louis Bergelson
Date: Sat, 16 Feb 2019 14:46:43 -0500
Subject: [PATCH 1/3] moving HttpUtilsTest to externalApi test task
* HttpUtilsTest is flakey because it connects to remote web servers.
Moved it to the testExternalAPIs test task
* changed a broken link so that the tests pass again
---
build.gradle | 4 +++-
src/test/java/htsjdk/samtools/util/HttpUtilsTest.java | 3 ++-
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/build.gradle b/build.gradle
index 6b272c171b..d525875540 100644
--- a/build.gradle
+++ b/build.gradle
@@ -109,6 +109,7 @@ test {
exclude "broken"
exclude "defaultReference"
exclude "ftp"
+ exclude "http"
exclude "sra"
exclude "ena"
@@ -127,11 +128,12 @@ task testFTP(type: Test) {
}
task testExternalApis(type: Test) {
- description = "Run the SRA and ENA tests (tests that interact with external APIs)"
+ description = "Run the SRA, ENA, and HTTP tests (tests that interact with external APIs)"
jvmArgs += '-Dsamjdk.sra_libraries_download=true'
tags {
include "sra"
+ include "http"
include "ena"
exclude "slow"
exclude "broken"
diff --git a/src/test/java/htsjdk/samtools/util/HttpUtilsTest.java b/src/test/java/htsjdk/samtools/util/HttpUtilsTest.java
index b759445802..827771f867 100644
--- a/src/test/java/htsjdk/samtools/util/HttpUtilsTest.java
+++ b/src/test/java/htsjdk/samtools/util/HttpUtilsTest.java
@@ -8,12 +8,13 @@
import htsjdk.HtsjdkTest;
+@Test(groups = "http")
public class HttpUtilsTest extends HtsjdkTest {
@DataProvider(name = "existing_urls")
public Object[][] testExistingURLsData() {
return new Object[][]{
{"http://broadinstitute.github.io/picard/testdata/index_test.bam"},
- {"http://ftp.1000genomes.ebi.ac.uk/vol1/ftp/current.tree"}
+ {"http://ftp.1000genomes.ebi.ac.uk/vol1/ftp/README_using_1000genomes_cram.md"}
};
}
From 50da9cd8b251ddf05dde7e9d34ee17b1ae6e1c8f Mon Sep 17 00:00:00 2001
From: Louis Bergelson
Date: Fri, 15 Feb 2019 17:42:49 -0500
Subject: [PATCH 2/3] Changes to FastaReferenceWriter
* make sure the OutputStreams are always buffered
* change close behavior to be more likely to close all output streams in
the event of an early shutdown
* loosen restriction from BufferedWriter to Writer in
SAMSequenceDictionaryCodec and SAMTextHeaderCodec
---
.../samtools/SAMSequenceDictionaryCodec.java | 3 ++-
.../htsjdk/samtools/SAMTextHeaderCodec.java | 4 ++--
.../reference/FastaReferenceWriter.java | 18 ++++++++++--------
.../reference/FastaReferenceWriterBuilder.java | 7 ++++---
4 files changed, 18 insertions(+), 14 deletions(-)
diff --git a/src/main/java/htsjdk/samtools/SAMSequenceDictionaryCodec.java b/src/main/java/htsjdk/samtools/SAMSequenceDictionaryCodec.java
index e6e3ba5924..4f45d7cc3c 100644
--- a/src/main/java/htsjdk/samtools/SAMSequenceDictionaryCodec.java
+++ b/src/main/java/htsjdk/samtools/SAMSequenceDictionaryCodec.java
@@ -26,6 +26,7 @@
import htsjdk.samtools.util.LineReader;
import java.io.BufferedWriter;
+import java.io.Writer;
/**
* "On the fly" codec SAMSequenceDictionaryCodec.
@@ -67,7 +68,7 @@ public class SAMSequenceDictionaryCodec {
private final SAMTextHeaderCodec codec;
- public SAMSequenceDictionaryCodec(final BufferedWriter writer) {
+ public SAMSequenceDictionaryCodec(final Writer writer) {
codec = new SAMTextHeaderCodec();
codec.setmFileHeader(EMPTY_HEADER);
codec.setWriter(writer);
diff --git a/src/main/java/htsjdk/samtools/SAMTextHeaderCodec.java b/src/main/java/htsjdk/samtools/SAMTextHeaderCodec.java
index 5e4f64cd34..11d5dfd71a 100644
--- a/src/main/java/htsjdk/samtools/SAMTextHeaderCodec.java
+++ b/src/main/java/htsjdk/samtools/SAMTextHeaderCodec.java
@@ -62,7 +62,7 @@ public class SAMTextHeaderCodec {
private ValidationStringency validationStringency = ValidationStringency.SILENT;
// These attributes are populated when generating text
- private BufferedWriter writer;
+ private Writer writer;
private static final String TAG_KEY_VALUE_SEPARATOR = ":";
private static final char TAG_KEY_VALUE_SEPARATOR_CHAR = ':';
@@ -73,7 +73,7 @@ public class SAMTextHeaderCodec {
public static final String COMMENT_PREFIX = HEADER_LINE_START + HeaderRecordType.CO.name() + FIELD_SEPARATOR;
private static final Log log = Log.getInstance(SAMTextHeaderCodec.class);
- void setWriter(final BufferedWriter writer) {
+ void setWriter(final Writer writer) {
this.writer = writer;
}
diff --git a/src/main/java/htsjdk/samtools/reference/FastaReferenceWriter.java b/src/main/java/htsjdk/samtools/reference/FastaReferenceWriter.java
index 847dfea55c..67c1d4cf86 100644
--- a/src/main/java/htsjdk/samtools/reference/FastaReferenceWriter.java
+++ b/src/main/java/htsjdk/samtools/reference/FastaReferenceWriter.java
@@ -32,6 +32,7 @@
import java.io.*;
import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.util.*;
@@ -183,15 +184,14 @@ public final class FastaReferenceWriter implements AutoCloseable {
* @param dictOutput the (uncompressed) output stream to the dictFile, if requested, {@code null} if none should be generated.
* @throws IllegalArgumentException if {@code fastaFile} is {@code null} or {@code basesPerLine} is 0 or negative.
*/
- protected FastaReferenceWriter(final int basesPerLine,
+ FastaReferenceWriter(final int basesPerLine,
final OutputStream fastaOutput,
final OutputStream indexOutput,
final OutputStream dictOutput) {
this.defaultBasePerLine = basesPerLine;
this.fastaStream = new CountingOutputStream(fastaOutput);
this.indexWriter = indexOutput == null ? NullWriter.NULL_WRITER : new OutputStreamWriter(indexOutput, CHARSET);
- final BufferedWriter dictWriter = new BufferedWriter(dictOutput == null ? NullWriter.NULL_WRITER : new OutputStreamWriter(dictOutput, CHARSET));
- this.dictWriter = dictWriter;
+ this.dictWriter = dictOutput == null ? NullWriter.NULL_WRITER : new OutputStreamWriter(dictOutput, CHARSET);
this.dictCodec = new SAMSequenceDictionaryCodec(dictWriter);
this.dictCodec.encodeHeaderLine(false);
}
@@ -441,7 +441,7 @@ private void writeDictEntry() {
*/
public FastaReferenceWriter appendBases(final String basesBases)
throws IOException {
- return appendBases(basesBases.getBytes("ASCII"));
+ return appendBases(basesBases.getBytes(StandardCharsets.US_ASCII));
}
/**
@@ -592,20 +592,22 @@ private void assertIsNotClosed() {
*
*
* @throws IOException if such exception is thrown when closing output writers and output streams.
+ * @throws IllegalStateException if closing without writing any sequences or closing when writing a sequence is in progress
*/
+ @Override
public void close() throws IOException {
- try {
- if (!closed) {
+ if (!closed) {
+ try {
closeSequence();
if (sequenceNames.isEmpty()) {
throw new IllegalStateException("no sequences were added to the reference");
}
+ } finally {
+ closed = true;
fastaStream.close();
indexWriter.close();
dictWriter.close();
}
- } finally {
- closed = true;
}
}
diff --git a/src/main/java/htsjdk/samtools/reference/FastaReferenceWriterBuilder.java b/src/main/java/htsjdk/samtools/reference/FastaReferenceWriterBuilder.java
index c0ff467e8f..29ea95e538 100644
--- a/src/main/java/htsjdk/samtools/reference/FastaReferenceWriterBuilder.java
+++ b/src/main/java/htsjdk/samtools/reference/FastaReferenceWriterBuilder.java
@@ -26,6 +26,7 @@
import htsjdk.utils.ValidationUtils;
+import java.io.BufferedOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.file.Files;
@@ -211,13 +212,13 @@ public FastaReferenceWriter build() throws IOException {
checkBasesPerLine(basesPerLine);
if (fastaFile != null) {
- fastaOutput = Files.newOutputStream(fastaFile);
+ fastaOutput = new BufferedOutputStream(Files.newOutputStream(fastaFile));
}
if (indexFile != null) {
- indexOutput = Files.newOutputStream(indexFile);
+ indexOutput = new BufferedOutputStream(Files.newOutputStream(indexFile));
}
if (dictFile != null) {
- dictOutput = Files.newOutputStream(dictFile);
+ dictOutput = new BufferedOutputStream(Files.newOutputStream(dictFile));
}
return new FastaReferenceWriter(basesPerLine, fastaOutput, indexOutput, dictOutput);
From d2aa1744e039a0c5cdf30b75325bc364dc6cb868 Mon Sep 17 00:00:00 2001
From: Louis Bergelson
Date: Sat, 16 Feb 2019 09:07:35 -0500
Subject: [PATCH 3/3] remove extraneous import
---
src/main/java/htsjdk/samtools/SAMSequenceDictionaryCodec.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/src/main/java/htsjdk/samtools/SAMSequenceDictionaryCodec.java b/src/main/java/htsjdk/samtools/SAMSequenceDictionaryCodec.java
index 4f45d7cc3c..4bdbbd0e22 100644
--- a/src/main/java/htsjdk/samtools/SAMSequenceDictionaryCodec.java
+++ b/src/main/java/htsjdk/samtools/SAMSequenceDictionaryCodec.java
@@ -25,7 +25,7 @@
package htsjdk.samtools;
import htsjdk.samtools.util.LineReader;
-import java.io.BufferedWriter;
+
import java.io.Writer;
/**