From 9c1ce50504ae679d33048330249c144e600eab4c Mon Sep 17 00:00:00 2001 From: Louis Bergelson Date: Sat, 16 Feb 2019 14:46:43 -0500 Subject: [PATCH 1/3] moving HttpUtilsTest to externalApi test task * HttpUtilsTest is flakey because it connects to remote web servers. Moved it to the testExternalAPIs test task * changed a broken link so that the tests pass again --- build.gradle | 4 +++- src/test/java/htsjdk/samtools/util/HttpUtilsTest.java | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index 6b272c171b..d525875540 100644 --- a/build.gradle +++ b/build.gradle @@ -109,6 +109,7 @@ test { exclude "broken" exclude "defaultReference" exclude "ftp" + exclude "http" exclude "sra" exclude "ena" @@ -127,11 +128,12 @@ task testFTP(type: Test) { } task testExternalApis(type: Test) { - description = "Run the SRA and ENA tests (tests that interact with external APIs)" + description = "Run the SRA, ENA, and HTTP tests (tests that interact with external APIs)" jvmArgs += '-Dsamjdk.sra_libraries_download=true' tags { include "sra" + include "http" include "ena" exclude "slow" exclude "broken" diff --git a/src/test/java/htsjdk/samtools/util/HttpUtilsTest.java b/src/test/java/htsjdk/samtools/util/HttpUtilsTest.java index b759445802..827771f867 100644 --- a/src/test/java/htsjdk/samtools/util/HttpUtilsTest.java +++ b/src/test/java/htsjdk/samtools/util/HttpUtilsTest.java @@ -8,12 +8,13 @@ import htsjdk.HtsjdkTest; +@Test(groups = "http") public class HttpUtilsTest extends HtsjdkTest { @DataProvider(name = "existing_urls") public Object[][] testExistingURLsData() { return new Object[][]{ {"http://broadinstitute.github.io/picard/testdata/index_test.bam"}, - {"http://ftp.1000genomes.ebi.ac.uk/vol1/ftp/current.tree"} + {"http://ftp.1000genomes.ebi.ac.uk/vol1/ftp/README_using_1000genomes_cram.md"} }; } From 50da9cd8b251ddf05dde7e9d34ee17b1ae6e1c8f Mon Sep 17 00:00:00 2001 From: Louis Bergelson Date: Fri, 15 Feb 2019 17:42:49 -0500 Subject: [PATCH 2/3] Changes to FastaReferenceWriter * make sure the OutputStreams are always buffered * change close behavior to be more likely to close all output streams in the event of an early shutdown * loosen restriction from BufferedWriter to Writer in SAMSequenceDictionaryCodec and SAMTextHeaderCodec --- .../samtools/SAMSequenceDictionaryCodec.java | 3 ++- .../htsjdk/samtools/SAMTextHeaderCodec.java | 4 ++-- .../reference/FastaReferenceWriter.java | 18 ++++++++++-------- .../reference/FastaReferenceWriterBuilder.java | 7 ++++--- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/src/main/java/htsjdk/samtools/SAMSequenceDictionaryCodec.java b/src/main/java/htsjdk/samtools/SAMSequenceDictionaryCodec.java index e6e3ba5924..4f45d7cc3c 100644 --- a/src/main/java/htsjdk/samtools/SAMSequenceDictionaryCodec.java +++ b/src/main/java/htsjdk/samtools/SAMSequenceDictionaryCodec.java @@ -26,6 +26,7 @@ import htsjdk.samtools.util.LineReader; import java.io.BufferedWriter; +import java.io.Writer; /** * "On the fly" codec SAMSequenceDictionaryCodec. @@ -67,7 +68,7 @@ public class SAMSequenceDictionaryCodec { private final SAMTextHeaderCodec codec; - public SAMSequenceDictionaryCodec(final BufferedWriter writer) { + public SAMSequenceDictionaryCodec(final Writer writer) { codec = new SAMTextHeaderCodec(); codec.setmFileHeader(EMPTY_HEADER); codec.setWriter(writer); diff --git a/src/main/java/htsjdk/samtools/SAMTextHeaderCodec.java b/src/main/java/htsjdk/samtools/SAMTextHeaderCodec.java index 5e4f64cd34..11d5dfd71a 100644 --- a/src/main/java/htsjdk/samtools/SAMTextHeaderCodec.java +++ b/src/main/java/htsjdk/samtools/SAMTextHeaderCodec.java @@ -62,7 +62,7 @@ public class SAMTextHeaderCodec { private ValidationStringency validationStringency = ValidationStringency.SILENT; // These attributes are populated when generating text - private BufferedWriter writer; + private Writer writer; private static final String TAG_KEY_VALUE_SEPARATOR = ":"; private static final char TAG_KEY_VALUE_SEPARATOR_CHAR = ':'; @@ -73,7 +73,7 @@ public class SAMTextHeaderCodec { public static final String COMMENT_PREFIX = HEADER_LINE_START + HeaderRecordType.CO.name() + FIELD_SEPARATOR; private static final Log log = Log.getInstance(SAMTextHeaderCodec.class); - void setWriter(final BufferedWriter writer) { + void setWriter(final Writer writer) { this.writer = writer; } diff --git a/src/main/java/htsjdk/samtools/reference/FastaReferenceWriter.java b/src/main/java/htsjdk/samtools/reference/FastaReferenceWriter.java index 847dfea55c..67c1d4cf86 100644 --- a/src/main/java/htsjdk/samtools/reference/FastaReferenceWriter.java +++ b/src/main/java/htsjdk/samtools/reference/FastaReferenceWriter.java @@ -32,6 +32,7 @@ import java.io.*; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.*; @@ -183,15 +184,14 @@ public final class FastaReferenceWriter implements AutoCloseable { * @param dictOutput the (uncompressed) output stream to the dictFile, if requested, {@code null} if none should be generated. * @throws IllegalArgumentException if {@code fastaFile} is {@code null} or {@code basesPerLine} is 0 or negative. */ - protected FastaReferenceWriter(final int basesPerLine, + FastaReferenceWriter(final int basesPerLine, final OutputStream fastaOutput, final OutputStream indexOutput, final OutputStream dictOutput) { this.defaultBasePerLine = basesPerLine; this.fastaStream = new CountingOutputStream(fastaOutput); this.indexWriter = indexOutput == null ? NullWriter.NULL_WRITER : new OutputStreamWriter(indexOutput, CHARSET); - final BufferedWriter dictWriter = new BufferedWriter(dictOutput == null ? NullWriter.NULL_WRITER : new OutputStreamWriter(dictOutput, CHARSET)); - this.dictWriter = dictWriter; + this.dictWriter = dictOutput == null ? NullWriter.NULL_WRITER : new OutputStreamWriter(dictOutput, CHARSET); this.dictCodec = new SAMSequenceDictionaryCodec(dictWriter); this.dictCodec.encodeHeaderLine(false); } @@ -441,7 +441,7 @@ private void writeDictEntry() { */ public FastaReferenceWriter appendBases(final String basesBases) throws IOException { - return appendBases(basesBases.getBytes("ASCII")); + return appendBases(basesBases.getBytes(StandardCharsets.US_ASCII)); } /** @@ -592,20 +592,22 @@ private void assertIsNotClosed() { *

* * @throws IOException if such exception is thrown when closing output writers and output streams. + * @throws IllegalStateException if closing without writing any sequences or closing when writing a sequence is in progress */ + @Override public void close() throws IOException { - try { - if (!closed) { + if (!closed) { + try { closeSequence(); if (sequenceNames.isEmpty()) { throw new IllegalStateException("no sequences were added to the reference"); } + } finally { + closed = true; fastaStream.close(); indexWriter.close(); dictWriter.close(); } - } finally { - closed = true; } } diff --git a/src/main/java/htsjdk/samtools/reference/FastaReferenceWriterBuilder.java b/src/main/java/htsjdk/samtools/reference/FastaReferenceWriterBuilder.java index c0ff467e8f..29ea95e538 100644 --- a/src/main/java/htsjdk/samtools/reference/FastaReferenceWriterBuilder.java +++ b/src/main/java/htsjdk/samtools/reference/FastaReferenceWriterBuilder.java @@ -26,6 +26,7 @@ import htsjdk.utils.ValidationUtils; +import java.io.BufferedOutputStream; import java.io.IOException; import java.io.OutputStream; import java.nio.file.Files; @@ -211,13 +212,13 @@ public FastaReferenceWriter build() throws IOException { checkBasesPerLine(basesPerLine); if (fastaFile != null) { - fastaOutput = Files.newOutputStream(fastaFile); + fastaOutput = new BufferedOutputStream(Files.newOutputStream(fastaFile)); } if (indexFile != null) { - indexOutput = Files.newOutputStream(indexFile); + indexOutput = new BufferedOutputStream(Files.newOutputStream(indexFile)); } if (dictFile != null) { - dictOutput = Files.newOutputStream(dictFile); + dictOutput = new BufferedOutputStream(Files.newOutputStream(dictFile)); } return new FastaReferenceWriter(basesPerLine, fastaOutput, indexOutput, dictOutput); From d2aa1744e039a0c5cdf30b75325bc364dc6cb868 Mon Sep 17 00:00:00 2001 From: Louis Bergelson Date: Sat, 16 Feb 2019 09:07:35 -0500 Subject: [PATCH 3/3] remove extraneous import --- src/main/java/htsjdk/samtools/SAMSequenceDictionaryCodec.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/htsjdk/samtools/SAMSequenceDictionaryCodec.java b/src/main/java/htsjdk/samtools/SAMSequenceDictionaryCodec.java index 4f45d7cc3c..4bdbbd0e22 100644 --- a/src/main/java/htsjdk/samtools/SAMSequenceDictionaryCodec.java +++ b/src/main/java/htsjdk/samtools/SAMSequenceDictionaryCodec.java @@ -25,7 +25,7 @@ package htsjdk.samtools; import htsjdk.samtools.util.LineReader; -import java.io.BufferedWriter; + import java.io.Writer; /**