From 3b3d107de954b71a026cc993a0e5e4cd930c3707 Mon Sep 17 00:00:00 2001 From: Louis Bergelson Date: Thu, 21 Feb 2019 16:59:32 -0500 Subject: [PATCH] adding new overloads to IOUtils with Path for some file only methods (#1296) * adding new overloads to IOUtils with Path for some file only methods * fixes #1294 --- .../java/htsjdk/samtools/util/IOUtil.java | 106 +++++++++++++----- .../java/htsjdk/samtools/util/IOUtilTest.java | 31 ++++- 2 files changed, 104 insertions(+), 33 deletions(-) diff --git a/src/main/java/htsjdk/samtools/util/IOUtil.java b/src/main/java/htsjdk/samtools/util/IOUtil.java index 83d0a6b461..30cedc34a9 100755 --- a/src/main/java/htsjdk/samtools/util/IOUtil.java +++ b/src/main/java/htsjdk/samtools/util/IOUtil.java @@ -26,7 +26,6 @@ import htsjdk.samtools.Defaults; import htsjdk.samtools.SAMException; -import htsjdk.samtools.SamStreams; import htsjdk.samtools.seekablestream.SeekableBufferedStream; import htsjdk.samtools.seekablestream.SeekableFileStream; import htsjdk.samtools.seekablestream.SeekableHTTPStream; @@ -55,11 +54,7 @@ import java.net.URISyntaxException; import java.net.URL; import java.nio.charset.Charset; -import java.nio.file.FileSystemNotFoundException; -import java.nio.file.FileSystems; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; +import java.nio.file.*; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -107,11 +102,13 @@ public class IOUtil { public static final String DICT_FILE_EXTENSION = ".dict"; - public static final Set BLOCK_COMPRESSED_EXTENSIONS = Collections.unmodifiableSet(new HashSet(Arrays.asList(".gz", ".gzip", ".bgz", ".bgzf"))); + public static final Set BLOCK_COMPRESSED_EXTENSIONS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(".gz", ".gzip", ".bgz", ".bgzf"))); /** number of bytes that will be read for the GZIP-header in the function {@link #isGZIPInputStream(InputStream)} */ public static final int GZIP_HEADER_READ_LENGTH = 8000; + private static final OpenOption[] EMPTY_OPEN_OPTIONS = new OpenOption[0]; + private static int compressionLevel = Defaults.COMPRESSION_LEVEL; /** * Sets the GZip compression level for subsequent GZIPOutputStream object creation. @@ -625,8 +622,7 @@ public static InputStream openFileForReading(final File file) { public static InputStream openFileForReading(final Path path) { try { - if (path.getFileName().toString().endsWith(".gz") || - path.getFileName().toString().endsWith(".bfq")) { + if (hasGzipFileExtension(path)) { return openGzipFileForReading(path); } else { @@ -672,30 +668,46 @@ public static InputStream openGzipFileForReading(final Path path) { * @return the output stream to write to */ public static OutputStream openFileForWriting(final File file) { - return openFileForWriting(file, false); + return openFileForWriting(toPath(file)); } /** - * Opens a file for writing + * Opens a file for writing, gzip it if it ends with ".gz" or "bfq" * * @param file the file to write to * @param append whether to append to the file if it already exists (we overwrite it if false) * @return the output stream to write to */ public static OutputStream openFileForWriting(final File file, final boolean append) { + return openFileForWriting(toPath(file), getAppendOpenOption(append)); + } + /** + * Opens a file for writing, gzip it if it ends with ".gz" or "bfq" + * + * @param path the file to write to + * @param openOptions options to use when opening the file + * @return the output stream to write to + */ + public static OutputStream openFileForWriting(final Path path, OpenOption... openOptions) { try { - if (file.getName().endsWith(".gz") || - file.getName().endsWith(".bfq")) { - return openGzipFileForWriting(file, append); - } - else { - return new FileOutputStream(file, append); + if (hasGzipFileExtension(path)) { + return openGzipFileForWriting(path, openOptions); + } else { + return Files.newOutputStream(path, openOptions); } + } catch (final IOException ioe) { + throw new SAMException("Error opening file for writing: " + path.toUri().toString(), ioe); } - catch (IOException ioe) { - throw new SAMException("Error opening file for writing: " + file.getName(), ioe); - } + } + + /** + * check if the file name ends with .gz, .gzip, or .bfq + */ + private static boolean hasGzipFileExtension(Path path) { + final List gzippedEndings = Arrays.asList(".gz", ".gzip", ".bfq"); + final String fileName = path.getFileName().toString(); + return gzippedEndings.stream().anyMatch(fileName::endsWith); } /** @@ -705,19 +717,32 @@ public static BufferedWriter openFileForBufferedWriting(final File file, final b return new BufferedWriter(new OutputStreamWriter(openFileForWriting(file, append)), Defaults.NON_ZERO_BUFFER_SIZE); } + /** + * Preferred over PrintStream and PrintWriter because an exception is thrown on I/O error + */ + public static BufferedWriter openFileForBufferedWriting(final Path path, final OpenOption ... openOptions) { + return new BufferedWriter(new OutputStreamWriter(openFileForWriting(path, openOptions)), Defaults.NON_ZERO_BUFFER_SIZE); + } + /** * Preferred over PrintStream and PrintWriter because an exception is thrown on I/O error */ public static BufferedWriter openFileForBufferedWriting(final File file) { - return openFileForBufferedWriting(file, false); + return openFileForBufferedWriting(IOUtil.toPath(file)); } /** * Preferred over PrintStream and PrintWriter because an exception is thrown on I/O error */ public static BufferedWriter openFileForBufferedUtf8Writing(final File file) { - return new BufferedWriter(new OutputStreamWriter(openFileForWriting(file), Charset.forName("UTF-8")), - Defaults.NON_ZERO_BUFFER_SIZE); + return openFileForBufferedUtf8Writing(IOUtil.toPath(file)); + } + + /** + * Preferred over PrintStream and PrintWriter because an exception is thrown on I/O error + */ + public static BufferedWriter openFileForBufferedUtf8Writing(final Path path) { + return new BufferedWriter(new OutputStreamWriter(openFileForWriting(path), Charset.forName("UTF-8")), Defaults.NON_ZERO_BUFFER_SIZE); } /** @@ -738,23 +763,42 @@ public static BufferedReader openFileForBufferedUtf8Reading(final File file) { * @return the output stream to write to */ public static OutputStream openGzipFileForWriting(final File file, final boolean append) { + return openGzipFileForWriting(IOUtil.toPath(file), getAppendOpenOption(append)); + } + + /** + * converts a boolean into an array containing either the append option or nothing + */ + private static OpenOption[] getAppendOpenOption(boolean append) { + return append ? new OpenOption[]{StandardOpenOption.APPEND} : EMPTY_OPEN_OPTIONS; + } + /** + * Opens a GZIP encoded file for writing + * + * @param path the file to write to + * @param openOptions options to control how the file is opened + * @return the output stream to write to + */ + public static OutputStream openGzipFileForWriting(final Path path, final OpenOption ... openOptions) { try { + final OutputStream out = Files.newOutputStream(path, openOptions); if (Defaults.BUFFER_SIZE > 0) { - return new CustomGzipOutputStream(new FileOutputStream(file, append), - Defaults.BUFFER_SIZE, - compressionLevel); + return new CustomGzipOutputStream(out, Defaults.BUFFER_SIZE, compressionLevel); } else { - return new CustomGzipOutputStream(new FileOutputStream(file, append), compressionLevel); + return new CustomGzipOutputStream(out, compressionLevel); } - } - catch (IOException ioe) { - throw new SAMException("Error opening file for writing: " + file.getName(), ioe); + } catch (final IOException ioe) { + throw new SAMException("Error opening file for writing: " + path.toUri().toString(), ioe); } } public static OutputStream openFileForMd5CalculatingWriting(final File file) { - return new Md5CalculatingOutputStream(IOUtil.openFileForWriting(file), new File(file.getAbsolutePath() + ".md5")); + return openFileForMd5CalculatingWriting(toPath(file)); + } + + public static OutputStream openFileForMd5CalculatingWriting(final Path file) { + return new Md5CalculatingOutputStream(IOUtil.openFileForWriting(file), file.resolve(".md5")); } /** diff --git a/src/test/java/htsjdk/samtools/util/IOUtilTest.java b/src/test/java/htsjdk/samtools/util/IOUtilTest.java index 05633abf11..8bdfd0cca9 100644 --- a/src/test/java/htsjdk/samtools/util/IOUtilTest.java +++ b/src/test/java/htsjdk/samtools/util/IOUtilTest.java @@ -67,7 +67,7 @@ public class IOUtilTest extends HtsjdkTest { private static final List SLURP_TEST_LINES = Arrays.asList("bacon and rice ", "for breakfast ", "wont you join me"); private static final String SLURP_TEST_LINE_SEPARATOR = "\n"; private static final String TEST_FILE_PREFIX = "htsjdk-IOUtilTest"; - private static final String TEST_FILE_EXTENSIONS[] = {".txt", ".txt.gz"}; + private static final String[] TEST_FILE_EXTENSIONS = {".txt", ".txt.gz"}; private static final String TEST_STRING = "bar!"; private File existingTempFile; @@ -154,7 +154,7 @@ public void testGetCanonicalPath() throws IOException { File lnToSymlink = new File(lnDir, "symlink.txt"); lnToSymlink.deleteOnExit(); - File files[] = {actual, symlink, lnToActual, lnToSymlink}; + File[] files = {actual, symlink, lnToActual, lnToSymlink}; for (File f : files) { Assert.assertEquals(IOUtil.getFullCanonicalPath(f), actual.getCanonicalPath()); } @@ -582,6 +582,33 @@ public void testCompressionLevel(final Path file, final String extension, final } } + @DataProvider + public Object[][] getExtensions(){ + return new Object[][]{ + {".gz", true}, + {".bfq", true}, + {".txt", false}}; + } + + @Test(dataProvider = "getExtensions") + public void testReadWriteJimfs(String extension, boolean gzipped) throws IOException { + final Path jmfsRoot = inMemoryFileSystem.getRootDirectories().iterator().next(); + final Path tmp = Files.createTempFile(jmfsRoot, "test", extension); + final String expected = "lorem ipswitch, nantucket, bucket"; + try (Writer out = IOUtil.openFileForBufferedWriting(tmp)){ + out.write(expected); + } + + try (InputStream in = new BufferedInputStream(Files.newInputStream(tmp))){ + Assert.assertEquals(IOUtil.isGZIPInputStream(in), gzipped); + } + + try (BufferedReader in = IOUtil.openFileForBufferedReading(tmp)){ + final String actual = in.readLine(); + Assert.assertEquals(actual, expected); + } + } + @DataProvider public static Object[][] badCompressionLevels() { return new Object[][]{