Skip to content

Commit

Permalink
adding new overloads to IOUtils with Path for some file only methods (#…
Browse files Browse the repository at this point in the history 
…1296)

* adding new overloads to IOUtils with Path for some file only methods
* fixes #1294
  • Loading branch information
@lbergelson
lbergelson authored Feb 21, 2019
1 parent 5e8b1fa commit 3b3d107
Show file tree
Hide file tree
Showing 2 changed files with 104 additions and 33 deletions.
106 changes: 75 additions & 31 deletions src/main/java/htsjdk/samtools/util/IOUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@

import htsjdk.samtools.Defaults;
import htsjdk.samtools.SAMException;
import htsjdk.samtools.SamStreams;
import htsjdk.samtools.seekablestream.SeekableBufferedStream;
import htsjdk.samtools.seekablestream.SeekableFileStream;
import htsjdk.samtools.seekablestream.SeekableHTTPStream;
Expand Down Expand Up @@ -55,11 +54,7 @@
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.Charset;
import java.nio.file.FileSystemNotFoundException;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
Expand Down Expand Up @@ -107,11 +102,13 @@ public class IOUtil {

public static final String DICT_FILE_EXTENSION = ".dict";

public static final Set<String> BLOCK_COMPRESSED_EXTENSIONS = Collections.unmodifiableSet(new HashSet<String>(Arrays.asList(".gz", ".gzip", ".bgz", ".bgzf")));
public static final Set<String> BLOCK_COMPRESSED_EXTENSIONS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(".gz", ".gzip", ".bgz", ".bgzf")));

/** number of bytes that will be read for the GZIP-header in the function {@link #isGZIPInputStream(InputStream)} */
public static final int GZIP_HEADER_READ_LENGTH = 8000;

private static final OpenOption[] EMPTY_OPEN_OPTIONS = new OpenOption[0];

private static int compressionLevel = Defaults.COMPRESSION_LEVEL;
/**
* Sets the GZip compression level for subsequent GZIPOutputStream object creation.
Expand Down Expand Up @@ -625,8 +622,7 @@ public static InputStream openFileForReading(final File file) {
public static InputStream openFileForReading(final Path path) {

try {
if (path.getFileName().toString().endsWith(".gz") ||
path.getFileName().toString().endsWith(".bfq")) {
if (hasGzipFileExtension(path)) {
return openGzipFileForReading(path);
}
else {
Expand Down Expand Up @@ -672,30 +668,46 @@ public static InputStream openGzipFileForReading(final Path path) {
* @return the output stream to write to
*/
public static OutputStream openFileForWriting(final File file) {
return openFileForWriting(file, false);
return openFileForWriting(toPath(file));
}

/**
* Opens a file for writing
* Opens a file for writing, gzip it if it ends with ".gz" or "bfq"
*
* @param file the file to write to
* @param append whether to append to the file if it already exists (we overwrite it if false)
* @return the output stream to write to
*/
public static OutputStream openFileForWriting(final File file, final boolean append) {
return openFileForWriting(toPath(file), getAppendOpenOption(append));
}

/**
* Opens a file for writing, gzip it if it ends with ".gz" or "bfq"
*
* @param path the file to write to
* @param openOptions options to use when opening the file
* @return the output stream to write to
*/
public static OutputStream openFileForWriting(final Path path, OpenOption... openOptions) {
try {
if (file.getName().endsWith(".gz") ||
file.getName().endsWith(".bfq")) {
return openGzipFileForWriting(file, append);
}
else {
return new FileOutputStream(file, append);
if (hasGzipFileExtension(path)) {
return openGzipFileForWriting(path, openOptions);
} else {
return Files.newOutputStream(path, openOptions);
}
} catch (final IOException ioe) {
throw new SAMException("Error opening file for writing: " + path.toUri().toString(), ioe);
}
catch (IOException ioe) {
throw new SAMException("Error opening file for writing: " + file.getName(), ioe);
}
}

/**
* check if the file name ends with .gz, .gzip, or .bfq
*/
private static boolean hasGzipFileExtension(Path path) {
final List<String> gzippedEndings = Arrays.asList(".gz", ".gzip", ".bfq");
final String fileName = path.getFileName().toString();
return gzippedEndings.stream().anyMatch(fileName::endsWith);
}

/**
Expand All @@ -705,19 +717,32 @@ public static BufferedWriter openFileForBufferedWriting(final File file, final b
return new BufferedWriter(new OutputStreamWriter(openFileForWriting(file, append)), Defaults.NON_ZERO_BUFFER_SIZE);
}

/**
* Preferred over PrintStream and PrintWriter because an exception is thrown on I/O error
*/
public static BufferedWriter openFileForBufferedWriting(final Path path, final OpenOption ... openOptions) {
return new BufferedWriter(new OutputStreamWriter(openFileForWriting(path, openOptions)), Defaults.NON_ZERO_BUFFER_SIZE);
}

/**
* Preferred over PrintStream and PrintWriter because an exception is thrown on I/O error
*/
public static BufferedWriter openFileForBufferedWriting(final File file) {
return openFileForBufferedWriting(file, false);
return openFileForBufferedWriting(IOUtil.toPath(file));
}

/**
* Preferred over PrintStream and PrintWriter because an exception is thrown on I/O error
*/
public static BufferedWriter openFileForBufferedUtf8Writing(final File file) {
return new BufferedWriter(new OutputStreamWriter(openFileForWriting(file), Charset.forName("UTF-8")),
Defaults.NON_ZERO_BUFFER_SIZE);
return openFileForBufferedUtf8Writing(IOUtil.toPath(file));
}

/**
* Preferred over PrintStream and PrintWriter because an exception is thrown on I/O error
*/
public static BufferedWriter openFileForBufferedUtf8Writing(final Path path) {
return new BufferedWriter(new OutputStreamWriter(openFileForWriting(path), Charset.forName("UTF-8")), Defaults.NON_ZERO_BUFFER_SIZE);
}

/**
Expand All @@ -738,23 +763,42 @@ public static BufferedReader openFileForBufferedUtf8Reading(final File file) {
* @return the output stream to write to
*/
public static OutputStream openGzipFileForWriting(final File file, final boolean append) {
return openGzipFileForWriting(IOUtil.toPath(file), getAppendOpenOption(append));
}

/**
* converts a boolean into an array containing either the append option or nothing
*/
private static OpenOption[] getAppendOpenOption(boolean append) {
return append ? new OpenOption[]{StandardOpenOption.APPEND} : EMPTY_OPEN_OPTIONS;
}

/**
* Opens a GZIP encoded file for writing
*
* @param path the file to write to
* @param openOptions options to control how the file is opened
* @return the output stream to write to
*/
public static OutputStream openGzipFileForWriting(final Path path, final OpenOption ... openOptions) {
try {
final OutputStream out = Files.newOutputStream(path, openOptions);
if (Defaults.BUFFER_SIZE > 0) {
return new CustomGzipOutputStream(new FileOutputStream(file, append),
Defaults.BUFFER_SIZE,
compressionLevel);
return new CustomGzipOutputStream(out, Defaults.BUFFER_SIZE, compressionLevel);
} else {
return new CustomGzipOutputStream(new FileOutputStream(file, append), compressionLevel);
return new CustomGzipOutputStream(out, compressionLevel);
}
}
catch (IOException ioe) {
throw new SAMException("Error opening file for writing: " + file.getName(), ioe);
} catch (final IOException ioe) {
throw new SAMException("Error opening file for writing: " + path.toUri().toString(), ioe);
}
}

public static OutputStream openFileForMd5CalculatingWriting(final File file) {
return new Md5CalculatingOutputStream(IOUtil.openFileForWriting(file), new File(file.getAbsolutePath() + ".md5"));
return openFileForMd5CalculatingWriting(toPath(file));
}

public static OutputStream openFileForMd5CalculatingWriting(final Path file) {
return new Md5CalculatingOutputStream(IOUtil.openFileForWriting(file), file.resolve(".md5"));
}

/**
Expand Down
31 changes: 29 additions & 2 deletions src/test/java/htsjdk/samtools/util/IOUtilTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ public class IOUtilTest extends HtsjdkTest {
private static final List<String> SLURP_TEST_LINES = Arrays.asList("bacon and rice ", "for breakfast ", "wont you join me");
private static final String SLURP_TEST_LINE_SEPARATOR = "\n";
private static final String TEST_FILE_PREFIX = "htsjdk-IOUtilTest";
private static final String TEST_FILE_EXTENSIONS[] = {".txt", ".txt.gz"};
private static final String[] TEST_FILE_EXTENSIONS = {".txt", ".txt.gz"};
private static final String TEST_STRING = "bar!";

private File existingTempFile;
Expand Down Expand Up @@ -154,7 +154,7 @@ public void testGetCanonicalPath() throws IOException {
File lnToSymlink = new File(lnDir, "symlink.txt");
lnToSymlink.deleteOnExit();

File files[] = {actual, symlink, lnToActual, lnToSymlink};
File[] files = {actual, symlink, lnToActual, lnToSymlink};
for (File f : files) {
Assert.assertEquals(IOUtil.getFullCanonicalPath(f), actual.getCanonicalPath());
}
Expand Down Expand Up @@ -582,6 +582,33 @@ public void testCompressionLevel(final Path file, final String extension, final
}
}

@DataProvider
public Object[][] getExtensions(){
return new Object[][]{
{".gz", true},
{".bfq", true},
{".txt", false}};
}

@Test(dataProvider = "getExtensions")
public void testReadWriteJimfs(String extension, boolean gzipped) throws IOException {
final Path jmfsRoot = inMemoryFileSystem.getRootDirectories().iterator().next();
final Path tmp = Files.createTempFile(jmfsRoot, "test", extension);
final String expected = "lorem ipswitch, nantucket, bucket";
try (Writer out = IOUtil.openFileForBufferedWriting(tmp)){
out.write(expected);
}

try (InputStream in = new BufferedInputStream(Files.newInputStream(tmp))){
Assert.assertEquals(IOUtil.isGZIPInputStream(in), gzipped);
}

try (BufferedReader in = IOUtil.openFileForBufferedReading(tmp)){
final String actual = in.readLine();
Assert.assertEquals(actual, expected);
}
}

@DataProvider
public static Object[][] badCompressionLevels() {
return new Object[][]{
Expand Down

0 comments on commit 3b3d107

Please sign in to comment.