Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

adding new overloads to IOUtils with Path for some file only methods #1296

Merged
merged 4 commits into from
Feb 21, 2019
Merged

adding new overloads to IOUtils with Path for some file only methods #1296

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
789e8ee
adding new overloads to IOUtils with Path for some file only methods
lbergelson Feb 20, 2019
cd9c2a1
spacing fix
Feb 21, 2019
2bb0a53
test zipped files for zipness
lbergelson Feb 21, 2019
0ad3238
Apply suggestions from code review
Feb 21, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 75 additions & 31 deletions src/main/java/htsjdk/samtools/util/IOUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@

import htsjdk.samtools.Defaults;
import htsjdk.samtools.SAMException;
import htsjdk.samtools.SamStreams;
import htsjdk.samtools.seekablestream.SeekableBufferedStream;
import htsjdk.samtools.seekablestream.SeekableFileStream;
import htsjdk.samtools.seekablestream.SeekableHTTPStream;
Expand Down Expand Up @@ -55,11 +54,7 @@
import java.net.URISyntaxException;
import java.net.URL;
import java.nio.charset.Charset;
import java.nio.file.FileSystemNotFoundException;
import java.nio.file.FileSystems;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.*;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
Expand Down Expand Up @@ -107,11 +102,13 @@ public class IOUtil {

public static final String DICT_FILE_EXTENSION = ".dict";

public static final Set<String> BLOCK_COMPRESSED_EXTENSIONS = Collections.unmodifiableSet(new HashSet<String>(Arrays.asList(".gz", ".gzip", ".bgz", ".bgzf")));
public static final Set<String> BLOCK_COMPRESSED_EXTENSIONS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(".gz", ".gzip", ".bgz", ".bgzf")));

/** number of bytes that will be read for the GZIP-header in the function {@link #isGZIPInputStream(InputStream)} */
public static final int GZIP_HEADER_READ_LENGTH = 8000;

private static final OpenOption[] EMPTY_OPEN_OPTIONS = new OpenOption[0];

private static int compressionLevel = Defaults.COMPRESSION_LEVEL;
/**
* Sets the GZip compression level for subsequent GZIPOutputStream object creation.
Expand Down Expand Up @@ -625,8 +622,7 @@ public static InputStream openFileForReading(final File file) {
public static InputStream openFileForReading(final Path path) {

try {
if (path.getFileName().toString().endsWith(".gz") ||
path.getFileName().toString().endsWith(".bfq")) {
if (hasGzipFileExtension(path)) {
return openGzipFileForReading(path);
}
else {
Expand Down Expand Up @@ -672,30 +668,46 @@ public static InputStream openGzipFileForReading(final Path path) {
* @return the output stream to write to
*/
public static OutputStream openFileForWriting(final File file) {
return openFileForWriting(file, false);
return openFileForWriting(toPath(file));
}

/**
* Opens a file for writing
* Opens a file for writing, gzip it if it ends with ".gz" or "bfq"
*
* @param file the file to write to
* @param append whether to append to the file if it already exists (we overwrite it if false)
* @return the output stream to write to
*/
public static OutputStream openFileForWriting(final File file, final boolean append) {
return openFileForWriting(toPath(file), getAppendOpenOption(append));
}

/**
* Opens a file for writing, gzip it if it ends with ".gz" or "bfq"
*
* @param path the file to write to
* @param openOptions options to use when opening the file
* @return the output stream to write to
*/
public static OutputStream openFileForWriting(final Path path, OpenOption... openOptions) {
try {
if (file.getName().endsWith(".gz") ||
file.getName().endsWith(".bfq")) {
return openGzipFileForWriting(file, append);
}
else {
return new FileOutputStream(file, append);
if (hasGzipFileExtension(path)) {
return openGzipFileForWriting(path, openOptions);
} else {
return Files.newOutputStream(path, openOptions);
}
} catch (final IOException ioe) {
throw new SAMException("Error opening file for writing: " + path.toUri().toString(), ioe);
}
catch (IOException ioe) {
throw new SAMException("Error opening file for writing: " + file.getName(), ioe);
}
}

/**
* check if the file name ends with .gz, .gzip, or .bfq
*/
private static boolean hasGzipFileExtension(Path path) {
final List<String> gzippedEndings = Arrays.asList(".gz", ".gzip", ".bfq");
final String fileName = path.getFileName().toString();
return gzippedEndings.stream().anyMatch(fileName::endsWith);
}

/**
Expand All @@ -705,19 +717,32 @@ public static BufferedWriter openFileForBufferedWriting(final File file, final b
return new BufferedWriter(new OutputStreamWriter(openFileForWriting(file, append)), Defaults.NON_ZERO_BUFFER_SIZE);
}

/**
* Preferred over PrintStream and PrintWriter because an exception is thrown on I/O error
*/
public static BufferedWriter openFileForBufferedWriting(final Path path, final OpenOption ... openOptions) {
return new BufferedWriter(new OutputStreamWriter(openFileForWriting(path, openOptions)), Defaults.NON_ZERO_BUFFER_SIZE);
}

/**
* Preferred over PrintStream and PrintWriter because an exception is thrown on I/O error
*/
public static BufferedWriter openFileForBufferedWriting(final File file) {
return openFileForBufferedWriting(file, false);
return openFileForBufferedWriting(IOUtil.toPath(file));
}

/**
* Preferred over PrintStream and PrintWriter because an exception is thrown on I/O error
*/
public static BufferedWriter openFileForBufferedUtf8Writing(final File file) {
return new BufferedWriter(new OutputStreamWriter(openFileForWriting(file), Charset.forName("UTF-8")),
Defaults.NON_ZERO_BUFFER_SIZE);
return openFileForBufferedUtf8Writing(IOUtil.toPath(file));
}

/**
* Preferred over PrintStream and PrintWriter because an exception is thrown on I/O error
*/
public static BufferedWriter openFileForBufferedUtf8Writing(final Path path) {
return new BufferedWriter(new OutputStreamWriter(openFileForWriting(path), Charset.forName("UTF-8")), Defaults.NON_ZERO_BUFFER_SIZE);
}

/**
Expand All @@ -738,23 +763,42 @@ public static BufferedReader openFileForBufferedUtf8Reading(final File file) {
* @return the output stream to write to
*/
public static OutputStream openGzipFileForWriting(final File file, final boolean append) {
return openGzipFileForWriting(IOUtil.toPath(file), getAppendOpenOption(append));
}

/**
* converts a boolean into an array containing either the append option or nothing
*/
private static OpenOption[] getAppendOpenOption(boolean append) {
return append ? new OpenOption[]{StandardOpenOption.APPEND} : EMPTY_OPEN_OPTIONS;
}

/**
* Opens a GZIP encoded file for writing
*
* @param path the file to write to
* @param openOptions options to control how the file is opened
* @return the output stream to write to
*/
public static OutputStream openGzipFileForWriting(final Path path, final OpenOption ... openOptions) {
try {
final OutputStream out = Files.newOutputStream(path, openOptions);
if (Defaults.BUFFER_SIZE > 0) {
return new CustomGzipOutputStream(new FileOutputStream(file, append),
Defaults.BUFFER_SIZE,
compressionLevel);
return new CustomGzipOutputStream(out, Defaults.BUFFER_SIZE, compressionLevel);
} else {
return new CustomGzipOutputStream(new FileOutputStream(file, append), compressionLevel);
return new CustomGzipOutputStream(out, compressionLevel);
}
}
catch (IOException ioe) {
throw new SAMException("Error opening file for writing: " + file.getName(), ioe);
} catch (final IOException ioe) {
throw new SAMException("Error opening file for writing: " + path.toUri().toString(), ioe);
}
}

public static OutputStream openFileForMd5CalculatingWriting(final File file) {
return new Md5CalculatingOutputStream(IOUtil.openFileForWriting(file), new File(file.getAbsolutePath() + ".md5"));
return openFileForMd5CalculatingWriting(toPath(file));
}

public static OutputStream openFileForMd5CalculatingWriting(final Path file) {
return new Md5CalculatingOutputStream(IOUtil.openFileForWriting(file), file.resolve(".md5"));
}

/**
Expand Down
31 changes: 29 additions & 2 deletions src/test/java/htsjdk/samtools/util/IOUtilTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ public class IOUtilTest extends HtsjdkTest {
private static final List<String> SLURP_TEST_LINES = Arrays.asList("bacon and rice ", "for breakfast ", "wont you join me");
private static final String SLURP_TEST_LINE_SEPARATOR = "\n";
private static final String TEST_FILE_PREFIX = "htsjdk-IOUtilTest";
private static final String TEST_FILE_EXTENSIONS[] = {".txt", ".txt.gz"};
private static final String[] TEST_FILE_EXTENSIONS = {".txt", ".txt.gz"};
private static final String TEST_STRING = "bar!";

private File existingTempFile;
Expand Down Expand Up @@ -154,7 +154,7 @@ public void testGetCanonicalPath() throws IOException {
File lnToSymlink = new File(lnDir, "symlink.txt");
lnToSymlink.deleteOnExit();

File files[] = {actual, symlink, lnToActual, lnToSymlink};
File[] files = {actual, symlink, lnToActual, lnToSymlink};
for (File f : files) {
Assert.assertEquals(IOUtil.getFullCanonicalPath(f), actual.getCanonicalPath());
}
Expand Down Expand Up @@ -582,6 +582,33 @@ public void testCompressionLevel(final Path file, final String extension, final
}
}

@DataProvider
public Object[][] getExtensions(){
return new Object[][]{
{".gz", true},
{".bfq", true},
{".txt", false}};
}

@Test(dataProvider = "getExtensions")
public void testReadWriteJimfs(String extension, boolean gzipped) throws IOException {
final Path jmfsRoot = inMemoryFileSystem.getRootDirectories().iterator().next();
final Path tmp = Files.createTempFile(jmfsRoot, "test", extension);
final String expected = "lorem ipswitch, nantucket, bucket";
try (Writer out = IOUtil.openFileForBufferedWriting(tmp)){
out.write(expected);
}

try (InputStream in = new BufferedInputStream(Files.newInputStream(tmp))){
Assert.assertEquals(IOUtil.isGZIPInputStream(in), gzipped);
}

try (BufferedReader in = IOUtil.openFileForBufferedReading(tmp)){
final String actual = in.readLine();
Assert.assertEquals(actual, expected);
}
}

@DataProvider
public static Object[][] badCompressionLevels() {
return new Object[][]{
Expand Down