From 298616aeadc846a71cf4500b816a8c05e4585624 Mon Sep 17 00:00:00 2001 From: Teddy Li Date: Sun, 13 Jun 2021 14:19:48 +0800 Subject: [PATCH] Core --- .gitignore | 7 + pom.xml | 43 +++++ .../ioutils/filesplit/ByteHelper.java | 156 ++++++++++++++++++ .../featurehouse/ioutils/filesplit/Core.java | 155 +++++++++++++++++ .../featurehouse/ioutils/filesplit/Main.java | 78 ++++++++- 5 files changed, 434 insertions(+), 5 deletions(-) create mode 100644 .gitignore create mode 100644 src/main/java/org/featurehouse/ioutils/filesplit/ByteHelper.java create mode 100644 src/main/java/org/featurehouse/ioutils/filesplit/Core.java diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..36ffc3a --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +.idea/ +.vscode/ +out/ +target/ +*.iml +*.ipr +*.iws \ No newline at end of file diff --git a/pom.xml b/pom.xml index 5bbb179..ee6d5a5 100644 --- a/pom.xml +++ b/pom.xml @@ -21,10 +21,53 @@ org.jetbrains annotations 21.0.1 + provided true + + org.zeroturnaround + zt-zip + 1.14 + compile + + + org.slf4j + slf4j-nop + 1.7.30 + compile + + + + + org.apache.maven.plugins + maven-assembly-plugin + 3.3.0 + + false + + jar-with-dependencies + + + + org.featurehouse.ioutils.filesplit.Main + + + + + + make-assembly + package + + single + + + + + + + The Apache Software License, Version 2.0 diff --git a/src/main/java/org/featurehouse/ioutils/filesplit/ByteHelper.java b/src/main/java/org/featurehouse/ioutils/filesplit/ByteHelper.java new file mode 100644 index 0000000..366fe16 --- /dev/null +++ b/src/main/java/org/featurehouse/ioutils/filesplit/ByteHelper.java @@ -0,0 +1,156 @@ +package org.featurehouse.ioutils.filesplit; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class ByteHelper { + public static byte[] fromInt(int i) { + return new byte[] { + (byte) (i >> 24), + (byte) (i >> 16), + (byte) (i >> 8), + (byte) i + }; + } + + public static int toInt(byte[] bs) { + if (bs.length < 4) return -1; + int[] intArray = new int[4]; + for (int i = 0; i < 4; ++i) { + if (bs[i] < 0) intArray[i] = 256 + bs[i]; + else intArray[i] = bs[i]; + } + return (intArray[0] << 24) + + (intArray[1] << 16) + + (intArray[2] << 8) + + intArray[3]; + } + + public static void writeString(OutputStream instance, String s) throws IOException { + byte[] b = s.getBytes(StandardCharsets.UTF_8); + instance.write(fromInt(b.length)); + instance.write(b); + } + + public static String readString(InputStream instance) + throws IOException, IllegalArgumentException { + byte[] b = new byte[4]; + Core.art(instance.read(b, 0, 4), i -> i != 4); + int l = toInt(b); // len + b = new byte[l]; + Core.art(instance.read(b, 0, l), i -> i != l); + return new String(b, StandardCharsets.UTF_8); + } + + /** + * Reads up to a specified number of bytes from the input stream. This + * method blocks until the requested number of bytes have been read, end + * of stream is detected, or an exception is thrown. This method does not + * close the input stream. + * + *

The length of the returned array equals the number of bytes read + * from the stream. If {@code len} is zero, then no bytes are read and + * an empty byte array is returned. Otherwise, up to {@code len} bytes + * are read from the stream. Fewer than {@code len} bytes may be read if + * end of stream is encountered. + * + *

When this stream reaches end of stream, further invocations of this + * method will return an empty byte array. + * + *

Note that this method is intended for simple cases where it is + * convenient to read the specified number of bytes into a byte array. The + * total amount of memory allocated by this method is proportional to the + * number of bytes read from the stream which is bounded by {@code len}. + * Therefore, the method may be safely called with very large values of + * {@code len} provided sufficient memory is available. + * + *

The behavior for the case where the input stream is asynchronously + * closed, or the thread interrupted during the read, is highly input + * stream specific, and therefore not specified. + * + *

If an I/O error occurs reading from the input stream, then it may do + * so after some, but not all, bytes have been read. Consequently the input + * stream may not be at end of stream and may be in an inconsistent state. + * It is strongly recommended that the stream be promptly closed if an I/O + * error occurs. + * + *

Since Java API 11. + * + * @implNote + * The number of bytes allocated to read data from this stream and return + * the result is bounded by {@code 2*(long)len}, inclusive. + * + * @param len the maximum number of bytes to read + * @return a byte array containing the bytes read from this input stream + * @throws IllegalArgumentException if {@code length} is negative + * @throws IOException if an I/O error occurs + * @throws OutOfMemoryError if an array of the required size cannot be + * allocated. + * + */ + public static byte[] readNBytes(InputStream instance, int len) throws IOException { + if (len < 0) { + throw new IllegalArgumentException("len < 0"); + } + + List bufs = null; + byte[] result = null; + int total = 0; + int remaining = len; + int n; + do { + byte[] buf = new byte[Math.min(remaining, 8192)]; + int nread = 0; + + // read to EOF which may read more or less than buffer size + while ((n = instance.read(buf, nread, + Math.min(buf.length - nread, remaining))) > 0) { + nread += n; + remaining -= n; + } + + if (nread > 0) { + if (2147483639 - total < nread) { + throw new OutOfMemoryError("Required array size too large"); + } + total += nread; + if (result == null) { + result = buf; + } else { + if (bufs == null) { + bufs = new ArrayList<>(); + bufs.add(result); + } + bufs.add(buf); + } + } + // if the last call to read returned -1 or the number of bytes + // requested have been read then break + } while (n >= 0 && remaining > 0); + + if (bufs == null) { + if (result == null) { + return new byte[0]; + } + return result.length == total ? + result : Arrays.copyOf(result, total); + } + + result = new byte[total]; + int offset = 0; + remaining = total; + for (byte[] b : bufs) { + int count = Math.min(b.length, remaining); + System.arraycopy(b, 0, result, offset, count); + offset += count; + remaining -= count; + } + + return result; + } +} diff --git a/src/main/java/org/featurehouse/ioutils/filesplit/Core.java b/src/main/java/org/featurehouse/ioutils/filesplit/Core.java new file mode 100644 index 0000000..3be65e7 --- /dev/null +++ b/src/main/java/org/featurehouse/ioutils/filesplit/Core.java @@ -0,0 +1,155 @@ +package org.featurehouse.ioutils.filesplit; + +import org.zeroturnaround.zip.ZipUtil; + +import java.io.*; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.Path; +import java.util.UUID; +import java.util.function.IntPredicate; + +import static org.featurehouse.ioutils.filesplit.ByteHelper.fromInt; +import static org.featurehouse.ioutils.filesplit.ByteHelper.toInt; + +public class Core { + /** + * @param file can be file or directory. Zip if directory. + */ + public static void encode(File file, int maxOneFileSize, Path outputDirectory) + throws Throwable { + System.out.println("Try encoding..."); + File outputDirectoryFile = outputDirectory.toFile(); + if (outputDirectoryFile.isDirectory()) { + throw new FileAlreadyExistsException("Directory " + outputDirectoryFile.getAbsolutePath() + " already exists!"); + } else if (!outputDirectoryFile.mkdirs()) + throw new IOException("Cannot create directory tree: " + outputDirectoryFile.getAbsolutePath()); + + final boolean inputDir = file.isDirectory(); + final File originFileBck = file; + if (inputDir) { + File tmpFile = File.createTempFile("{" + UUID.randomUUID() + '}', "zip"); + tmpFile.deleteOnExit(); + ZipUtil.pack(file, tmpFile); + file = tmpFile; + } + + int i = 0; + OutputStream fileOutputStream; + InputStream inputStream = new BufferedInputStream(new FileInputStream(file)); + while (inputStream.available() != 0) { + //bytes = ByteHelper.readNBytes(inputStream, maxOneFileSize - 4); + //if (bytes.length == 0) break; + + File outputOneFile = //new File(outputDirectory + '/' + i + ".fsplit"); + outputDirectory.resolve(i + ".fsplit").toFile(); + if (!outputOneFile.createNewFile()) { + throw new IOException("cannot create new file: " + outputOneFile.getName()); + } + + fileOutputStream = new BufferedOutputStream(new FileOutputStream(outputOneFile)); + fileOutputStream.write(fromInt(Main.COMMON_HEADER)); + for (int t = 0; t < maxOneFileSize - 4; ++t) { + int b = inputStream.read(); + if (b < 0) break; + fileOutputStream.write(b); + } + //fileOutputStream.write(fromInt(Main.COMMON_HEADER)); + //fileOutputStream.write(bytes); + fileOutputStream.close(); + ++i; + } + inputStream.close(); + fileOutputStream = new BufferedOutputStream(new FileOutputStream(outputDirectory.resolve("INFO.fsplitinfo").toFile())); + fileOutputStream.write(fromInt(Main.INFO_HEADER)); // Magic number 4 + fileOutputStream.write(Main.INFO_VERSION); // fsplitinfo version 1 + fileOutputStream.write(fromInt(i)); // Split file 4 + ByteHelper.writeString(fileOutputStream, originFileBck.getName()); // Filename 4+* + + fileOutputStream.write(inputDir ? 1 : 0); // If Use Zipped Directory 1 + // TODO: support gzipped file + + + fileOutputStream.close(); + + System.out.println("Process finished. Successfully created directory " + outputDirectoryFile.getAbsolutePath()); + } + + public static void decode(File file) throws Throwable { + System.out.println("Try decoding..."); + String directoryWithSlash = file.getAbsolutePath() + '/'; + byte[] cache = new byte[4]; + int iCache; + + InputStream inputStream = new BufferedInputStream(new FileInputStream(directoryWithSlash + "INFO.fsplitinfo")); + //Fsplitinfo fsplitinfo = ByteFileExecutorKt.infoFromStream(inputStream); + iCache = inputStream.read(cache, 0, 4); + if (iCache != 4 || toInt(cache) != Main.INFO_HEADER) throw new IllegalArgumentException("Not a valid INFO file"); + int fileSplitVersion = art(inputStream.read(), i -> i < 1); + if (fileSplitVersion > Main.INFO_VERSION) throw new + UnsupportedOperationException(String.format("INFO version too high: 0x%x. Greater than supported (0x%x).", fileSplitVersion, Main.INFO_VERSION)); + art(inputStream.read(cache, 0, 4), i -> i != 4); + int maxFileCount = toInt(cache); + String newFileName = ByteHelper.readString(inputStream); + + boolean unzipDirectory = false; + if (fileSplitVersion >= 2) { + iCache = inputStream.read(); + if (iCache == 1) unzipDirectory = true; + else if (iCache != 0) throw new IllegalArgumentException("Not a valid INFO file"); + } + + + //String newFilePath = file.getParent() + '/' + newFileName; + File newFile; + if (unzipDirectory) { + newFile = File.createTempFile("{" + UUID.randomUUID() + '}', "zip"); + newFile.deleteOnExit(); + } else { + newFile = new File(file.getParentFile(), newFileName); + if (newFile.exists()) + throw new FileAlreadyExistsException(newFileName); + if (!newFile.createNewFile()) + throw new IOException("Cannot create new file: " + newFileName); + } + + OutputStream outputStream = new BufferedOutputStream(new FileOutputStream(newFile)); + + int iCache2; + for (iCache = 0; iCache < maxFileCount; ++iCache) { + //inputStream = new BufferedInputStream(new FileInputStream(file.getName() + '/' + iCache + ".fsplit")); + inputStream = new BufferedInputStream(new FileInputStream(new File(file, iCache + ".fsplit"))); + /*if (iCache2 != 4 || toInt(cache) != VersionKt.fsplitHeader) { + throw new InvalidFileException(iCache + ".fsplit", 0x00000002); + }*/ + art(inputStream.read(cache, 0, 4), i -> i != 4 || toInt(cache) != Main.COMMON_HEADER); + //cache = ByteHelper.readNBytes(inputStream, Integer.MAX_VALUE); // SEE InputStream#readAllBytes() + //outputStream.write(cache); + while ((iCache2 = inputStream.read()) >= 0) { + outputStream.write(iCache2); + } inputStream.close(); + } + outputStream.close(); + + if (unzipDirectory) { + File targetDirectory = new File(file.getParentFile(), newFileName); + if (!targetDirectory.mkdir()) { + throw new IOException("Cannot create file " + targetDirectory.getAbsolutePath()); + } + ZipUtil.unpack(newFile, targetDirectory); + newFile = targetDirectory; + } + + System.out.println("Successfully decode file at " + newFile.getAbsolutePath()); + } + + /** + * @throws IllegalArgumentException if matches {@code con}. + * @return o if does not match {@code con} + * @param con if matches, throw {@link IllegalArgumentException} + */ + static int art(int o, IntPredicate con) throws IllegalArgumentException { + if (con.test(o)) throw new IllegalArgumentException("Not a valid INFO file"); + return o; + } + +} diff --git a/src/main/java/org/featurehouse/ioutils/filesplit/Main.java b/src/main/java/org/featurehouse/ioutils/filesplit/Main.java index 1e5b1bd..4fa6905 100644 --- a/src/main/java/org/featurehouse/ioutils/filesplit/Main.java +++ b/src/main/java/org/featurehouse/ioutils/filesplit/Main.java @@ -4,16 +4,26 @@ import org.jetbrains.annotations.Nullable; import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; public class Main { + public static final String VERSION = "0.2.0"; + + public static final int COMMON_HEADER = 0x49a73b19; + public static final int INFO_HEADER = 0x49a73b1a; + public static final byte INFO_VERSION = 2; + /** * [0]: enum {encode, decode}
* [1]: filename
- * encode: --size max-one-file-size (example: 512 2G 8.6M) - * --output path/to/output/directory - * decode: --github|--gitee repo:branch(default=master):path/to/directory + * encode: --size max-one-file-size (example: 512 2G 8.6M)
+ * --output path/to/output/directory
+ * decode: --github|--gitee repo:branch(default=master):path/to/directory */ public static void main(String[] args) { + long l = System.nanoTime(); + if (args.length < 2) help(); CodecStatus codecStatus = readCodec(args[0]); if (codecStatus == null) help(); @@ -23,11 +33,43 @@ public static void main(String[] args) { System.out.println("ERROR: " + args[1] + "is not a correct " + codecStatus.getPreFileType() + '!'); System.exit(1); } + if (codecStatus == CodecStatus.DECODE) { + try { + Core.decode(file); + System.out.printf("Process finished in %.2fms.\n", (System.nanoTime() - l) / 1e6); + } catch (Throwable t) { + System.gc(); + System.err.println("An exception has occurred"); + t.printStackTrace(); + } + return; + } // else ENCODE + int maxOneFileSize = FILE_SIZE_DEFAULT; + Path outputDirectory = Paths.get(System.getProperty("user.home"), "filesplit-" + file.getName()); + for (int i = 2; i < args.length; ++i) { + if (args[i].startsWith("--")) { + if (args[i].equalsIgnoreCase("--size")) + maxOneFileSize = maxOneFileSize(args[++i]); + else if (args[i].equalsIgnoreCase("--output")) + outputDirectory = Paths.get(args[++i]); + } + } + + try { + Core.encode(file, maxOneFileSize, outputDirectory); + System.out.printf("Process finished in %.2fms.\n", (System.nanoTime() - l) / 1e6); + } catch (Throwable t) { + System.gc(); + System.err.println("An exception has occurred"); + t.printStackTrace(); + } } + protected static final int FILE_SIZE_DEFAULT = 98566144; + private static void help() { - System.out.println("Usage: java -jar FileSpliterator.jar [args]\n" + + System.out.println("Usage: java -jar FileSplit-" + VERSION + "-.jar [args]\n" + "encode: arg: filename\n" + "decode: arg: directory name\n" + "more args:\n [max_one_file_size] default 99.4MB\n" + @@ -35,6 +77,32 @@ private static void help() { System.exit(0); } + public static int maxOneFileSize(String size) { + return (int) maxOneFileSize0(size); + } + + private static float maxOneFileSize0(String string) { + if (string.endsWith("B")) string = string.substring(0, string.length() - 1); + + final String copy = string; + try { + if (string.endsWith("K")) { + string = string.substring(0, string.length() - 1); + return Float.parseFloat(string) * 1024; + } else if (string.endsWith("M")) { + string = string.substring(0, string.length() - 1); + return Float.parseFloat(string) * (1024 * 1024); + } else if (string.endsWith("G")) { + string = string.substring(0, string.length() - 1); + return Float.parseFloat(string) * (1024 * 1024 * 1024); + } return Integer.parseInt(string); + } catch (NumberFormatException e) { + System.err.println("Invalid file size: " + copy + + ". Will be set to default."); + return FILE_SIZE_DEFAULT; + } + } + private static CodecStatus readCodec(String string) { if ("encode".equalsIgnoreCase(string)) return CodecStatus.ENCODE; if ("decode".equalsIgnoreCase(string)) return CodecStatus.DECODE; @@ -44,7 +112,7 @@ private static CodecStatus readCodec(String string) { private static @Nullable File file(String string, @NotNull CodecStatus codecStatus) { File file = new File(string); if (codecStatus == CodecStatus.ENCODE) - return file.isFile() ? file : null; + return file.exists() ? file : null; else return file.isDirectory() ? file : null; }