Core

.gitignore

@@ -0,0 +1,7 @@
+.idea/
+.vscode/
+out/
+target/
+*.iml
+*.ipr
+*.iws

pom.xml

@@ -21,10 +21,53 @@
             <groupId>org.jetbrains</groupId>
             <artifactId>annotations</artifactId>
             <version>21.0.1</version>
+            <scope>provided</scope>
             <optional>true</optional>
         </dependency>
+        <dependency>
+            <groupId>org.zeroturnaround</groupId>
+            <artifactId>zt-zip</artifactId>
+            <version>1.14</version>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-nop</artifactId>
+            <version>1.7.30</version>
+            <scope>compile</scope>
+        </dependency>
     </dependencies>
 
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-assembly-plugin</artifactId>
+                <version>3.3.0</version>
+                <configuration>
+                    <appendAssemblyId>false</appendAssemblyId>
+                    <descriptorRefs>
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                    </descriptorRefs>
+                    <archive>
+                        <manifest>
+                            <mainClass>org.featurehouse.ioutils.filesplit.Main</mainClass>
+                        </manifest>
+                    </archive>
+                </configuration>
+                <executions>
+                    <execution>
+                        <id>make-assembly</id>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>single</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
     <licenses>
         <license>
             <name>The Apache Software License, Version 2.0</name>

src/main/java/org/featurehouse/ioutils/filesplit/ByteHelper.java

@@ -0,0 +1,156 @@
+package org.featurehouse.ioutils.filesplit;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+public class ByteHelper {
+    public static byte[] fromInt(int i) {
+        return new byte[] {
+                (byte) (i >> 24),
+                (byte) (i >> 16),
+                (byte) (i >> 8),
+                (byte)  i
+        };
+    }
+
+    public static int toInt(byte[] bs) {
+        if (bs.length < 4) return -1;
+        int[] intArray = new int[4];
+        for (int i = 0; i < 4; ++i) {
+            if (bs[i] < 0) intArray[i] = 256 + bs[i];
+            else intArray[i] = bs[i];
+        }
+        return  (intArray[0] << 24) +
+                (intArray[1] << 16) +
+                (intArray[2] << 8)  +
+                 intArray[3];
+    }
+
+    public static void writeString(OutputStream instance, String s) throws IOException {
+        byte[] b = s.getBytes(StandardCharsets.UTF_8);
+        instance.write(fromInt(b.length));
+        instance.write(b);
+    }
+
+    public static String readString(InputStream instance)
+            throws IOException, IllegalArgumentException {
+        byte[] b = new byte[4];
+        Core.art(instance.read(b, 0, 4), i -> i != 4);
+        int l = toInt(b);   // len
+        b = new byte[l];
+        Core.art(instance.read(b, 0, l), i -> i != l);
+        return new String(b, StandardCharsets.UTF_8);
+    }
+
+    /**
+     * Reads up to a specified number of bytes from the input stream. This
+     * method blocks until the requested number of bytes have been read, end
+     * of stream is detected, or an exception is thrown. This method does not
+     * close the input stream.
+     *
+     * <p> The length of the returned array equals the number of bytes read
+     * from the stream. If {@code len} is zero, then no bytes are read and
+     * an empty byte array is returned. Otherwise, up to {@code len} bytes
+     * are read from the stream. Fewer than {@code len} bytes may be read if
+     * end of stream is encountered.
+     *
+     * <p> When this stream reaches end of stream, further invocations of this
+     * method will return an empty byte array.
+     *
+     * <p> Note that this method is intended for simple cases where it is
+     * convenient to read the specified number of bytes into a byte array. The
+     * total amount of memory allocated by this method is proportional to the
+     * number of bytes read from the stream which is bounded by {@code len}.
+     * Therefore, the method may be safely called with very large values of
+     * {@code len} provided sufficient memory is available.
+     *
+     * <p> The behavior for the case where the input stream is <i>asynchronously
+     * closed</i>, or the thread interrupted during the read, is highly input
+     * stream specific, and therefore not specified.
+     *
+     * <p> If an I/O error occurs reading from the input stream, then it may do
+     * so after some, but not all, bytes have been read. Consequently the input
+     * stream may not be at end of stream and may be in an inconsistent state.
+     * It is strongly recommended that the stream be promptly closed if an I/O
+     * error occurs.
+     *
+     * <p> Since Java API 11.
+     *
+     * @implNote
+     * The number of bytes allocated to read data from this stream and return
+     * the result is bounded by {@code 2*(long)len}, inclusive.
+     *
+     * @param len the maximum number of bytes to read
+     * @return a byte array containing the bytes read from this input stream
+     * @throws IllegalArgumentException if {@code length} is negative
+     * @throws IOException if an I/O error occurs
+     * @throws OutOfMemoryError if an array of the required size cannot be
+     *         allocated.
+     *
+     */
+    public static byte[] readNBytes(InputStream instance, int len) throws IOException {
+        if (len < 0) {
+            throw new IllegalArgumentException("len < 0");
+        }
+
+        List<byte[]> bufs = null;
+        byte[] result = null;
+        int total = 0;
+        int remaining = len;
+        int n;
+        do {
+            byte[] buf = new byte[Math.min(remaining, 8192)];
+            int nread = 0;
+
+            // read to EOF which may read more or less than buffer size
+            while ((n = instance.read(buf, nread,
+                    Math.min(buf.length - nread, remaining))) > 0) {
+                nread += n;
+                remaining -= n;
+            }
+
+            if (nread > 0) {
+                if (2147483639 - total < nread) {
+                    throw new OutOfMemoryError("Required array size too large");
+                }
+                total += nread;
+                if (result == null) {
+                    result = buf;
+                } else {
+                    if (bufs == null) {
+                        bufs = new ArrayList<>();
+                        bufs.add(result);
+                    }
+                    bufs.add(buf);
+                }
+            }
+            // if the last call to read returned -1 or the number of bytes
+            // requested have been read then break
+        } while (n >= 0 && remaining > 0);
+
+        if (bufs == null) {
+            if (result == null) {
+                return new byte[0];
+            }
+            return result.length == total ?
+                    result : Arrays.copyOf(result, total);
+        }
+
+        result = new byte[total];
+        int offset = 0;
+        remaining = total;
+        for (byte[] b : bufs) {
+            int count = Math.min(b.length, remaining);
+            System.arraycopy(b, 0, result, offset, count);
+            offset += count;
+            remaining -= count;
+        }
+
+        return result;
+    }
+}

src/main/java/org/featurehouse/ioutils/filesplit/Core.java

@@ -0,0 +1,155 @@
+package org.featurehouse.ioutils.filesplit;
+
+import org.zeroturnaround.zip.ZipUtil;
+
+import java.io.*;
+import java.nio.file.FileAlreadyExistsException;
+import java.nio.file.Path;
+import java.util.UUID;
+import java.util.function.IntPredicate;
+
+import static org.featurehouse.ioutils.filesplit.ByteHelper.fromInt;
+import static org.featurehouse.ioutils.filesplit.ByteHelper.toInt;
+
+public class Core {
+    /**
+     * @param file can be file or directory. Zip if directory.
+     */
+    public static void encode(File file, int maxOneFileSize, Path outputDirectory)
+            throws Throwable {
+        System.out.println("Try encoding...");
+        File outputDirectoryFile = outputDirectory.toFile();
+        if (outputDirectoryFile.isDirectory()) {
+            throw new FileAlreadyExistsException("Directory " + outputDirectoryFile.getAbsolutePath() + " already exists!");
+        } else if (!outputDirectoryFile.mkdirs())
+            throw new IOException("Cannot create directory tree: " + outputDirectoryFile.getAbsolutePath());
+
+        final boolean inputDir = file.isDirectory();
+        final File originFileBck = file;
+        if (inputDir) {
+            File tmpFile = File.createTempFile("{" + UUID.randomUUID() + '}', "zip");
+            tmpFile.deleteOnExit();
+            ZipUtil.pack(file, tmpFile);
+            file = tmpFile;
+        }
+
+        int i = 0;
+        OutputStream fileOutputStream;
+        InputStream inputStream = new BufferedInputStream(new FileInputStream(file));
+        while (inputStream.available() != 0) {
+            //bytes = ByteHelper.readNBytes(inputStream, maxOneFileSize - 4);
+            //if (bytes.length == 0) break;
+
+            File outputOneFile = //new File(outputDirectory + '/' + i + ".fsplit");
+                    outputDirectory.resolve(i + ".fsplit").toFile();
+            if (!outputOneFile.createNewFile()) {
+                throw new IOException("cannot create new file: " + outputOneFile.getName());
+            }
+
+            fileOutputStream = new BufferedOutputStream(new FileOutputStream(outputOneFile));
+            fileOutputStream.write(fromInt(Main.COMMON_HEADER));
+            for (int t = 0; t < maxOneFileSize - 4; ++t) {
+                int b = inputStream.read();
+                if (b < 0) break;
+                fileOutputStream.write(b);
+            }
+            //fileOutputStream.write(fromInt(Main.COMMON_HEADER));
+            //fileOutputStream.write(bytes);
+            fileOutputStream.close();
+            ++i;
+        }
+        inputStream.close();
+        fileOutputStream = new BufferedOutputStream(new FileOutputStream(outputDirectory.resolve("INFO.fsplitinfo").toFile()));
+        fileOutputStream.write(fromInt(Main.INFO_HEADER));                      // Magic number 4
+        fileOutputStream.write(Main.INFO_VERSION);                              // fsplitinfo version 1
+        fileOutputStream.write(fromInt(i));                                     // Split file 4
+        ByteHelper.writeString(fileOutputStream, originFileBck.getName());    // Filename 4+*
+
+        fileOutputStream.write(inputDir ? 1 : 0);                               // If Use Zipped Directory 1
+        // TODO: support gzipped file
+
+
+        fileOutputStream.close();
+
+        System.out.println("Process finished. Successfully created directory " + outputDirectoryFile.getAbsolutePath());
+    }
+
+    public static void decode(File file) throws Throwable {
+        System.out.println("Try decoding...");
+        String directoryWithSlash = file.getAbsolutePath() + '/';
+        byte[] cache = new byte[4];
+        int iCache;
+
+        InputStream inputStream = new BufferedInputStream(new FileInputStream(directoryWithSlash + "INFO.fsplitinfo"));
+        //Fsplitinfo fsplitinfo = ByteFileExecutorKt.infoFromStream(inputStream);
+        iCache = inputStream.read(cache, 0, 4);
+        if (iCache != 4 || toInt(cache) != Main.INFO_HEADER) throw new IllegalArgumentException("Not a valid INFO file");
+        int fileSplitVersion = art(inputStream.read(), i -> i < 1);
+        if (fileSplitVersion > Main.INFO_VERSION) throw new
+                UnsupportedOperationException(String.format("INFO version too high: 0x%x. Greater than supported (0x%x).", fileSplitVersion, Main.INFO_VERSION));
+        art(inputStream.read(cache, 0, 4), i -> i != 4);
+        int maxFileCount = toInt(cache);
+        String newFileName = ByteHelper.readString(inputStream);
+
+        boolean unzipDirectory = false;
+        if (fileSplitVersion >= 2) {
+            iCache = inputStream.read();
+            if (iCache == 1) unzipDirectory = true;
+            else if (iCache != 0) throw new IllegalArgumentException("Not a valid INFO file");
+        }
+
+
+        //String newFilePath = file.getParent() + '/' + newFileName;
+        File newFile;
+        if (unzipDirectory) {
+            newFile = File.createTempFile("{" + UUID.randomUUID() + '}', "zip");
+            newFile.deleteOnExit();
+        } else {
+            newFile = new File(file.getParentFile(), newFileName);
+            if (newFile.exists())
+                throw new FileAlreadyExistsException(newFileName);
+            if (!newFile.createNewFile())
+                throw new IOException("Cannot create new file: " + newFileName);
+        }
+
+        OutputStream outputStream = new BufferedOutputStream(new FileOutputStream(newFile));
+
+        int iCache2;
+        for (iCache = 0; iCache < maxFileCount; ++iCache) {
+            //inputStream = new BufferedInputStream(new FileInputStream(file.getName() + '/' + iCache + ".fsplit"));
+            inputStream = new BufferedInputStream(new FileInputStream(new File(file, iCache + ".fsplit")));
+            /*if (iCache2 != 4 || toInt(cache) != VersionKt.fsplitHeader) {
+                throw new InvalidFileException(iCache + ".fsplit", 0x00000002);
+            }*/
+            art(inputStream.read(cache, 0, 4), i -> i != 4 || toInt(cache) != Main.COMMON_HEADER);
+            //cache = ByteHelper.readNBytes(inputStream, Integer.MAX_VALUE);  // SEE InputStream#readAllBytes()
+            //outputStream.write(cache);
+            while ((iCache2 = inputStream.read()) >= 0) {
+                outputStream.write(iCache2);
+            } inputStream.close();
+        }
+        outputStream.close();
+
+        if (unzipDirectory) {
+            File targetDirectory = new File(file.getParentFile(), newFileName);
+            if (!targetDirectory.mkdir()) {
+                throw new IOException("Cannot create file " + targetDirectory.getAbsolutePath());
+            }
+            ZipUtil.unpack(newFile, targetDirectory);
+            newFile = targetDirectory;
+        }
+
+        System.out.println("Successfully decode file at " + newFile.getAbsolutePath());
+    }
+
+    /**
+     * @throws IllegalArgumentException if matches {@code con}.
+     * @return o if does not match {@code con}
+     * @param con if matches, throw {@link IllegalArgumentException}
+     */
+    static int art(int o, IntPredicate con) throws IllegalArgumentException {
+        if (con.test(o)) throw new IllegalArgumentException("Not a valid INFO file");
+        return o;
+    }
+
+}