Implement zstd Sequence Producer API

src/main/java/com/github/luben/zstd/SequenceProducer.java

@@ -0,0 +1,30 @@
+package com.github.luben.zstd;
+
+/**
+ * Interface for an extenal sequence producer. To register a sequence producer,
+ * pass an object implementing this interface to
+ * {@link ZstdCompressCtx#registerSequenceProducer(SequenceProducer)}.
+ */
+public interface SequenceProducer {
+  /**
+   * Returns a pointer to the sequence producer function. This method is called
+   * once in {@link ZstdCompressCtx#registerSequenceProducer(SequenceProducer)}.
+   * @return A function pointer of type {@code ZSTD_sequenceProducer_F *}
+   */
+  public long getFunctionPointer();
+
+  /**
+   * Allocate the sequence producer state. The returned pointer will be passed
+   * to the sequence producer function. This method is called once in
+   * {@link ZstdCompressCtx#registerSequenceProducer(SequenceProducer)}.
+   * @return A {@code void *} pointer to the sequence producer state
+   */
+  public long createState();
+
+  /**
+   * Free the sequence producer state. This method is called when closing the
+   * {@link ZstdCompressCtx} or registering a different sequence producer.
+   * @param statePointer the state pointer to be freed
+   */
+  public void freeState(long statePointer);
+}

src/main/java/com/github/luben/zstd/Zstd.java

@@ -8,10 +8,40 @@
 import com.github.luben.zstd.ZstdDecompressCtx;
 
 public class Zstd {
-
     static {
         Native.load();
     }
+
+    /**
+     * Note: This enum controls features which are conditionally beneficial.
+     * Zstd typically will make a final decision on whether or not to enable the
+     * feature ({@link AUTO}), but setting the switch to {@link ENABLE} or
+     * {@link DISABLE} allows for force enabling/disabling the feature.
+     */
+    public static enum ParamSwitch {
+        /**
+         * Let the library automatically determine whether the feature shall be enabled
+         */
+        AUTO(0),
+        /**
+         * Force-enable the feature
+         */
+        ENABLE(1),
+        /**
+         * Do not use the feature
+         */
+        DISABLE(2);
+
+        private int val;
+        ParamSwitch(int val) {
+            this.val = val;
+        }
+
+        public int getValue() {
+            return val;
+        }
+    }
+
     /**
      * Compresses buffer 'src' into buffer 'dst'.
      *
@@ -560,6 +590,10 @@ public static long decompressDirectByteBufferFastDict(ByteBuffer dst, int dstOff
     public static native int loadFastDictDecompress(long stream, ZstdDictDecompress dict);
     public static native int loadDictCompress(long stream, byte[] dict, int dict_size);
     public static native int loadFastDictCompress(long stream, ZstdDictCompress dict);
+    public static native void registerSequenceProducer(long stream, long seqProdState, long seqProdFunction);
+    static native void generateSequences(long stream, long outSeqs, long outSeqsSize, long src, long srcSize);
+    static native long getBuiltinSequenceProducer(); // Used in tests
+    static native long getStubSequenceProducer();    // Used in tests
     public static native int setCompressionChecksums(long stream, boolean useChecksums);
     public static native int setCompressionMagicless(long stream, boolean useMagicless);
     public static native int setCompressionLevel(long stream, int level);
@@ -577,6 +611,10 @@ public static long decompressDirectByteBufferFastDict(ByteBuffer dst, int dstOff
     public static native int setDecompressionLongMax(long stream, int windowLogMax);
     public static native int setDecompressionMagicless(long stream, boolean useMagicless);
     public static native int setRefMultipleDDicts(long stream, boolean useMultiple);
+    public static native int setValidateSequences(long stream, int validateSequences);
+    public static native int setSequenceProducerFallback(long stream, boolean fallbackFlag);
+    public static native int setSearchForExternalRepcodes(long stream, int searchRepcodes);
+    public static native int setEnableLongDistanceMatching(long stream, int enableLDM);
 
     /* Utility methods */
     /**

src/main/java/com/github/luben/zstd/ZstdCompressCtx.java

@@ -16,6 +16,10 @@ public class ZstdCompressCtx extends AutoCloseBase {
 
     private ZstdDictCompress compression_dict = null;
 
+    private SequenceProducer seqprod = null;
+
+    private long seqprod_state = 0;
+
     private static native long init();
 
     private static native void free(long ptr);
@@ -36,6 +40,10 @@ void doClose() {
         if (nativePtr != 0) {
             free(nativePtr);
             nativePtr = 0;
+            if (seqprod != null) {
+                seqprod.freeState(seqprod_state);
+                seqprod = null;
+            }
         }
     }
 
@@ -268,6 +276,117 @@ public ZstdCompressCtx setLong(int windowLog) {
         return this;
     }
 
+    /**
+     * Register an external sequence producer
+     * @param producer the user-defined {@link SequenceProducer} to register.
+     */
+    public ZstdCompressCtx registerSequenceProducer(SequenceProducer producer) {
+        ensureOpen();
+        acquireSharedLock();
+        try {
+            if (this.seqprod != null) {
+                this.seqprod.freeState(seqprod_state);
+                this.seqprod = null;
+            }
+
+            if (producer == null) {
+                Zstd.registerSequenceProducer(nativePtr, 0, 0);
+            } else {
+                seqprod_state = producer.createState();
+                Zstd.registerSequenceProducer(nativePtr, seqprod_state, producer.getFunctionPointer());
+                this.seqprod = producer;
+            }
+        } catch (Exception e) {
+            this.seqprod = null;
+            Zstd.registerSequenceProducer(nativePtr, 0, 0);
+            releaseSharedLock();
+            throw e;
+        } finally {
+            releaseSharedLock();
+        }
+        return this;
+    }
+
+    /**
+     * Enable or disable sequence producer fallback
+     * @param fallbackFlag fall back to the default internal sequence producer if an external
+     *                     sequence producer returns an error code, default: false
+     */
+    public ZstdCompressCtx setSequenceProducerFallback(boolean fallbackFlag) {
+        ensureOpen();
+        acquireSharedLock();
+        try {
+            long result = Zstd.setSequenceProducerFallback(nativePtr, fallbackFlag);
+            if (Zstd.isError(result)) {
+                throw new ZstdException(result);
+            }
+        } finally {
+            releaseSharedLock();
+        }
+        return this;
+    }
+
+    /**
+     * Set whether to search external sequences for repeated offsets that can be
+     * encoded as repcodes.
+     * @param searchRepcodes whether to search for repcodes
+     */
+    public ZstdCompressCtx setSearchForExternalRepcodes(Zstd.ParamSwitch searchRepcodes) {
+        ensureOpen();
+        acquireSharedLock();
+        try {
+            long result = Zstd.setSearchForExternalRepcodes(nativePtr, searchRepcodes.getValue());
+            if (Zstd.isError(result)) {
+                throw new ZstdException(result);
+            }
+        } finally {
+            releaseSharedLock();
+        }
+        return this;
+    }
+
+    /**
+     * Enable or disable sequence validation. Useful for the sequence-level API
+     * and with external sequence producers.
+     * @param validateSequences whether to enable sequence validation
+     */
+    public ZstdCompressCtx setValidateSequences(Zstd.ParamSwitch validateSequences) {
+        ensureOpen();
+        acquireSharedLock();
+        try {
+            long result = Zstd.setValidateSequences(nativePtr, validateSequences.getValue());
+            if (Zstd.isError(result)) {
+                throw new ZstdException(result);
+            }
+        } finally {
+            releaseSharedLock();
+        }
+        return this;
+    }
+
+    /**
+     * Enable or disable long-distance matching.
+     * @param ldm whether to enable long-distance matching.
+     */
+    public ZstdCompressCtx setEnableLongDistanceMatching(Zstd.ParamSwitch enableLDM) {
+        ensureOpen();
+        acquireSharedLock();
+        try {
+            long result = Zstd.setEnableLongDistanceMatching(nativePtr, enableLDM.getValue());
+            if (Zstd.isError(result)) {
+                throw new ZstdException(result);
+            }
+        } finally {
+            releaseSharedLock();
+        }
+        return this;
+    }
+
+    // Used in tests
+    long getNativePtr() {
+        return nativePtr;
+    }
+
     /**
      * Load compression dictionary to be used for subsequently compressed frames.
      *

src/main/native/jni_zstd.c

@@ -293,6 +293,69 @@ JNIEXPORT jint JNICALL Java_com_github_luben_zstd_Zstd_loadFastDictCompress
     return ZSTD_CCtx_refCDict((ZSTD_CCtx *)(intptr_t) stream, cdict);
 }
 
+size_t builtinSequenceProducer(
+  void* sequenceProducerState,
+  ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
+  const void* src, size_t srcSize,
+  const void* dict, size_t dictSize,
+  int compressionLevel,
+  size_t windowSize
+) {
+  ZSTD_CCtx *zc = (ZSTD_CCtx *)sequenceProducerState;
+  int windowLog = 0;
+  while (windowSize > 1) {
+    windowLog++;
+    windowSize >>= 1;
+  }
+  ZSTD_CCtx_setParameter(zc, ZSTD_c_compressionLevel, compressionLevel);
+  ZSTD_CCtx_setParameter(zc, ZSTD_c_windowLog, windowSize);
+  size_t numSeqs = ZSTD_generateSequences((ZSTD_CCtx *)sequenceProducerState, outSeqs, outSeqsCapacity, src, srcSize);
+  return ZSTD_isError(numSeqs) ? ZSTD_SEQUENCE_PRODUCER_ERROR : numSeqs;
+}
+
+size_t stubSequenceProducer(
+  void* sequenceProducerState,
+  ZSTD_Sequence* outSeqs, size_t outSeqsCapacity,
+  const void* src, size_t srcSize,
+  const void* dict, size_t dictSize,
+  int compressionLevel,
+  size_t windowSize
+) {
+  return ZSTD_SEQUENCE_PRODUCER_ERROR;
+}
+
+/*
+ * Class:     com_github_luben_zstd_Zstd
+ * Method:    getBuiltinSequenceProducer
+ * Signature: ()J
+ */
+JNIEXPORT jlong JNICALL Java_com_github_luben_zstd_Zstd_getBuiltinSequenceProducer
+  (JNIEnv *env, jclass obj) {
+    return (jlong)(intptr_t)&builtinSequenceProducer;
+}
+
+/*
+ * Class:     com_github_luben_zstd_Zstd
+ * Method:    getBuiltinSequenceProducer
+ * Signature: ()J
+ */
+JNIEXPORT jlong JNICALL Java_com_github_luben_zstd_Zstd_getStubSequenceProducer
+  (JNIEnv *env, jclass obj) {
+    return (jlong)(intptr_t)&stubSequenceProducer;
+}
+
+/*
+ * Class:     com_github_luben_zstd_Zstd
+ * Method:    registerSequenceProducer
+ * Signature: (JJJ)V
+ */
+JNIEXPORT void JNICALL Java_com_github_luben_zstd_Zstd_registerSequenceProducer
+  (JNIEnv *env, jclass obj, jlong stream, jlong seqProdState, jlong seqProdFunction) {
+    ZSTD_registerSequenceProducer((ZSTD_CCtx *)(intptr_t) stream,
+                                  (void *)(intptr_t) seqProdState,
+                                  (ZSTD_sequenceProducer_F *)(intptr_t) seqProdFunction);
+}
+
 /*
  * Class:     com_github_luben_zstd_Zstd
  * Method:    setCompressionChecksums
@@ -335,10 +398,10 @@ JNIEXPORT jint JNICALL Java_com_github_luben_zstd_Zstd_setCompressionLong
     ZSTD_CCtx* cctx = (ZSTD_CCtx*)(intptr_t) stream;
     if (windowLog < ZSTD_WINDOWLOG_MIN || windowLog > ZSTD_WINDOWLOG_LIMIT_DEFAULT) {
       // disable long matching and reset to default windowLog size
-      ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 0);
+      ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_disable);
       ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, 0);
     } else {
-      ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, 1);
+      ZSTD_CCtx_setParameter(cctx, ZSTD_c_enableLongDistanceMatching, ZSTD_ps_enable);
       ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, windowLog);
     }
     return 0;
@@ -477,6 +540,46 @@ JNIEXPORT jint JNICALL Java_com_github_luben_zstd_Zstd_setRefMultipleDDicts
     return ZSTD_DCtx_setParameter((ZSTD_DCtx *)(intptr_t) stream, ZSTD_d_refMultipleDDicts, value);
 }
 
+/*
+ * Class:     com_github_luben_zstd_Zstd
+ * Method:    setValidateSequences
+ * Signature: (JI)I
+ */
+JNIEXPORT jint JNICALL Java_com_github_luben_zstd_Zstd_setValidateSequences
+  (JNIEnv *env, jclass obj, jlong stream, jint validateSequences) {
+    return ZSTD_CCtx_setParameter((ZSTD_CCtx *)(intptr_t) stream, ZSTD_c_validateSequences, validateSequences);
+}
+
+/*
+ * Class:     com_github_luben_zstd_Zstd
+ * Method:    setSequenceProducerFallback
+ * Signature: (JZ)I
+ */
+JNIEXPORT jint JNICALL Java_com_github_luben_zstd_Zstd_setSequenceProducerFallback
+  (JNIEnv *env, jclass obj, jlong stream, jboolean fallbackFlag) {
+    return ZSTD_CCtx_setParameter((ZSTD_CCtx*)(intptr_t) stream, ZSTD_c_enableSeqProducerFallback, (fallbackFlag == JNI_TRUE));
+}
+
+/*
+ * Class:     com_github_luben_zstd_Zstd
+ * Method:    setSearchForExternalRepcodes
+ * Signature: (JZ)I
+ */
+JNIEXPORT jint JNICALL Java_com_github_luben_zstd_Zstd_setSearchForExternalRepcodes
+  (JNIEnv *env, jclass obj, jlong stream, jint searchRepcodes) {
+    return ZSTD_CCtx_setParameter((ZSTD_CCtx*)(intptr_t) stream, ZSTD_c_searchForExternalRepcodes, searchRepcodes);
+}
+
+/*
+ * Class:     com_github_luben_zstd_Zstd
+ * Method:    setEnableLongDistanceMatching
+ * Signature: (JZ)I
+ */
+JNIEXPORT jint JNICALL Java_com_github_luben_zstd_Zstd_setEnableLongDistanceMatching
+  (JNIEnv *env, jclass obj, jlong stream, jint enableLDM) {
+    return ZSTD_CCtx_setParameter((ZSTD_CCtx*)(intptr_t) stream, ZSTD_c_enableLongDistanceMatching, enableLDM);
+}
+
 /*
  * Class:     com_github_luben_zstd_Zstd
  * Methods:   header constants access