Update (google#555)

Update: * new CLI; bro -> brotli; + man page * JNI wrappers preparation (for bazel build) * add raw binary dictionary representation `dictionary.bin` * add ability to side-load brotli RFC dictionary * decoder persists last error now * fix `BrotliDecoderDecompress` documentation * go reader don't block until necessary * more consistent bazel target names * Java dictionary data compiled footprint reduced * Java tests refactoring
Unity-Technologies · May 13, 2023 · 9ecca84 · 9ecca84
1 parent d47e05d
commit 9ecca84
Show file tree

Hide file tree

Showing 39 changed files with 2,187 additions and 686 deletions.
diff --git a/.gitignore b/.gitignore
@@ -13,5 +13,5 @@ __pycache__/
 
 # Tests
 *.txt.uncompressed
-*.bro
-*.unbro
+*.br
+*.unbr
diff --git a/BUILD b/BUILD
@@ -9,6 +9,46 @@ licenses(["notice"])  # MIT
 
 exports_files(["LICENSE"])
 
+# >>> JNI headers
+
+config_setting(
+    name = "darwin",
+    values = {"cpu": "darwin"},
+    visibility = ["//visibility:public"],
+)
+
+config_setting(
+    name = "darwin_x86_64",
+    values = {"cpu": "darwin_x86_64"},
+    visibility = ["//visibility:public"],
+)
+
+genrule(
+    name = "copy_link_jni_header",
+    srcs = ["@openjdk_linux//:jni_h"],
+    outs = ["jni/jni.h"],
+    cmd = "cp -f $< $@",
+)
+
+genrule(
+    name = "copy_link_jni_md_header",
+    srcs = select({
+        ":darwin": ["@openjdk_macos//:jni_md_h"],
+        ":darwin_x86_64": ["@openjdk_macos//:jni_md_h"],
+        "//conditions:default": ["@openjdk_linux//:jni_md_h"],
+    }),
+    outs = ["jni/jni_md.h"],
+    cmd = "cp -f $< $@",
+)
+
+cc_library(
+    name = "jni_inc",
+    hdrs = [":jni/jni.h", ":jni/jni_md.h"],
+    includes = ["jni"],
+)
+
+# <<< JNI headers
+
 STRICT_C_OPTIONS = [
     "--pedantic-errors",
     "-Wall",
@@ -59,7 +99,7 @@ filegroup(
 )
 
 cc_library(
-    name = "brotli",
+    name = "brotli_inc",
     hdrs = [":public_headers"],
     copts = STRICT_C_OPTIONS,
     includes = ["c/include"],
@@ -70,7 +110,7 @@ cc_library(
     srcs = [":common_sources"],
     hdrs = [":common_headers"],
     copts = STRICT_C_OPTIONS,
-    deps = [":brotli"],
+    deps = [":brotli_inc"],
 )
 
 cc_library(
@@ -91,8 +131,8 @@ cc_library(
 )
 
 cc_binary(
-    name = "bro",
-    srcs = ["c/tools/bro.c"],
+    name = "brotli",
+    srcs = ["c/tools/brotli.c"],
     copts = STRICT_C_OPTIONS,
     linkstatic = 1,
     deps = [

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -181,14 +181,14 @@ if(BROTLI_PARENT_DIRECTORY)
   set(BROTLI_LIBRARIES "${BROTLI_LIBRARIES}" PARENT_SCOPE)
 endif()
 
-# Build the bro executable
-add_executable(bro c/tools/bro.c)
-target_link_libraries(bro ${BROTLI_LIBRARIES})
+# Build the brotli executable
+add_executable(brotli c/tools/brotli.c)
+target_link_libraries(brotli ${BROTLI_LIBRARIES})
 
 # Installation
 if(NOT BROTLI_BUNDLED_MODE)
   install(
-    TARGETS bro
+    TARGETS brotli
     RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}"
   )
 
@@ -243,7 +243,7 @@ if(NOT BROTLI_DISABLE_TESTS)
       add_test(NAME "${BROTLI_TEST_PREFIX}roundtrip/${INPUT}/${quality}"
         COMMAND "${CMAKE_COMMAND}"
           -DBROTLI_WRAPPER=${BROTLI_WINE}
-          -DBROTLI_CLI=$<TARGET_FILE:bro>
+          -DBROTLI_CLI=$<TARGET_FILE:brotli>
           -DQUALITY=${quality}
           -DINPUT=${INPUT_FILE}
   	  -DOUTPUT=${OUTPUT_FILE}.${quality}
@@ -260,7 +260,7 @@ if(NOT BROTLI_DISABLE_TESTS)
     add_test(NAME "${BROTLI_TEST_PREFIX}compatibility/${INPUT}"
       COMMAND "${CMAKE_COMMAND}"
         -DBROTLI_WRAPPER=${BROTLI_WINE}
-        -DBROTLI_CLI=$<TARGET_FILE:bro>
+        -DBROTLI_CLI=$<TARGET_FILE:brotli>
         -DINPUT=${CMAKE_CURRENT_SOURCE_DIR}/${INPUT}
         -P ${CMAKE_CURRENT_SOURCE_DIR}/tests/run-compatibility-test.cmake)
   endforeach()

diff --git a/MANIFEST.in b/MANIFEST.in
@@ -13,4 +13,4 @@ include python/brotlimodule.cc
 include python/README.md
 include README.md
 include setup.py
-include c/tools/bro.c
+include c/tools/brotli.c
diff --git a/Makefile b/Makefile
@@ -1,12 +1,13 @@
 OS := $(shell uname)
-LIBSOURCES = $(wildcard c/common/*.c) $(wildcard c/dec/*.c) $(wildcard c/enc/*.c)
-SOURCES = $(LIBSOURCES) c/tools/bro.c
+LIBSOURCES = $(wildcard c/common/*.c) $(wildcard c/dec/*.c) \
+             $(wildcard c/enc/*.c)
+SOURCES = $(LIBSOURCES) c/tools/brotli.c
 BINDIR = bin
 OBJDIR = $(BINDIR)/obj
 LIBOBJECTS = $(addprefix $(OBJDIR)/, $(LIBSOURCES:.c=.o))
 OBJECTS = $(addprefix $(OBJDIR)/, $(SOURCES:.c=.o))
 LIB_A = libbrotli.a
-EXECUTABLE = bro
+EXECUTABLE = brotli
 DIRS = $(OBJDIR)/c/common $(OBJDIR)/c/dec $(OBJDIR)/c/enc \
        $(OBJDIR)/c/tools $(BINDIR)/tmp
 CFLAGS += -O2

diff --git a/WORKSPACE b/WORKSPACE
@@ -13,6 +13,38 @@ git_repository(
     remote = "https://github.com/bazelbuild/rules_go.git",
     tag = "0.4.1",
 )
-load("@io_bazel_rules_go//go:def.bzl", "go_repositories")
 
+new_http_archive(
+    name = "openjdk_linux",
+    url = "https://bazel-mirror.storage.googleapis.com/openjdk/azul-zulu-8.20.0.5-jdk8.0.121/zulu8.20.0.5-jdk8.0.121-linux_x64.tar.gz",
+    sha256 = "7fdfb17d890406470b2303d749d3138e7f353749e67a0a22f542e1ab3e482745",
+    build_file_content = """
+package(
+    default_visibility = ["//visibility:public"],
+)
+filegroup(
+    name = "jni_h",
+    srcs = ["zulu8.20.0.5-jdk8.0.121-linux_x64/include/jni.h"],
+)
+filegroup(
+    name = "jni_md_h",
+    srcs = ["zulu8.20.0.5-jdk8.0.121-linux_x64/include/linux/jni_md.h"],
+)""",
+)
+
+new_http_archive(
+    name = "openjdk_macos",
+    url = "https://bazel-mirror.storage.googleapis.com/openjdk/azul-zulu-8.20.0.5-jdk8.0.121/zulu8.20.0.5-jdk8.0.121-macosx_x64.zip",
+    sha256 = "2a58bd1d9b0cbf0b3d8d1bcdd117c407e3d5a0ec01e2f53565c9bec5cf9ea78b",
+    build_file_content = """
+package(
+    default_visibility = ["//visibility:public"],
+)
+filegroup(
+    name = "jni_md_h",
+    srcs = ["zulu8.20.0.5-jdk8.0.121-macosx_x64/include/darwin/jni_md.h"],
+)""",
+)
+
+load("@io_bazel_rules_go//go:def.bzl", "go_repositories")
 go_repositories()
diff --git a/c/common/dictionary.bin b/c/common/dictionary.bin
diff --git a/c/common/dictionary.c b/c/common/dictionary.c
@@ -10,24 +10,8 @@
 extern "C" {
 #endif
 
-static const BrotliDictionary kBrotliDictionary = {
-  /* size_bits_by_length */
-  {
-    0, 0, 0, 0, 10, 10, 11, 11,
-    10, 10, 10, 10, 10, 9, 9, 8,
-    7, 7, 8, 7, 7, 6, 6, 5,
-    5, 0, 0, 0, 0, 0, 0, 0
-  },
-
-  /* offsets_by_length */
-  {
-    0, 0, 0, 0, 0, 4096, 9216, 21504,
-    35840, 44032, 53248, 63488, 74752, 87040, 93696, 100864,
-    104704, 106752, 108928, 113536, 115968, 118528, 119872, 121280,
-    122016, 122784, 122784, 122784, 122784, 122784, 122784, 122784
-  },
-
-  /* data */
+#ifndef BROTLI_EXTERNAL_DICTIONARY_DATA
+static const uint8_t kBrotliDictionaryData[] =
 {
 116,105,109,101,100,111,119,110,108,105,102,101,108,101,102,116,98,97,99,107,99,
 111,100,101,100,97,116,97,115,104,111,119,111,110,108,121,115,105,116,101,99,105
@@ -5875,12 +5859,47 @@ static const BrotliDictionary kBrotliDictionary = {
 ,164,181,224,164,190,224,164,136,224,164,184,224,164,149,224,165,141,224,164,176
 ,224,164,191,224,164,175,224,164,164,224,164,190
 }
+;
+#endif  /* !BROTLI_EXTERNAL_DICTIONARY_DATA */
+
+static BrotliDictionary kBrotliDictionary = {
+  /* size_bits_by_length */
+  {
+    0, 0, 0, 0, 10, 10, 11, 11,
+    10, 10, 10, 10, 10, 9, 9, 8,
+    7, 7, 8, 7, 7, 6, 6, 5,
+    5, 0, 0, 0, 0, 0, 0, 0
+  },
+
+  /* offsets_by_length */
+  {
+    0, 0, 0, 0, 0, 4096, 9216, 21504,
+    35840, 44032, 53248, 63488, 74752, 87040, 93696, 100864,
+    104704, 106752, 108928, 113536, 115968, 118528, 119872, 121280,
+    122016, 122784, 122784, 122784, 122784, 122784, 122784, 122784
+  },
+
+  /* data_size ==  sizeof(kBrotliDictionaryData) */
+  122784,
+
+  /* data */
+#ifdef BROTLI_EXTERNAL_DICTIONARY_DATA
+  NULL
+#else
+  kBrotliDictionaryData
+#endif
 };
 
 const BrotliDictionary* BrotliGetDictionary() {
   return &kBrotliDictionary;
 }
 
+void BrotliSetDictionaryData(const uint8_t* data) {
+  if (!!data && !kBrotliDictionary.data) {
+    kBrotliDictionary.data = data;
+  }
+}
+
 #if defined(__cplusplus) || defined(c_plusplus)
 }  /* extern "C" */
 #endif
diff --git a/c/common/dictionary.h b/c/common/dictionary.h
@@ -27,19 +27,33 @@ typedef struct BrotliDictionary {
    * Dictionary consists of words with length of [4..24] bytes.
    * Values at [0..3] and [25..31] indices should not be addressed.
    */
-  uint8_t size_bits_by_length[32];
+  const uint8_t size_bits_by_length[32];
 
   /* assert(offset[i + 1] == offset[i] + (bits[i] ? (i << bits[i]) : 0)) */
-  uint32_t offsets_by_length[32];
+  const uint32_t offsets_by_length[32];
+
+  /* assert(data_size == offsets_by_length[31]) */
+  const size_t data_size;
 
   /* Data array is not bound, and should obey to size_bits_by_length values.
-     Specified size matches default (RFC 7932) dictionary. */
-  /* assert(sizeof(data) == offsets_by_length[31]) */
-  uint8_t data[122784];
+     Specified size matches default (RFC 7932) dictionary. Its size is
+     defined by data_size */
+  const uint8_t* data;
 } BrotliDictionary;
 
 BROTLI_COMMON_API extern const BrotliDictionary* BrotliGetDictionary(void);
 
+/**
+ * Sets dictionary data.
+ *
+ * When dictionary data is already set / present, this method is no-op.
+ *
+ * Dictionary data MUST be provided before BrotliGetDictionary is invoked.
+ * This method is used ONLY in multi-client environment (e.g. C + Java),
+ * to reduce storage by sharing single dictionary between implementations.
+ */
+BROTLI_COMMON_API void BrotliSetDictionaryData(const uint8_t* data);
+
 #define BROTLI_MIN_DICTIONARY_WORD_LENGTH 4
 #define BROTLI_MAX_DICTIONARY_WORD_LENGTH 24
 

diff --git a/c/dec/decode.c b/c/dec/decode.c
@@ -66,7 +66,6 @@ BrotliDecoderState* BrotliDecoderCreateInstance(
   }
   BrotliDecoderStateInitWithCustomAllocators(
       state, alloc_func, free_func, opaque);
-  state->error_code = BROTLI_DECODER_NO_ERROR;
   return state;
 }
 
@@ -1747,6 +1746,9 @@ static BROTLI_INLINE BrotliDecoderErrorCode ProcessCommandsInternal(
       /* Compensate double distance-ring-buffer roll. */
       s->dist_rb_idx += s->distance_context;
       offset += word_idx * i;
+      if (BROTLI_PREDICT_FALSE(!s->dictionary->data)) {
+        return BROTLI_FAILURE(BROTLI_DECODER_ERROR_DICTIONARY_NOT_SET);
+      }
       if (transform_idx < kNumTransforms) {
         const uint8_t* word = &s->dictionary->data[offset];
         int len = i;
@@ -1899,6 +1901,10 @@ BrotliDecoderResult BrotliDecoderDecompressStream(
     size_t* available_out, uint8_t** next_out, size_t* total_out) {
   BrotliDecoderErrorCode result = BROTLI_DECODER_SUCCESS;
   BrotliBitReader* br = &s->br;
+  /* Do not try to process further in a case of unrecoverable error. */
+  if ((int)s->error_code < 0) {
+    return BROTLI_DECODER_RESULT_ERROR;
+  }
   if (*available_out && (!next_out || !*next_out)) {
     return SaveErrorCode(
         s, BROTLI_FAILURE(BROTLI_DECODER_ERROR_INVALID_ARGUMENTS));

diff --git a/c/dec/state.c b/c/dec/state.c
@@ -41,6 +41,8 @@ void BrotliDecoderStateInitWithCustomAllocators(BrotliDecoderState* s,
     s->memory_manager_opaque = opaque;
   }
 
+  s->error_code = 0; /* BROTLI_DECODER_NO_ERROR */
+
   BrotliInitBitReader(&s->br);
   s->state = BROTLI_STATE_UNINITED;
   s->substate_metablock_header = BROTLI_STATE_METABLOCK_HEADER_NONE;

diff --git a/c/include/brotli/decode.h b/c/include/brotli/decode.h
@@ -84,8 +84,9 @@ typedef enum {
   BROTLI_ERROR_CODE(_ERROR_FORMAT_, PADDING_1, -14) SEPARATOR              \
   BROTLI_ERROR_CODE(_ERROR_FORMAT_, PADDING_2, -15) SEPARATOR              \
                                                                            \
-  /* -16..-19 codes are reserved */                                        \
+  /* -16..-18 codes are reserved */                                        \
                                                                            \
+  BROTLI_ERROR_CODE(_ERROR_, DICTIONARY_NOT_SET, -19) SEPARATOR            \
   BROTLI_ERROR_CODE(_ERROR_, INVALID_ARGUMENTS, -20) SEPARATOR             \
                                                                            \
   /* Memory allocation problems */                                         \
@@ -207,9 +208,9 @@ BROTLI_DEC_API BrotliDecoderResult BrotliDecoderDecompress(
  *          allocation failed, arguments were invalid, etc.;
  *          use ::BrotliDecoderGetErrorCode to get detailed error code
  * @returns ::BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT decoding is blocked until
- *          more output space is provided
- * @returns ::BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT decoding is blocked until
  *          more input data is provided
+ * @returns ::BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT decoding is blocked until
+ *          more output space is provided
  * @returns ::BROTLI_DECODER_RESULT_SUCCESS decoding is finished, no more
  *          input might be consumed and no more output will be produced
  */