kram - added ZipStream, added fallthrough tests

Author: alecazam

Diff for: build2/kram.xcodeproj/project.pbxproj

@@ -171,6 +171,7 @@
 				CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
 				CLANG_WARN_EMPTY_BODY = YES;
 				CLANG_WARN_ENUM_CONVERSION = YES;
+				CLANG_WARN_IMPLICIT_FALLTHROUGH = YES_ERROR;
 				CLANG_WARN_INFINITE_RECURSION = YES;
 				CLANG_WARN_INT_CONVERSION = YES;
 				CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
@@ -236,6 +237,7 @@
 				CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
 				CLANG_WARN_EMPTY_BODY = YES;
 				CLANG_WARN_ENUM_CONVERSION = YES;
+				CLANG_WARN_IMPLICIT_FALLTHROUGH = YES_ERROR;
 				CLANG_WARN_INFINITE_RECURSION = YES;
 				CLANG_WARN_INT_CONVERSION = YES;
 				CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;

Diff for: build2/kramc.xcodeproj/project.pbxproj

@@ -537,6 +537,7 @@
 				CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
 				CLANG_WARN_EMPTY_BODY = YES;
 				CLANG_WARN_ENUM_CONVERSION = YES;
+				CLANG_WARN_IMPLICIT_FALLTHROUGH = YES_ERROR;
 				CLANG_WARN_INFINITE_RECURSION = YES;
 				CLANG_WARN_INT_CONVERSION = YES;
 				CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;
@@ -610,6 +611,7 @@
 				CLANG_WARN_DOCUMENTATION_COMMENTS = YES;
 				CLANG_WARN_EMPTY_BODY = YES;
 				CLANG_WARN_ENUM_CONVERSION = YES;
+				CLANG_WARN_IMPLICIT_FALLTHROUGH = YES_ERROR;
 				CLANG_WARN_INFINITE_RECURSION = YES;
 				CLANG_WARN_INT_CONVERSION = YES;
 				CLANG_WARN_NON_LITERAL_NULL_CONVERSION = YES;

Diff for: build2/kramv.xcodeproj/project.pbxproj

@@ -4,14 +4,17 @@
 //#include <iterator> // for copy_if on Win
 #include <vector>
 #include <string>
+#include <unordered_map>
+#include <mutex>
 
 #include "miniz.h"
 
 // test for perf of this compared to one in miniz also see
 // comments about faster algs.
 // libcompress can only encode lvl 5, but here it's only decompress.
+// This seems to fail when used for kramv zip archives, so disable fo now
 #ifndef USE_LIBCOMPRESSION
-#define USE_LIBCOMPRESSION (KRAM_MAC || KRAM_IOS)
+#define USE_LIBCOMPRESSION 0 // (KRAM_MAC || KRAM_IOS)
 #endif
 
 #if USE_LIBCOMPRESSION
@@ -21,8 +24,6 @@
 // Throwing this in for now, since it's the only .cpp file
 #if KRAM_MAC || KRAM_IOS
 #include <cxxabi.h> // demangle
-#include <unordered_map>
-#include <mutex>
 #endif
 
 using namespace NAMESPACE_STL;
@@ -407,11 +408,12 @@ bool ZipHelper::extract(const ZipEntry& entry, void* buffer, uint64_t bufferSize
         return false;
     }
     // need to extra data and header
+    char scratchBuffer[compression_decode_scratch_buffer_size(COMPRESSION_ZLIB)];
 
     uint64_t bytesDecoded = compression_decode_buffer(
         (uint8_t*)buffer, entry.uncompressedSize,
         (const uint8_t*)data, entry.compressedSize,
-        NULL, // scratch-buffer that could speed up to pass
+        scratchBuffer,
         COMPRESSION_ZLIB);
 
     bool success = false;

Diff for: kram-profile/Source/KramZipHelper.cpp

@@ -37,12 +37,18 @@
 
 #if ASTCENC_NEON
 
-    #define ASTCENC_POPCNT 0
-    #define ASTCENC_F16C 0
+    // Intel simd ops
     #define ASTCENC_SSE 0
     #define ASTCENC_AVX 0
+
+    // Keep alignment at 16B
     #define ASTCENC_VECALIGN 16
 
+    // These have equivalents in Neon
+    #define ASTCENC_POPCNT 0
+    #define ASTCENC_F16C 0
+
+
 #else
 
     #ifndef ASTCENC_SSE
@@ -69,6 +75,7 @@
       #endif
     #endif
 
+    // must set -fpopcount
     #ifndef ASTCENC_POPCNT
       #if defined(__POPCNT__)
         #define ASTCENC_POPCNT 1
@@ -77,8 +84,8 @@
       #endif
     #endif
 
+    // must set -mf16c only on x86_64 build, avx not enough on clang
     #ifndef ASTCENC_F16C
-      // must set -mf16c only on x86_64 build, avx not enough on clang
       #if defined(__F16C__)
         #define ASTCENC_F16C 1
       #else

Diff for: libkram/astc-encoder/astcenc_mathlib.h

@@ -210,6 +210,7 @@ int lerp(int a, int b, int i, int denom) {
     case 3:
         denom *= 5;
         i *= 5;    // fall through to case 15
+        [[fallthrough]];
     case 15:
         weights = g_aWeights4;
         break;

Diff for: libkram/compressonator/bc6h/bc6h_decode.cpp

@@ -72,6 +72,7 @@ float lerpf(float a, float b, int i, int denom) {
     case 3:
         denom *= 5;
         i *= 5;    // fall through to case 15
+        [[fallthrough]];
     case 7:
         weights = g_aWeights3;
         break;

Diff for: libkram/compressonator/bc6h/hdr_encode.cpp

@@ -28,17 +28,19 @@
 
 // not including this in KramConfig.h - used for pool
 #include "BlockedLinearAllocator.h"
+#include "KramZipStream.h"
+
 
 // Heavily modifed by Alec Miller 10/1/23
 // This codebase was frozen by DropBox with very little effort put into it.
 // And I liked the readability of the code.  Optimized with ImmutableStrings
 // and a BlockedLinearAllocator.
 //
-// This is DOM reader/writer.  Building up stl data structures in a DOM
-// to write isn't great memory wise.  May move to a SAX writer.
+// json11 is DOM reader/writer.  Building up stl data structures in a DOM
+// to write isn't great memory wise.  Moved to custom SAX writer.
 // Times to read font atlas file on M1 MBP 14".  1/21/24
 //
-// json11
+// json11 reader
 // Release - parsed 101 KB of json using 576 KB of memory in 14.011ms
 // Debug   - parsed 101 KB of json using 576 KB of memory in 26.779ms
 //
@@ -128,19 +130,18 @@ const char* JsonWriter::escapedString(const char* str)
 
     return _escapedString.c_str();
 }
-  
+
 void JsonWriter::pushObject(const char* key) {
     if (key[0])
     {
         KASSERT(isObject());
         writeCommaAndNewline();
         int indent = _stack.size();
-        sprintf(*_out, "%*s\"%s\":{\n", indent, "", key);
+        writeFormat("%*s\"%s\":{\n", indent, "", key);
     }
     else
     {
-        _out->push_back('{');
-        _out->push_back('\n');
+        writeFormat("{\n");
     }
     _stack.push_back('}');
     _isFirst.push_back(false);
@@ -151,12 +152,11 @@ void JsonWriter::pushArray(const char* key) {
         KASSERT(isObject());
         writeCommaAndNewline();
         int indent = _stack.size();
-        sprintf(*_out, "%*s\"%s\":[\n", indent, "", key);
+        writeFormat("%*s\"%s\":[\n", indent, "", key);
     }
     else
     {
-        _out->push_back('[');
-        _out->push_back('\n');
+        writeFormat("[\n");
     }
     _stack.push_back(']');
     _isFirst.push_back(false);
@@ -166,8 +166,7 @@ void JsonWriter::pop() {
     KASSERT(_stack.empty());
     char c = _stack.back();
 
-    _out->push_back(c);
-    _out->push_back('\n');
+    writeFormat("%c\n", c);
 
     _stack.pop_back();
     _isFirst.pop_back();
@@ -193,87 +192,117 @@ void JsonWriter::writeString(const char* key, const char* value) {
     KASSERT(isObject());
     writeCommaAndNewline();
     int indent = _stack.size();
-    append_sprintf(*_out, "%*s\"%s\":\"%s\"", indent, "", key, escapedString(value));
+    writeFormat("%*s\"%s\":\"%s\"", indent, "", key, escapedString(value));
 }
 void JsonWriter::writeDouble(const char* key, double value) {
     KASSERT(isObject());
     writeCommaAndNewline();
     int indent = _stack.size();
-    append_sprintf(*_out, "%*s\"%s\":%f", indent, "", key, value);
+    writeFormat("%*s\"%s\":%f", indent, "", key, value);
 }
 void JsonWriter::writeInt32(const char* key, int32_t value) {
     KASSERT(isObject());
     writeCommaAndNewline();
     int indent = _stack.size();
-    append_sprintf(*_out, "%*s\"%s\":\"%d\"", indent, "", key, value);
+    writeFormat("%*s\"%s\":\"%d\"", indent, "", key, value);
 
 }
 void JsonWriter::writeBool(const char* key, bool value) {
     KASSERT(isObject());
     writeCommaAndNewline();
     int indent = _stack.size();
-    append_sprintf(*_out, "%*s\"%s\":%s", indent, "", key, value ? "true" : "false");
+    writeFormat("%*s\"%s\":%s", indent, "", key, value ? "true" : "false");
 }
 void JsonWriter::writeNull(const char* key) {
     KASSERT(isObject());
     writeCommaAndNewline();
     int indent = _stack.size();
-    append_sprintf(*_out, "%*s\"%s\":%s", indent, "", key, "null");
+    writeFormat("%*s\"%s\":%s", indent, "", key, "null");
 }
 
 // can write out json in parallel and combine
-void JsonWriter::writeJson(const JsonWriter& json)
-{
+void JsonWriter::writeJson(const JsonWriter& json) {
     KASSERT(_stack.empty());
     KASSERT(this != &json);
 
     // TODO: indent won't be correct on this
     // so caller may want to set indent
-    _out->append(*json._out);
+    writeFormat("%s", json._out->c_str());
 }
 
 void JsonWriter::writeString(const char* value) {
     KASSERT(isArray());
     // only if in array
     writeCommaAndNewline();
     int indent = _stack.size();
-    append_sprintf(*_out, "%*s\"%s\"", indent, "", escapedString(value));
+    writeFormat("%*s\"%s\"", indent, "", escapedString(value));
 }
 void JsonWriter::writeDouble(double value) {
     KASSERT(isArray());
     writeCommaAndNewline();
     int indent = _stack.size();
-    append_sprintf(*_out, "%*s%f", indent, "", value);
+    writeFormat("%*s%f", indent, "", value);
 }
 void JsonWriter::writeInt32(int32_t value) {
     KASSERT(isArray());
     writeCommaAndNewline();
     int indent = _stack.size();
-    append_sprintf(*_out, "%*s\"%d\"", indent, "", value);
+    writeFormat("%*s\"%d\"", indent, "", value);
 }
 void JsonWriter::writeBool(bool value) {
     KASSERT(isArray());
     writeCommaAndNewline();
     int indent = _stack.size();
-    append_sprintf(*_out, "%*s%s", indent, "", value ? "true" : "false");
+    writeFormat("%*s%s", indent, "", value ? "true" : "false");
 }
 void JsonWriter::writeNull() {
     KASSERT(isArray());
     writeCommaAndNewline();
     int indent = _stack.size();
-    append_sprintf(*_out, "%*s%s", indent, "", "null");
+    writeFormat("%*s%s", indent, "", "null");
 }
 
 void JsonWriter::writeCommaAndNewline() {
     bool isFirst = _isFirst.back();
     if (!isFirst)
-        _out->push_back(',');
-    _out->push_back('\n');
+        writeFormat(",\n");
+    else
+        writeFormat("\n");
 
     // vector<bool> is special
     _isFirst[_isFirst.size()-1] = true;
 }
 
+void JsonWriter::writeFormat(const char* fmt, ...) {
+    // append to the string, string may grow
+    va_list args;
+    va_start(args, fmt);
+    append_vsprintf(*_out, fmt, args);
+    va_end(args);
+    
+    // when string reach certain length, flush to compressed file and/or buffer
+    if (_stream && _out->size() >= _stream->compressLimit())
+    {
+        // flush the output to a compression stream
+        _stream->compress(Slice((uint8_t*)_out->data(), _out->size())); // losing const
+        
+        // reset the buffer
+        _out->clear();
+    }
+}
+
+JsonWriter::~JsonWriter()
+{
+    if (_stream)  {
+        if (!_out->empty()) {
+            _stream->compress(Slice((uint8_t*)_out->data(), _out->size())); // losing const
+        }
+    }
+}
+
+    
+
+
 
 /*
 void JsonWriter::writeText(const char* text) {

Diff for: libkram/json11/json11.cpp

@@ -54,6 +54,10 @@
 
 #include "ImmutableString.h"
 
+namespace kram {
+class ICompressedStream;
+}
+
 namespace json11 {
 
 using namespace NAMESPACE_STL;
@@ -64,7 +68,7 @@ class JsonReaderData;
 
 //--------------------------
 
-/* Don't want to maintain this form.  Use SAX not DOM for writer.
+/* Don't want to maintain this form from json11.  Use SAX not DOM for writer.
 // Write json nodes out to a string.  String data is encoded.
 class JsonWriter final {
 public:
@@ -99,8 +103,13 @@ class JsonWriter final {
 // And keys go out in the order added.
 class JsonWriter final {
 public:
+    // This writes into a buffer
     JsonWriter(string* str) : _out(str) {}
 
+    // This writes into a small buffer, and then into a compressed stream
+    JsonWriter(string* str, ICompressedStream* stream) : _out(str), _stream(stream) {}
+    ~JsonWriter();
+    
     void pushObject(const char* key = "");
     void popObject();
 
@@ -125,6 +134,8 @@ class JsonWriter final {
     void writeJson(const JsonWriter& json);
 
 private:
+    void writeFormat(const char* fmt, ...) __printflike(2, 3);
+    
     bool isArray() const { return _stack.back() == ']'; }
     bool isObject() const { return _stack.back() == '}'; }
 
@@ -137,6 +148,7 @@ class JsonWriter final {
     string* _out = nullptr;
     string _stack;
     string _escapedString;
+    ICompressedStream* _stream = nullptr;
 };
 
 class JsonArrayScope {

Diff for: libkram/json11/json11.h

@@ -1884,7 +1884,7 @@ bool KTXImage::unpackLevel(uint32_t mipNumber, const uint8_t* srcData, uint8_t*
                 size_t dstDataSizeMiniz = compression_decode_buffer(
                     (uint8_t*)dstData, dstDataSize,
                     (const uint8_t*)srcData, srcDataSize,
-                    scratchBuffer, // scratch-buffer that could speed up to pass
+                    scratchBuffer,
                     COMPRESSION_ZLIB);
 #else
                 mz_ulong dstDataSizeMiniz = 0;