mtmd: use sha256 for input hashing

CMakeLists.txt

@@ -224,10 +224,12 @@ add_subdirectory(src)
 # utils, programs, examples and tests
 #
 
+# mtmd needs this even when common is not built
+add_subdirectory(vendor/hash)
+
 if (LLAMA_BUILD_COMMON)
     add_subdirectory(common)
     add_subdirectory(vendor/cpp-httplib)
-    add_subdirectory(vendor/hash)
 endif()
 
 if (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TESTS AND NOT CMAKE_JS_VERSION)

examples/gguf-hash/gguf-hash.cpp

@@ -18,13 +18,16 @@ extern "C" {
 #endif
 
 #include "xxhash/xxhash.h"
-#include "sha1/sha1.h"
 #include "sha256/sha256.h"
 
 #ifdef __cplusplus
 }
 #endif
 
+// sha1 is compiled as C++ and lives in a namespace, see scripts/sync_vendor.py
+#include "sha1/sha1.h"
+using namespace vendor_hash;
+
 
 // uuid.uuid5(uuid.NAMESPACE_URL, 'en.wikipedia.org/wiki/Llama.cpp')
 #define UUID_NAMESPACE_LLAMA_CPP "ef001206-dadc-5f6d-a15f-3359e577d4e5"

scripts/sync_vendor.py

@@ -56,6 +56,44 @@
         '    && (defined(_MSC_VER) && (_MSC_VER >= 1000) || !defined(_MSC_VER)) /* >= C11 */\n'
     )],
 
+    # sha1 exports a bare "SHA1" symbol, which clashes with the boringssl one at link time.
+    # we compile it as C++ (see vendor/hash/CMakeLists.txt) and put it in a namespace.
+    "vendor/hash/sha1/sha1.h": [
+        (
+            '#if defined(__cplusplus)\n'
+            'extern "C" {\n'
+            '#endif\n',
+
+            'namespace vendor_hash {\n'
+        ),
+        (
+            '#if defined(__cplusplus)\n'
+            '}\n'
+            '#endif\n',
+
+            '} // namespace vendor_hash\n'
+        ),
+    ],
+
+    "vendor/hash/sha1/sha1.c": [
+        (
+            '#include "sha1.h"\n',
+
+            '#include "sha1.h"\n'
+            '\n'
+            'namespace vendor_hash {\n'
+        ),
+        (
+            '    SHA1Final((unsigned char *)hash_out, &ctx);\n'
+            '}\n',
+
+            '    SHA1Final((unsigned char *)hash_out, &ctx);\n'
+            '}\n'
+            '\n'
+            '} // namespace vendor_hash\n'
+        ),
+    ],
+
     # silence a maybe-uninitialized warning
     "vendor/hash/sha256/sha256.c": [(
         "  uint32_t W[16];\n",

tools/mtmd/CMakeLists.txt

@@ -78,7 +78,7 @@ set_target_properties(mtmd PROPERTIES
 )
 
 target_link_libraries     (mtmd PUBLIC ggml llama)
-target_link_libraries     (mtmd PRIVATE Threads::Threads)
+target_link_libraries     (mtmd PRIVATE Threads::Threads vendor-hash)
 target_include_directories(mtmd PUBLIC  .)
 target_include_directories(mtmd PRIVATE ../..)
 target_include_directories(mtmd PRIVATE ../../vendor)

tools/mtmd/mtmd-helper.cpp

@@ -12,6 +12,8 @@
 #include "mtmd-helper-common.h"
 #include "llama.h"
 
+#include "hash.h"
+
 #include <algorithm>
 #include <cinttypes>
 #include <vector>
@@ -356,25 +358,14 @@ static bool decode_audio_from_buf(const unsigned char * buf_in, size_t len, int
 
 } // namespace audio_helpers
 
-// Computes FNV-1a hash of the data
-static std::string fnv_hash(const uint8_t * data, size_t len) {
-    const uint64_t fnv_prime = 0x100000001b3ULL;
-    uint64_t hash = 0xcbf29ce484222325ULL;
-
-    for (size_t i = 0; i < len; ++i) {
-        hash ^= data[i];
-        hash *= fnv_prime;
-    }
-    return std::to_string(hash);
-}
-
 mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder) {
     // calculate the hash if needed
     std::string id;
     mtmd_bitmap * result = nullptr;
 
     if (!placeholder) {
-        id = fnv_hash(buf, len);
+        // use sha256 to prevent cache poisoning
+        id = hash_sha256_hex(buf, len);
     }
 
     if (audio_helpers::is_audio_file((const char *)buf, len)) {

tools/mtmd/mtmd-helper.h

@@ -49,7 +49,7 @@ MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtm
 // note:
 //   - for now, video input is only supported via C++ helper functions
 //   - audio files will be auto-detected based on magic bytes
-//   - output bitmap will have FNV hash as the ID
+//   - output bitmap will have SHA-256 hash (hex string) as the ID
 // returns nullptr on failure
 // this function is thread-safe
 MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder);

tools/ui/embed.cpp

@@ -259,6 +259,8 @@ int main(int argc, char ** argv) {
             }
             cpp += fmt("static const unsigned char asset_%d_data[] = {", i);
             append_bytes_hex(cpp, bytes);
+
+            // note: this is a simple hash for cache busting, not a cryptographic hash; fnv is enough here
             const auto hash = fnv_hash(bytes.data(), bytes.size());
 
             cpp += fmt("};\nstatic const std::size_t   asset_%d_size = %zu;\n",

vendor/hash/CMakeLists.txt

@@ -4,18 +4,30 @@ llama_add_compile_flags()
 
 set(CMAKE_POSITION_INDEPENDENT_CODE ON)
 
-add_library(${TARGET} STATIC
+set(VENDOR_SRCS
     xxhash/xxhash.c
     sha1/sha1.c
     sha256/sha256.c
 )
 
-# disable warnings in 3rd party code
+add_library(${TARGET} STATIC
+    hash.cpp
+    hash.h
+    ${VENDOR_SRCS}
+)
+
+target_compile_features(${TARGET} PRIVATE cxx_std_17)
+
+# disable warnings in 3rd party code, but keep them for hash.cpp
 if (CMAKE_C_COMPILER_ID STREQUAL "MSVC")
-    target_compile_options(${TARGET} PRIVATE /w)
+    set(NO_WARN_FLAG /w)
 else()
-    target_compile_options(${TARGET} PRIVATE -w)
+    set(NO_WARN_FLAG -w)
 endif()
+set_source_files_properties(${VENDOR_SRCS} PROPERTIES COMPILE_OPTIONS ${NO_WARN_FLAG})
+
+# sha1 lives in a namespace to avoid a clash with boringssl, see scripts/sync_vendor.py
+set_source_files_properties(sha1/sha1.c PROPERTIES LANGUAGE CXX)
 
 # sha256.c includes "rotate-bits/rotate-bits.h", so consumers get this dir too
 target_include_directories(${TARGET} PUBLIC .)

vendor/hash/hash.cpp

@@ -0,0 +1,23 @@
+#include "hash.h"
+
+extern "C" {
+#include "sha256/sha256.h"
+}
+
+static std::string to_hex(const unsigned char * digest, size_t len) {
+    static const char hex[] = "0123456789abcdef";
+
+    std::string out;
+    out.reserve(2*len);
+    for (size_t i = 0; i < len; ++i) {
+        out += hex[digest[i] >> 4];
+        out += hex[digest[i] & 0xf];
+    }
+    return out;
+}
+
+std::string hash_sha256_hex(const void * data, size_t len) {
+    unsigned char digest[SHA256_DIGEST_SIZE];
+    sha256_hash(digest, (const unsigned char *) data, len);
+    return to_hex(digest, SHA256_DIGEST_SIZE);
+}

vendor/hash/hash.h

@@ -0,0 +1,9 @@
+#pragma once
+
+// C++ wrapper for the vendored hash functions
+
+#include <cstddef>
+#include <string>
+
+// returns the SHA-256 digest as a lowercase hex string
+std::string hash_sha256_hex(const void * data, size_t len);

vendor/hash/sha1/sha1.c

@@ -25,6 +25,8 @@ A million repetitions of "a"
 
 #include "sha1.h"
 
+namespace vendor_hash {
+
 
 #define rol(value, bits) (((value) << (bits)) | ((value) >> (32 - (bits))))
 
@@ -293,3 +295,5 @@ void SHA1(
     SHA1Final((unsigned char *)hash_out, &ctx);
 }
 
+} // namespace vendor_hash
+

vendor/hash/sha1/sha1.h

@@ -9,9 +9,7 @@
 
 #include "stdint.h"
 
-#if defined(__cplusplus)
-extern "C" {
-#endif
+namespace vendor_hash {
 
 typedef struct
 {
@@ -45,8 +43,6 @@ void SHA1(
     const char *str,
     uint32_t len);
 
-#if defined(__cplusplus)
-}
-#endif
+} // namespace vendor_hash
 
 #endif /* SHA1_H */