diff --git a/CMakeLists.txt b/CMakeLists.txt index 4e1fd45dd1dc..19f14e0d18fa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -224,10 +224,12 @@ add_subdirectory(src) # utils, programs, examples and tests # +# mtmd needs this even when common is not built +add_subdirectory(vendor/hash) + if (LLAMA_BUILD_COMMON) add_subdirectory(common) add_subdirectory(vendor/cpp-httplib) - add_subdirectory(vendor/hash) endif() if (LLAMA_BUILD_COMMON AND LLAMA_BUILD_TESTS AND NOT CMAKE_JS_VERSION) diff --git a/examples/gguf-hash/gguf-hash.cpp b/examples/gguf-hash/gguf-hash.cpp index 331de301ffc5..43de6300d9f1 100644 --- a/examples/gguf-hash/gguf-hash.cpp +++ b/examples/gguf-hash/gguf-hash.cpp @@ -18,13 +18,16 @@ extern "C" { #endif #include "xxhash/xxhash.h" -#include "sha1/sha1.h" #include "sha256/sha256.h" #ifdef __cplusplus } #endif +// sha1 is compiled as C++ and lives in a namespace, see scripts/sync_vendor.py +#include "sha1/sha1.h" +using namespace vendor_hash; + // uuid.uuid5(uuid.NAMESPACE_URL, 'en.wikipedia.org/wiki/Llama.cpp') #define UUID_NAMESPACE_LLAMA_CPP "ef001206-dadc-5f6d-a15f-3359e577d4e5" diff --git a/scripts/sync_vendor.py b/scripts/sync_vendor.py index 3ab62f34c1db..18a94e1c6962 100755 --- a/scripts/sync_vendor.py +++ b/scripts/sync_vendor.py @@ -56,6 +56,44 @@ ' && (defined(_MSC_VER) && (_MSC_VER >= 1000) || !defined(_MSC_VER)) /* >= C11 */\n' )], + # sha1 exports a bare "SHA1" symbol, which clashes with the boringssl one at link time. + # we compile it as C++ (see vendor/hash/CMakeLists.txt) and put it in a namespace. + "vendor/hash/sha1/sha1.h": [ + ( + '#if defined(__cplusplus)\n' + 'extern "C" {\n' + '#endif\n', + + 'namespace vendor_hash {\n' + ), + ( + '#if defined(__cplusplus)\n' + '}\n' + '#endif\n', + + '} // namespace vendor_hash\n' + ), + ], + + "vendor/hash/sha1/sha1.c": [ + ( + '#include "sha1.h"\n', + + '#include "sha1.h"\n' + '\n' + 'namespace vendor_hash {\n' + ), + ( + ' SHA1Final((unsigned char *)hash_out, &ctx);\n' + '}\n', + + ' SHA1Final((unsigned char *)hash_out, &ctx);\n' + '}\n' + '\n' + '} // namespace vendor_hash\n' + ), + ], + # silence a maybe-uninitialized warning "vendor/hash/sha256/sha256.c": [( " uint32_t W[16];\n", diff --git a/tools/mtmd/CMakeLists.txt b/tools/mtmd/CMakeLists.txt index 769a44e0b73d..db758395ffd6 100644 --- a/tools/mtmd/CMakeLists.txt +++ b/tools/mtmd/CMakeLists.txt @@ -78,7 +78,7 @@ set_target_properties(mtmd PROPERTIES ) target_link_libraries (mtmd PUBLIC ggml llama) -target_link_libraries (mtmd PRIVATE Threads::Threads) +target_link_libraries (mtmd PRIVATE Threads::Threads vendor-hash) target_include_directories(mtmd PUBLIC .) target_include_directories(mtmd PRIVATE ../..) target_include_directories(mtmd PRIVATE ../../vendor) diff --git a/tools/mtmd/mtmd-helper.cpp b/tools/mtmd/mtmd-helper.cpp index d77c93966471..bce8e38cc310 100644 --- a/tools/mtmd/mtmd-helper.cpp +++ b/tools/mtmd/mtmd-helper.cpp @@ -12,6 +12,8 @@ #include "mtmd-helper-common.h" #include "llama.h" +#include "hash.h" + #include #include #include @@ -356,25 +358,14 @@ static bool decode_audio_from_buf(const unsigned char * buf_in, size_t len, int } // namespace audio_helpers -// Computes FNV-1a hash of the data -static std::string fnv_hash(const uint8_t * data, size_t len) { - const uint64_t fnv_prime = 0x100000001b3ULL; - uint64_t hash = 0xcbf29ce484222325ULL; - - for (size_t i = 0; i < len; ++i) { - hash ^= data[i]; - hash *= fnv_prime; - } - return std::to_string(hash); -} - mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder) { // calculate the hash if needed std::string id; mtmd_bitmap * result = nullptr; if (!placeholder) { - id = fnv_hash(buf, len); + // use sha256 to prevent cache poisoning + id = hash_sha256_hex(buf, len); } if (audio_helpers::is_audio_file((const char *)buf, len)) { diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h index 832f7171ac71..5c6b92419763 100644 --- a/tools/mtmd/mtmd-helper.h +++ b/tools/mtmd/mtmd-helper.h @@ -49,7 +49,7 @@ MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_file(mtm // note: // - for now, video input is only supported via C++ helper functions // - audio files will be auto-detected based on magic bytes -// - output bitmap will have FNV hash as the ID +// - output bitmap will have SHA-256 hash (hex string) as the ID // returns nullptr on failure // this function is thread-safe MTMD_API struct mtmd_helper_bitmap_wrapper mtmd_helper_bitmap_init_from_buf(mtmd_context * ctx, const unsigned char * buf, size_t len, bool placeholder); diff --git a/tools/ui/embed.cpp b/tools/ui/embed.cpp index 914d51fa1d8c..b76c9047f289 100644 --- a/tools/ui/embed.cpp +++ b/tools/ui/embed.cpp @@ -259,6 +259,8 @@ int main(int argc, char ** argv) { } cpp += fmt("static const unsigned char asset_%d_data[] = {", i); append_bytes_hex(cpp, bytes); + + // note: this is a simple hash for cache busting, not a cryptographic hash; fnv is enough here const auto hash = fnv_hash(bytes.data(), bytes.size()); cpp += fmt("};\nstatic const std::size_t asset_%d_size = %zu;\n", diff --git a/vendor/hash/CMakeLists.txt b/vendor/hash/CMakeLists.txt index 122a6419f9f8..efdf58e63f5e 100644 --- a/vendor/hash/CMakeLists.txt +++ b/vendor/hash/CMakeLists.txt @@ -4,18 +4,30 @@ llama_add_compile_flags() set(CMAKE_POSITION_INDEPENDENT_CODE ON) -add_library(${TARGET} STATIC +set(VENDOR_SRCS xxhash/xxhash.c sha1/sha1.c sha256/sha256.c ) -# disable warnings in 3rd party code +add_library(${TARGET} STATIC + hash.cpp + hash.h + ${VENDOR_SRCS} +) + +target_compile_features(${TARGET} PRIVATE cxx_std_17) + +# disable warnings in 3rd party code, but keep them for hash.cpp if (CMAKE_C_COMPILER_ID STREQUAL "MSVC") - target_compile_options(${TARGET} PRIVATE /w) + set(NO_WARN_FLAG /w) else() - target_compile_options(${TARGET} PRIVATE -w) + set(NO_WARN_FLAG -w) endif() +set_source_files_properties(${VENDOR_SRCS} PROPERTIES COMPILE_OPTIONS ${NO_WARN_FLAG}) + +# sha1 lives in a namespace to avoid a clash with boringssl, see scripts/sync_vendor.py +set_source_files_properties(sha1/sha1.c PROPERTIES LANGUAGE CXX) # sha256.c includes "rotate-bits/rotate-bits.h", so consumers get this dir too target_include_directories(${TARGET} PUBLIC .) diff --git a/vendor/hash/hash.cpp b/vendor/hash/hash.cpp new file mode 100644 index 000000000000..6493716bf766 --- /dev/null +++ b/vendor/hash/hash.cpp @@ -0,0 +1,23 @@ +#include "hash.h" + +extern "C" { +#include "sha256/sha256.h" +} + +static std::string to_hex(const unsigned char * digest, size_t len) { + static const char hex[] = "0123456789abcdef"; + + std::string out; + out.reserve(2*len); + for (size_t i = 0; i < len; ++i) { + out += hex[digest[i] >> 4]; + out += hex[digest[i] & 0xf]; + } + return out; +} + +std::string hash_sha256_hex(const void * data, size_t len) { + unsigned char digest[SHA256_DIGEST_SIZE]; + sha256_hash(digest, (const unsigned char *) data, len); + return to_hex(digest, SHA256_DIGEST_SIZE); +} diff --git a/vendor/hash/hash.h b/vendor/hash/hash.h new file mode 100644 index 000000000000..1298b4196b4c --- /dev/null +++ b/vendor/hash/hash.h @@ -0,0 +1,9 @@ +#pragma once + +// C++ wrapper for the vendored hash functions + +#include +#include + +// returns the SHA-256 digest as a lowercase hex string +std::string hash_sha256_hex(const void * data, size_t len); diff --git a/vendor/hash/sha1/sha1.c b/vendor/hash/sha1/sha1.c index 76cd6ca3381d..4d84340d43e6 100644 --- a/vendor/hash/sha1/sha1.c +++ b/vendor/hash/sha1/sha1.c @@ -25,6 +25,8 @@ A million repetitions of "a" #include "sha1.h" +namespace vendor_hash { + #define rol(value, bits) (((value) << (bits)) | ((value) >> (32 - (bits)))) @@ -293,3 +295,5 @@ void SHA1( SHA1Final((unsigned char *)hash_out, &ctx); } +} // namespace vendor_hash + diff --git a/vendor/hash/sha1/sha1.h b/vendor/hash/sha1/sha1.h index f492009c976f..4ec5df0f4fe6 100644 --- a/vendor/hash/sha1/sha1.h +++ b/vendor/hash/sha1/sha1.h @@ -9,9 +9,7 @@ #include "stdint.h" -#if defined(__cplusplus) -extern "C" { -#endif +namespace vendor_hash { typedef struct { @@ -45,8 +43,6 @@ void SHA1( const char *str, uint32_t len); -#if defined(__cplusplus) -} -#endif +} // namespace vendor_hash #endif /* SHA1_H */