ggml : add CPU backend reference implementation (wip)

ggml/CMakeLists.txt

@@ -263,8 +263,9 @@ option(GGML_ZENDNN                          "ggml: use ZenDNN"
 option(ZENDNN_ROOT                          "ggml: path to ZenDNN installation"               "")
 
 # extra artifacts
-option(GGML_BUILD_TESTS    "ggml: build tests"    ${GGML_STANDALONE})
-option(GGML_BUILD_EXAMPLES "ggml: build examples" ${GGML_STANDALONE})
+option(GGML_BUILD_TESTS     "ggml: build tests"    ${GGML_STANDALONE})
+option(GGML_CPU_REF_BACKEND "ggml: build reference CPU backend for testing" OFF)
+option(GGML_BUILD_EXAMPLES  "ggml: build examples" ${GGML_STANDALONE})
 
 #
 # dependencies
@@ -294,7 +295,9 @@ add_subdirectory(src)
 
 if (GGML_BUILD_TESTS)
     enable_testing()
-    add_subdirectory(tests)
+    if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/tests")
+        add_subdirectory(tests)
+    endif ()
 endif ()
 
 if (GGML_BUILD_EXAMPLES)

ggml/include/ggml-backend.h

@@ -245,6 +245,9 @@ extern "C" {
     // Load all known backends from dynamic libraries
     GGML_API void               ggml_backend_load_all(void);
     GGML_API void               ggml_backend_load_all_from_path(const char * dir_path);
+    // Load all variants for a backend and register them
+    GGML_API void               ggml_backend_load_all_variants(const char * backend_name);
+    GGML_API void               ggml_backend_load_variant(const char * backend_name, const char * variant);
 
     //
     // Backend scheduler

ggml/include/ggml-cpu.h

@@ -75,6 +75,7 @@ extern "C" {
     //
 
     // x86
+    GGML_BACKEND_API int ggml_cpu_has_sse2       (void);
     GGML_BACKEND_API int ggml_cpu_has_sse3       (void);
     GGML_BACKEND_API int ggml_cpu_has_ssse3      (void);
     GGML_BACKEND_API int ggml_cpu_has_avx        (void);

ggml/src/CMakeLists.txt

@@ -459,6 +459,33 @@ ggml_add_backend(OpenCL)
 ggml_add_backend(Hexagon)
 ggml_add_backend(ZenDNN)
 
+if (GGML_CPU_REF_BACKEND)
+    if (NOT GGML_BACKEND_DL)
+        message(FATAL_ERROR "GGML_CPU_REF_BACKEND requires GGML_BACKEND_DL")
+    endif()
+    set(GGML_SYSTEM_ARCH "cpu-ref")
+    set(GGML_LLAMAFILE    OFF)
+    set(GGML_CPU_HBM      OFF)
+    set(GGML_CPU_REPACK   OFF)
+    set(GGML_OPENMP       OFF)
+    set(GGML_CPU_KLEIDIAI OFF)
+    set(GGML_ACCELERATE   OFF)
+
+    ggml_add_cpu_backend_variant(ref)
+
+    if (GGML_SYSTEM_ARCH MATCHES "arm|aarch64|ARM|AARCH64")
+        target_compile_options(ggml-cpu-ref PRIVATE
+              -U__ARM_NEON
+              -U__ARM_FEATURE_FMA
+              -U__ARM_FEATURE_FP16_VECTOR_ARITHMETIC
+              -U__ARM_FEATURE_DOTPROD
+              -U__ARM_FEATURE_MATMUL_INT8
+              -U__ARM_FEATURE_SVE
+          )
+    endif()
+    target_compile_definitions(ggml PRIVATE GGML_USE_CPU_REF)
+endif()
+
 foreach (target ggml-base ggml)
     target_include_directories(${target} PUBLIC    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../include> $<INSTALL_INTERFACE:include>)
     target_compile_features   (${target} PRIVATE c_std_11 cxx_std_17) # don't bump

ggml/src/ggml-backend-reg.cpp

@@ -519,12 +519,11 @@ static fs::path backend_filename_extension() {
 #endif
 }
 
-static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent, const char * user_search_path) {
-    // enumerate all the files that match [lib]ggml-name-*.[so|dll] in the search paths
-    const fs::path name_path = fs::u8path(name);
-    const fs::path file_prefix = backend_filename_prefix().native() + name_path.native() + fs::u8path("-").native();
-    const fs::path file_extension = backend_filename_extension();
+static fs::path backend_filename_prefix_with_name(const char * backend_name) {
+    return backend_filename_prefix().native() + fs::u8path(backend_name).native() + fs::u8path("-").native();
+}
 
+static std::vector<fs::path> get_backend_search_paths(const char * user_search_path = nullptr) {
     std::vector<fs::path> search_paths;
     if (user_search_path == nullptr) {
 #ifdef GGML_BACKEND_DIR
@@ -536,6 +535,16 @@ static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent,
     } else {
         search_paths.push_back(fs::u8path(user_search_path));
     }
+    return search_paths;
+}
+
+static ggml_backend_reg_t ggml_backend_load_best(const char * name, bool silent, const char * user_search_path) {
+    // enumerate all the files that match [lib]ggml-name-*.[so|dll] in the search paths
+    const fs::path name_path = fs::u8path(name);
+    const fs::path file_prefix = backend_filename_prefix_with_name(name);
+    const fs::path file_extension = backend_filename_extension();
+
+    std::vector<fs::path> search_paths = get_backend_search_paths(user_search_path);
 
     int best_score = 0;
     fs::path best_path;
@@ -629,4 +638,60 @@ void ggml_backend_load_all_from_path(const char * dir_path) {
     if (backend_path) {
         ggml_backend_load(backend_path);
     }
+#ifdef GGML_USE_CPU_REF
+    ggml_backend_load_best("cpu-ref", silent, dir_path);
+#endif
+}
+
+void ggml_backend_load_all_variants(const char * backend_name) {
+    const fs::path file_prefix = backend_filename_prefix_with_name(backend_name);
+    const fs::path file_extension = backend_filename_extension();
+
+    std::vector<fs::path> search_paths = get_backend_search_paths();
+
+    // enumerate all the files that match [lib]ggml-name-*.[so|dll] in the search paths
+    for (const auto & search_path : search_paths) {
+        if (!fs::exists(search_path)) {
+            GGML_LOG_DEBUG("%s: search path %s does not exist\n", __func__, path_str(search_path).c_str());
+            continue;
+        }
+
+        for (const auto & entry : fs::directory_iterator(search_path, fs::directory_options::skip_permission_denied)) {
+            if (entry.is_regular_file()) {
+                auto filename = entry.path().filename();
+                auto ext = entry.path().extension();
+                if (filename.native().find(file_prefix.native()) == 0 && ext == file_extension) {
+                    fs::path path = search_path / filename;
+                    ggml_backend_reg_t backend = get_reg().load_backend(path, false);
+                    if (backend == nullptr) {
+                        GGML_LOG_ERROR("%s: failed to load backend variant %s\n", __func__, path_str(entry.path()).c_str());
+                    }
+                }
+            }
+        }
+    }
+}
+
+void ggml_backend_load_variant(const char * backend_name, const char * variant) {
+    const fs::path file_prefix = backend_filename_prefix_with_name(backend_name);
+    const fs::path target_filename = file_prefix.native() + fs::u8path(variant).native() + backend_filename_extension().native();
+
+    std::vector<fs::path> search_paths = get_backend_search_paths();
+
+    for (const auto & search_path : search_paths) {
+        if (!fs::exists(search_path)) {
+            GGML_LOG_DEBUG("%s: search path %s does not exist\n", __func__, path_str(search_path).c_str());
+            continue;
+        }
+
+        fs::path full_path = search_path / target_filename;
+        if (fs::exists(full_path) && fs::is_regular_file(full_path)) {
+            ggml_backend_reg_t backend = get_reg().load_backend(full_path, false);
+            if (backend == nullptr) {
+                GGML_LOG_ERROR("%s: failed to load backend variant %s\n", __func__, path_str(full_path).c_str());
+            } else {
+                return;
+            }
+        }
+    }
 }

ggml/src/ggml-cpu/CMakeLists.txt

@@ -52,6 +52,12 @@ function(ggml_add_cpu_backend_variant_impl tag_name)
     target_compile_features(${GGML_CPU_NAME} PRIVATE c_std_11 cxx_std_17)
     target_include_directories(${GGML_CPU_NAME} PRIVATE . ggml-cpu)
 
+    if (tag_name)
+        target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_CPU_VARIANT_NAME="CPU-${tag_name}")
+    else()
+        target_compile_definitions(${GGML_CPU_NAME} PRIVATE GGML_CPU_VARIANT_NAME="CPU")
+    endif()
+
     if (APPLE AND GGML_ACCELERATE)
         find_library(ACCELERATE_FRAMEWORK Accelerate)
         if (ACCELERATE_FRAMEWORK)

ggml/src/ggml-cpu/ggml-cpu.c

@@ -3558,6 +3558,14 @@ int ggml_cpu_has_llamafile(void) {
 #endif
 }
 
+int ggml_cpu_has_sse2(void) {
+#if defined(__SSE2__)
+    return 1;
+#else
+    return 0;
+#endif
+}
+
 int ggml_cpu_has_sse3(void) {
 #if defined(__SSE3__)
     return 1;

ggml/src/ggml-cpu/ggml-cpu.cpp

@@ -108,7 +108,7 @@ struct ggml_backend_cpu_context {
 };
 
 static const char * ggml_backend_cpu_get_name(ggml_backend_t backend) {
-    return "CPU";
+    return GGML_CPU_VARIANT_NAME;
 
     GGML_UNUSED(backend);
 }
@@ -337,7 +337,7 @@ struct ggml_backend_cpu_device_context {
 };
 
 static const char * ggml_backend_cpu_device_get_name(ggml_backend_dev_t dev) {
-    return "CPU";
+    return GGML_CPU_VARIANT_NAME;
 
     GGML_UNUSED(dev);
 }
@@ -516,6 +516,9 @@ static ggml_backend_feature * ggml_backend_cpu_get_features(ggml_backend_reg_t r
         ggml_cpu_init();
 
         std::vector<ggml_backend_feature> features;
+        if (ggml_cpu_has_sse2()) {
+            features.push_back({ "SSE2", "1" });
+        }
         if (ggml_cpu_has_sse3()) {
             features.push_back({ "SSE3", "1" });
         }

tests/CMakeLists.txt

@@ -218,6 +218,9 @@ if (NOT LLAMA_SANITIZE_ADDRESS AND NOT GGML_SCHED_NO_REALLOC)
 endif()
 llama_build_and_test(test-gguf.cpp)
 llama_build_and_test(test-backend-ops.cpp)
+target_sources(test-backend-ops PRIVATE ${PROJECT_SOURCE_DIR}/ggml/src/ggml.c)
+target_compile_definitions(test-backend-ops PRIVATE GGML_BUILD GGML_VERSION=\"${GGML_VERSION}\" GGML_COMMIT=\"${GGML_COMMIT}\")
+target_include_directories(test-backend-ops PRIVATE ${PROJECT_SOURCE_DIR}/ggml/src)
 
 llama_build_and_test(test-model-load-cancel.cpp  LABEL "model")
 llama_build_and_test(test-autorelease.cpp        LABEL "model")