diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 840573deb8282..7ca48f5ee1881 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -79,8 +79,9 @@ jobs: run: | set -e -x BINARY_SIZE_THRESHOLD_ARGS="" - echo "Binary size threshold in bytes: 1599488" - BINARY_SIZE_THRESHOLD_ARGS="--threshold_size_in_bytes 1599488" + BINARY_SIZE_THRESHOLD_IN_BYTES=1601536 + echo "Binary size threshold in bytes: ${BINARY_SIZE_THRESHOLD_IN_BYTES}" + BINARY_SIZE_THRESHOLD_ARGS="--threshold_size_in_bytes ${BINARY_SIZE_THRESHOLD_IN_BYTES}" # Ensure ANDROID_NDK_HOME is available and get its real path if [ -z "$ANDROID_NDK_HOME" ]; then diff --git a/.github/workflows/linux-wasm-ci-build-and-test-workflow.yml b/.github/workflows/linux-wasm-ci-build-and-test-workflow.yml index 8e9359a44e91d..5d05a92892aba 100644 --- a/.github/workflows/linux-wasm-ci-build-and-test-workflow.yml +++ b/.github/workflows/linux-wasm-ci-build-and-test-workflow.yml @@ -34,6 +34,10 @@ on: required: false type: boolean default: false + build_webgpu_static_plugin: + required: false + type: boolean + default: false jobs: build-wasm: @@ -159,6 +163,24 @@ jobs: --skip_tests working-directory: ${{ github.workspace }} + # Build-only coverage for registering the WebGPU EP as a statically linked plugin EP under Emscripten. + # The output is deliberately not published: it has the same file names as the published WebGPU build, and + # the ORT Web test harness selects the wasm binary by file name, so both variants cannot ship together. + # This leg becomes redundant once the published WebGPU wasm build switches over to the plugin EP. + - name: Build (simd + threads + WebGPU experimental, static plugin EP) + if: ${{ inputs.build_webgpu_static_plugin == true }} + run: | + python ./tools/ci_build/build.py \ + ${{ env.common_build_args }} \ + --build_dir ${{ github.workspace }}/build/wasm_inferencing_webgpu_static_plugin \ + --use_webgpu static_plugin \ + --use_webnn \ + --target onnxruntime_webassembly \ + ${{ inputs.build_config == 'Release' && '--enable_wasm_api_exception_catching' || '' }} \ + ${{ env.reduced_size_build_args }} \ + --skip_tests + working-directory: ${{ github.workspace }} + - name: Create Artifacts if: ${{ inputs.skip_publish != true }} run: | diff --git a/.github/workflows/linux_webgpu.yml b/.github/workflows/linux_webgpu.yml index 234adcaee320c..44aad2400af75 100644 --- a/.github/workflows/linux_webgpu.yml +++ b/.github/workflows/linux_webgpu.yml @@ -37,6 +37,28 @@ jobs: secrets: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Pass token for reusable workflow needs (e.g., docker build action) + # Builds the same WebGPU plugin EP sources linked into the ORT binary rather than as a separate + # shared library. Build-only, like the job above; this exists mainly to compile the static plugin + # configuration with GCC as well as MSVC. See docs/design/webgpu_ep_extraction/ + # plugin_boundary_and_web_integration/static_plugin_ep_registration_design.md + build-linux-webgpu-static-plugin-x64-release: + name: Build Linux WebGPU Static Plugin x64 Release + uses: ./.github/workflows/reusable_linux_build.yml + with: + pool_name: "onnxruntime-github-Ubuntu2204-AMD-CPU" # Build pool + build_config: Release + architecture: x64 + dockerfile_path: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_webgpu + docker_image_repo: onnxruntimecpubuildpythonx64 + extra_build_flags: '--use_binskim_compliant_compile_flags --build_wheel --use_webgpu static_plugin --cmake_extra_defines onnxruntime_BUILD_UNIT_TESTS=ON' + python_path_prefix: 'PATH=/opt/python/cp310-cp310/bin:$PATH' + run_tests: false + upload_build_output: false + execution_providers: 'webgpu' + job_identifier: build-linux-webgpu-static-plugin-x64-release + secrets: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Pass token for reusable workflow needs (e.g., docker build action) + # TODO: the following test step is currently failing. Need to fix and re-enable it # test-linux-webgpu-x64-release: diff --git a/.github/workflows/web.yml b/.github/workflows/web.yml index 0e2ba5e440756..25eac1871b13c 100644 --- a/.github/workflows/web.yml +++ b/.github/workflows/web.yml @@ -56,6 +56,7 @@ jobs: extra_build_args: "--target onnxruntime_webassembly --skip_tests --disable_rtti" build_jsep: true build_webgpu: true + build_webgpu_static_plugin: true wasm_Release_static_library: needs: precheck diff --git a/.github/workflows/windows_webgpu.yml b/.github/workflows/windows_webgpu.yml index 3ef0fb28166e4..5c18e4b04a3c5 100644 --- a/.github/workflows/windows_webgpu.yml +++ b/.github/workflows/windows_webgpu.yml @@ -442,6 +442,77 @@ jobs: exit $lastExitCode } + # Same plugin EP sources as webgpu_plugin_build_x64_RelWithDebInfo, but linked into the ORT binary + # instead of built as a separate shared library. + webgpu_static_plugin_build_x64_RelWithDebInfo: + runs-on: [ + "self-hosted", + "1ES.Pool=onnxruntime-github-Win2022-GPU-A10", + "JobId=webgpu_static_plugin_build_x64_RelWithDebInfo-${{ github.run_id }}-${{ github.run_number }}-${{ github.run_attempt }}" + ] + timeout-minutes: 300 + env: + OnnxRuntimeBuildDirectory: ${{ github.workspace }} + setVcvars: true + ALLOW_RELEASED_ONNX_OPSET_ONLY: "0" + DocUpdateNeeded: false + NVIDIA_TF32_OVERRIDE: "0" + ONNXRUNTIME_TEST_GPU_DEVICE_ID: "0" + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + submodules: none + + - name: Setup Python 3.12 + uses: actions/setup-python@v6 + with: + python-version: "3.12" + architecture: x64 + + - name: Locate vcvarsall and Setup Env + uses: ./.github/actions/locate-vcvarsall-and-setup-env + with: + architecture: x64 + + - name: Install python modules + run: python -m pip install -r tools\ci_build\github\windows\python\requirements.txt + shell: cmd + working-directory: ${{ github.workspace }} + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: "20.x" + + - uses: actions/cache@v5 + id: onnx-node-tests-cache + with: + path: ${{ github.workspace }}/js/test/ + key: onnxnodetests-${{ hashFiles('js/scripts/prepare-onnx-node-tests.ts') }} + + # onnxruntime_BUILD_DAWN_SHARED_LIBRARY is deliberately not enabled here. It is incompatible with + # onnxruntime_USE_EP_API_ADAPTERS, which --use_webgpu static_plugin turns on. + - name: Build and Test + shell: pwsh + run: | + python.exe ${{ github.workspace }}\tools\ci_build\build.py ` + --config RelWithDebInfo ` + --build_dir ${{ github.workspace }} ` + --skip_submodule_sync ` + --parallel ` + --use_binskim_compliant_compile_flags ` + --cmake_generator "Visual Studio 17 2022" ` + --enable_onnx_tests ` + --use_webgpu static_plugin ` + --use_vcpkg --use_vcpkg_ms_internal_asset_cache ` + --cmake_extra_defines onnxruntime_BUILD_UNIT_TESTS=ON onnxruntime_ENABLE_DAWN_BACKEND_D3D12=1 onnxruntime_ENABLE_DAWN_BACKEND_VULKAN=1 + + if ($lastExitCode -ne 0) { + exit $lastExitCode + } + webgpu_external_dawn_build_x64_RelWithDebInfo: runs-on: [ "self-hosted", diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index de7c6faf316e3..8aeb7ce38dbf2 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -180,7 +180,8 @@ option(onnxruntime_USE_CANN "Build with CANN support" OFF) option(onnxruntime_USE_XNNPACK "Build with XNNPACK support. Provides an alternative math library on ARM, WebAssembly and x86." OFF) option(onnxruntime_USE_WEBNN "Build with WebNN support. Enable hardware acceleration in web browsers." OFF) option(onnxruntime_USE_WEBGPU "Build with WebGPU support. Enable WebGPU via C/C++ interface." OFF) -option(onnxruntime_USE_EP_API_ADAPTERS "Build EP (e.g. WebGPU) as a plugin EP shared library using EP API adapters" OFF) +option(onnxruntime_USE_EP_API_ADAPTERS "Build EP (e.g. WebGPU) as a plugin EP library using EP API adapters" OFF) +option(onnxruntime_WEBGPU_STATIC_PLUGIN "Build the WebGPU plugin EP into the ORT binary instead of a separate shared library. Requires onnxruntime_USE_EP_API_ADAPTERS." OFF) option(onnxruntime_USE_EXTERNAL_DAWN "Build with treating Dawn as external dependency. Will not link Dawn at build time." OFF) option(onnxruntime_CUSTOM_DAWN_SRC_PATH "Path to custom Dawn src dir.") set(onnxruntime_DAWN_PREBUILT_DIR "" CACHE PATH "Directory containing Dawn headers and standalone proc sources for external Dawn") @@ -456,6 +457,34 @@ if(onnxruntime_USE_EP_API_ADAPTERS) add_compile_definitions(ORT_USE_EP_API_ADAPTERS=1) endif() +# The WebGPU plugin EP can either be a separate shared library or be linked into the ORT binary. In the latter case +# ORT core registers it at OrtEnv creation time instead of the application loading it by path. +if(onnxruntime_WEBGPU_STATIC_PLUGIN) + if(NOT onnxruntime_USE_WEBGPU) + message(FATAL_ERROR "onnxruntime_WEBGPU_STATIC_PLUGIN=ON requires onnxruntime_USE_WEBGPU=ON.") + endif() + if(NOT onnxruntime_USE_EP_API_ADAPTERS) + message(FATAL_ERROR "onnxruntime_WEBGPU_STATIC_PLUGIN=ON requires onnxruntime_USE_EP_API_ADAPTERS=ON.") + endif() + # A minimal build excludes core/session/plugin_ep, so the registration that a static plugin EP depends on is not + # compiled in. Without this check the EP would be linked into the binary but never registered, with no diagnostic + # at configure, build or run time. Supporting the combination is possible but requires making the plugin EP + # registration available to minimal builds first. + if(onnxruntime_MINIMAL_BUILD) + message(FATAL_ERROR "onnxruntime_WEBGPU_STATIC_PLUGIN=ON is not supported in a minimal build: the plugin EP " + "registration in core/session/plugin_ep is excluded, so the EP would be linked in but " + "never registered.") + endif() + add_compile_definitions(ORT_WEBGPU_STATIC_PLUGIN=1) +endif() + +# True when the WebGPU EP is compiled into the ORT binary rather than built as a separate shared library. +if(onnxruntime_USE_WEBGPU AND (NOT onnxruntime_USE_EP_API_ADAPTERS OR onnxruntime_WEBGPU_STATIC_PLUGIN)) + set(onnxruntime_WEBGPU_LINKED_INTO_HOST ON) +else() + set(onnxruntime_WEBGPU_LINKED_INTO_HOST OFF) +endif() + # General C# properties if (onnxruntime_BUILD_CSHARP) check_language(CSharp) diff --git a/cmake/onnxruntime.cmake b/cmake/onnxruntime.cmake index 137a426d120a9..6eeb48a0a3086 100644 --- a/cmake/onnxruntime.cmake +++ b/cmake/onnxruntime.cmake @@ -256,7 +256,7 @@ if (onnxruntime_BUILD_QNN_EP_STATIC_LIB) list(APPEND onnxruntime_INTERNAL_PROVIDER_LIBRARIES onnxruntime_providers_qnn) endif() -if (onnxruntime_USE_WEBGPU AND NOT onnxruntime_USE_EP_API_ADAPTERS) +if (onnxruntime_WEBGPU_LINKED_INTO_HOST) list(APPEND onnxruntime_INTERNAL_PROVIDER_LIBRARIES onnxruntime_providers_webgpu) endif() diff --git a/cmake/onnxruntime_common.cmake b/cmake/onnxruntime_common.cmake index b4666ef596272..f64a80448a45f 100644 --- a/cmake/onnxruntime_common.cmake +++ b/cmake/onnxruntime_common.cmake @@ -104,6 +104,9 @@ elseif (LINUX) elseif (APPLE) list(APPEND onnxruntime_common_src_patterns "${ONNXRUNTIME_ROOT}/core/platform/apple/device_discovery.cc") +elseif (CMAKE_SYSTEM_NAME STREQUAL "Emscripten") + list(APPEND onnxruntime_common_src_patterns + "${ONNXRUNTIME_ROOT}/core/platform/emscripten/device_discovery.cc") else() list(APPEND onnxruntime_common_src_patterns "${ONNXRUNTIME_ROOT}/core/platform/device_discovery_default.cc") diff --git a/cmake/onnxruntime_providers_webgpu.cmake b/cmake/onnxruntime_providers_webgpu.cmake index 52f5340d2374e..40402e3b0638d 100644 --- a/cmake/onnxruntime_providers_webgpu.cmake +++ b/cmake/onnxruntime_providers_webgpu.cmake @@ -25,7 +25,7 @@ if(NOT onnxruntime_USE_EP_API_ADAPTERS) # - # Build WebGPU EP as a static library + # Build WebGPU EP as an internal (non-plugin) static library # # For static library build, exclude the 'ep' folder @@ -39,6 +39,24 @@ onnxruntime_add_static_library(onnxruntime_providers_webgpu ${onnxruntime_providers_webgpu_cc_srcs}) onnxruntime_add_include_to_target(onnxruntime_providers_webgpu onnxruntime_common onnx onnx_proto flatbuffers::flatbuffers Boost::mp11 safeint_interface) + elseif(onnxruntime_WEBGPU_STATIC_PLUGIN) + # + # Build WebGPU EP as a plugin EP that is statically linked into the ORT binary. + # + # This uses the same plugin EP sources ('ep' folder) and EP API adapters as the shared library build below. + # The difference is only in how the EP reaches the application: ORT core registers it at OrtEnv creation time + # instead of the application calling RegisterExecutionProviderLibrary() with a library path. + # + source_group(TREE ${ONNXRUNTIME_ROOT} FILES ${onnxruntime_providers_webgpu_cc_srcs}) + onnxruntime_add_static_library(onnxruntime_providers_webgpu ${onnxruntime_providers_webgpu_cc_srcs}) + onnxruntime_add_include_to_target(onnxruntime_providers_webgpu + onnxruntime_common onnx onnx_proto flatbuffers::flatbuffers Boost::mp11 safeint_interface) + + # ORT_PLUGIN_EP_SKIP_API_MANUAL_INIT tells the shared plugin EP headers that OrtGetApiBase() is available + # in-process, so the C++ API must not be built with ORT_API_MANUAL_INIT (the rest of the binary is not). + target_compile_definitions(onnxruntime_providers_webgpu PRIVATE + ORT_PLUGIN_EP_STATICALLY_LINKED=1 + ORT_PLUGIN_EP_SKIP_API_MANUAL_INIT=1) else() # # Build WebGPU EP as a shared library @@ -77,27 +95,6 @@ add_definitions("-DONNX_NAMESPACE=onnx") add_definitions("-DONNX_USE_LITE_PROTO=1") - # Default plugin EP version to ORT_VERSION with "-dev" suffix if not explicitly provided. - if(NOT DEFINED onnxruntime_PLUGIN_EP_VERSION) - set(onnxruntime_PLUGIN_EP_VERSION "${ORT_VERSION}-dev") - endif() - - # Set preprocessor definition for plugin EP version - target_compile_definitions(onnxruntime_providers_webgpu PRIVATE - ORT_PLUGIN_EP_VERSION="${onnxruntime_PLUGIN_EP_VERSION}") - - # Bake the minimum compatible ORT version (the single source of truth lives in - # plugin-ep-webgpu/MIN_ONNXRUNTIME_VERSION) into the EP DLL so it can be enforced at runtime. - # Format is strict "MAJOR.MINOR.PATCH". - set(_ORT_PLUGIN_EP_WEBGPU_MIN_ORT_VERSION_FILE "${REPO_ROOT}/plugin-ep-webgpu/MIN_ONNXRUNTIME_VERSION") - file(STRINGS "${_ORT_PLUGIN_EP_WEBGPU_MIN_ORT_VERSION_FILE}" _ORT_PLUGIN_EP_WEBGPU_MIN_ORT_VERSION LIMIT_COUNT 1) - if(NOT _ORT_PLUGIN_EP_WEBGPU_MIN_ORT_VERSION) - message(FATAL_ERROR "WebGPU plugin EP minimum ORT version file is missing or empty: " - "${_ORT_PLUGIN_EP_WEBGPU_MIN_ORT_VERSION_FILE}") - endif() - target_compile_definitions(onnxruntime_providers_webgpu PRIVATE - ORT_PLUGIN_EP_MIN_ORT_VERSION="${_ORT_PLUGIN_EP_WEBGPU_MIN_ORT_VERSION}") - # Set preprocessor definitions used in onnxruntime_providers_webgpu.rc if(WIN32) set(WEBGPU_DLL_FILE_DESCRIPTION "ONNX Runtime WebGPU Provider") @@ -131,14 +128,50 @@ message(FATAL_ERROR "WebGPU EP shared library build does not support build cache. Please disable build cache or use static library build.") endif() if (CMAKE_SYSTEM_NAME STREQUAL "Emscripten") - message(FATAL_ERROR "WebGPU EP shared library build is not supported on Emscripten. Please use static library build.") + message(FATAL_ERROR "WebGPU EP shared library build is not supported on Emscripten. Please use '--use_webgpu static_plugin'.") + endif() + endif() + + if (onnxruntime_USE_EP_API_ADAPTERS) + # + # Settings common to both plugin EP builds (shared library and statically linked). + # + + # Default plugin EP version to ORT_VERSION with "-dev" suffix if not explicitly provided. + if(NOT DEFINED onnxruntime_PLUGIN_EP_VERSION) + set(onnxruntime_PLUGIN_EP_VERSION "${ORT_VERSION}-dev") endif() - # Configure precompiled headers for shared library build - # PCH ensures ep/adapters.h is included first and improves compilation speed + # Set preprocessor definition for plugin EP version + target_compile_definitions(onnxruntime_providers_webgpu PRIVATE + ORT_PLUGIN_EP_VERSION="${onnxruntime_PLUGIN_EP_VERSION}") + + # Bake the minimum compatible ORT version (the single source of truth lives in + # plugin-ep-webgpu/MIN_ONNXRUNTIME_VERSION) into the EP so it can be enforced at runtime. + # Format is strict "MAJOR.MINOR.PATCH". + set(_ORT_PLUGIN_EP_WEBGPU_MIN_ORT_VERSION_FILE "${REPO_ROOT}/plugin-ep-webgpu/MIN_ONNXRUNTIME_VERSION") + file(STRINGS "${_ORT_PLUGIN_EP_WEBGPU_MIN_ORT_VERSION_FILE}" _ORT_PLUGIN_EP_WEBGPU_MIN_ORT_VERSION LIMIT_COUNT 1) + if(NOT _ORT_PLUGIN_EP_WEBGPU_MIN_ORT_VERSION) + message(FATAL_ERROR "WebGPU plugin EP minimum ORT version file is missing or empty: " + "${_ORT_PLUGIN_EP_WEBGPU_MIN_ORT_VERSION_FILE}") + endif() + target_compile_definitions(onnxruntime_providers_webgpu PRIVATE + ORT_PLUGIN_EP_MIN_ORT_VERSION="${_ORT_PLUGIN_EP_WEBGPU_MIN_ORT_VERSION}") + + # Configure precompiled headers for the plugin EP build. + # PCH ensures ep/adapters.h is included first and improves compilation speed. target_precompile_headers(onnxruntime_providers_webgpu PRIVATE "${REPO_ROOT}/include/onnxruntime/ep/adapters.h" ) + + # Suppress -Werror=maybe-uninitialized. The EP adapter types are defined entirely inline in headers, + # so GCC sees through the failure and exception paths of accessors like OpKernelInfo::GetAttr<> and + # falsely warns about values that are only read once the call has succeeded. It also produces false + # positives deep inside absl::InlinedVector for locals that are always constructed. + # This mirrors the same suppression on the CUDA plugin EP target. + target_compile_options(onnxruntime_providers_webgpu PRIVATE + $<$,$>:-Wno-maybe-uninitialized> + ) endif() set_target_properties(onnxruntime_providers_webgpu PROPERTIES CXX_STANDARD_REQUIRED ON) @@ -261,13 +294,14 @@ endif() endif() - # In the plugin/adapter build (onnxruntime_USE_EP_API_ADAPTERS) the WebGPU EP is its own DLL - # (onnxruntime_providers_webgpu.dll) rather than being statically linked into onnxruntime.dll, so the + # In the plugin EP shared library build the WebGPU EP is its own DLL + # (onnxruntime_providers_webgpu.dll) rather than being linked into onnxruntime.dll, so the # delay-load flags accumulated above (notably /DELAYLOAD:user32.dll) must be applied to THIS target. # Without it the DLL keeps a static import on user32.dll and fails to load in a Win32k-lockdown sandbox # (e.g. Chromium's WebNN compiler process): 'depends on "USER32.dll" which is missing' (Error 1359). - # In the static build these same flags are applied to the onnxruntime target in onnxruntime.cmake. - if (WIN32 AND onnxruntime_USE_EP_API_ADAPTERS AND onnxruntime_DELAYLOAD_FLAGS) + # When the EP is linked into the ORT binary these same flags are applied to the onnxruntime target in + # onnxruntime.cmake. + if (WIN32 AND NOT onnxruntime_WEBGPU_LINKED_INTO_HOST AND onnxruntime_DELAYLOAD_FLAGS) target_link_options(onnxruntime_providers_webgpu PRIVATE ${onnxruntime_DELAYLOAD_FLAGS}) target_link_libraries(onnxruntime_providers_webgpu PRIVATE delayimp.lib) endif() diff --git a/cmake/onnxruntime_python.cmake b/cmake/onnxruntime_python.cmake index ccbfe1d9306cd..2c8ff7567702c 100644 --- a/cmake/onnxruntime_python.cmake +++ b/cmake/onnxruntime_python.cmake @@ -213,7 +213,7 @@ set(onnxruntime_pybind11_state_static_providers if(onnxruntime_BUILD_QNN_EP_STATIC_LIB) list(APPEND onnxruntime_pybind11_state_static_providers PRIVATE onnxruntime_providers_qnn) endif() -if(onnxruntime_USE_WEBGPU AND NOT onnxruntime_USE_EP_API_ADAPTERS) +if(onnxruntime_WEBGPU_LINKED_INTO_HOST) list(APPEND onnxruntime_pybind11_state_static_providers PRIVATE onnxruntime_providers_webgpu) endif() if(WIN32) diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index 4c0451f7bdab7..18bba1c0c38e2 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -734,7 +734,7 @@ set(ONNXRUNTIME_TEST_STATIC_PROVIDER_LIBS if (onnxruntime_BUILD_QNN_EP_STATIC_LIB) list(APPEND ONNXRUNTIME_TEST_STATIC_PROVIDER_LIBS onnxruntime_providers_qnn) endif() -if (onnxruntime_USE_WEBGPU AND NOT onnxruntime_USE_EP_API_ADAPTERS) +if (onnxruntime_WEBGPU_LINKED_INTO_HOST) list(APPEND ONNXRUNTIME_TEST_STATIC_PROVIDER_LIBS onnxruntime_providers_webgpu) endif() @@ -1262,7 +1262,11 @@ endfunction() # Set environment variables for plugin EP tests when run via CTest. function(onnxruntime_set_plugin_ep_test_environment target) - if(onnxruntime_USE_WEBGPU AND onnxruntime_USE_EP_API_ADAPTERS) + if(onnxruntime_WEBGPU_STATIC_PLUGIN) + # The WebGPU plugin EP is linked into the test binary and registered by ORT core, so there is no + # library to register by path here. onnxruntime_set_webgpu_plugin_ep_test_definitions() below gives + # test_main.cc an equivalent compiled-in default config. + elseif(onnxruntime_USE_WEBGPU AND onnxruntime_USE_EP_API_ADAPTERS) set(ORT_PLUGIN_EP_JSON_CONFIG "{\"ep_library_registration_name\": \"WebGPU_PluginEP\", \"ep_library_path\": \"$\", \"selected_ep_name\": \"WebGpuExecutionProvider\"}") set_tests_properties(${target} PROPERTIES ENVIRONMENT "ORT_UNIT_TEST_MAIN_DYNAMIC_PLUGIN_EP_CONFIG_JSON=${ORT_PLUGIN_EP_JSON_CONFIG}" @@ -1272,6 +1276,33 @@ function(onnxruntime_set_plugin_ep_test_environment target) endif() endfunction() +# Route the WebGPU EP through the dynamic plugin EP infrastructure in plugin builds +# (--use_webgpu shared_lib or --use_webgpu static_plugin). +# Without initializing the infra in test_main.cc, WebGpuExecutionProviderWithOptions() (default_providers.cc, +# adapters branch) returns null and every WebGPU test skips itself, leaving the plugin path with no test coverage. +# This must be applied to every test target that links test_main.cc and runs WebGPU tests. +function(onnxruntime_set_webgpu_plugin_ep_test_definitions target) + if(NOT (onnxruntime_USE_WEBGPU AND onnxruntime_USE_EP_API_ADAPTERS)) + return() + endif() + + target_compile_definitions(${target} PRIVATE + ORT_UNIT_TEST_ENABLE_DYNAMIC_PLUGIN_EP_USAGE + ORT_UNIT_TEST_HAS_WEBGPU_PLUGIN_EP=1) + + if (onnxruntime_WEBGPU_STATIC_PLUGIN) + # The plugin EP is linked into the test binary and registered by ORT core during environment creation, + # so there is no library path to dlopen and no separate build-order dependency needed. + target_compile_definitions(${target} PRIVATE ORT_UNIT_TEST_HAS_WEBGPU_STATIC_PLUGIN_EP=1) + else() + target_compile_definitions(${target} PRIVATE + ORT_UNIT_TEST_WEBGPU_PLUGIN_EP_LIBRARY_PATH="$") + # The plugin EP DLL is dlopen'd at test-run time (not linked), so add an explicit build-order + # dependency to ensure it (and its co-located dawn/dxcompiler DLLs) exist before the tests run. + add_dependencies(${target} onnxruntime_providers_webgpu) + endif() +endfunction() + function(onnxruntime_apply_emscripten_test_link_settings target) if (CMAKE_SYSTEM_NAME STREQUAL "Emscripten") set_target_properties(${target} PROPERTIES LINK_DEPENDS ${TEST_SRC_DIR}/wasm/onnxruntime_test_adapter.js) @@ -1320,17 +1351,7 @@ if (onnxruntime_USE_CUDA AND onnxruntime_BUILD_CUDA_EP_AS_PLUGIN) endif() if (onnxruntime_USE_WEBGPU AND onnxruntime_USE_EP_API_ADAPTERS) - # Route the WebGPU EP through the dynamic plugin EP infrastructure in plugin builds - # (--use_webgpu shared_lib). Same rationale as the CUDA-as-plugin block above: without initializing the - # infra in test_main.cc, WebGpuExecutionProviderWithOptions() (default_providers.cc, adapters branch) - # returns null and every WebGPU test skips itself, leaving the plugin path with no test coverage. - target_compile_definitions(onnxruntime_test_all PRIVATE - ORT_UNIT_TEST_ENABLE_DYNAMIC_PLUGIN_EP_USAGE - ORT_UNIT_TEST_WEBGPU_PLUGIN_EP_LIBRARY_PATH="$" - ORT_UNIT_TEST_HAS_WEBGPU_PLUGIN_EP=1) - # The plugin EP DLL is dlopen'd at test-run time (not linked), so add an explicit build-order - # dependency to ensure it (and its co-located dawn/dxcompiler DLLs) exist before the tests run. - add_dependencies(onnxruntime_test_all onnxruntime_providers_webgpu) + onnxruntime_set_webgpu_plugin_ep_test_definitions(onnxruntime_test_all) endif() if (MSVC) @@ -1580,6 +1601,7 @@ block() # enable dynamic plugin EP usage target_compile_definitions(${onnxruntime_provider_test_target} PRIVATE ORT_UNIT_TEST_ENABLE_DYNAMIC_PLUGIN_EP_USAGE) + onnxruntime_set_webgpu_plugin_ep_test_definitions(${onnxruntime_provider_test_target}) onnxruntime_apply_emscripten_test_link_settings(${onnxruntime_provider_test_target}) if (IOS) diff --git a/cmake/onnxruntime_webassembly.cmake b/cmake/onnxruntime_webassembly.cmake index f930e138f7346..aac2c4d729cf7 100644 --- a/cmake/onnxruntime_webassembly.cmake +++ b/cmake/onnxruntime_webassembly.cmake @@ -274,7 +274,7 @@ else() if (onnxruntime_ENABLE_WEBASSEMBLY_JSPI) target_link_options(onnxruntime_webassembly PRIVATE "SHELL:-s JSPI=1" - "SHELL:-s JSPI_EXPORTS=[OrtAppendExecutionProvider,OrtCreateSession,OrtRun,OrtRunWithBinding,OrtBindInput]" + "SHELL:-s JSPI_EXPORTS=[OrtAppendExecutionProvider,OrtAppendExecutionProviderV2,OrtCreateSession,OrtRun,OrtRunWithBinding,OrtBindInput]" ) else() # NOTE: "-s ASYNCIFY=1" is required for JSEP to work with WebGPU diff --git a/docs/design/webgpu_ep_extraction/plugin_boundary_and_web_integration/plugin_boundary_and_web_integration_workstream.md b/docs/design/webgpu_ep_extraction/plugin_boundary_and_web_integration/plugin_boundary_and_web_integration_workstream.md index 8a975f8edb85c..cd90a252aaa2f 100644 --- a/docs/design/webgpu_ep_extraction/plugin_boundary_and_web_integration/plugin_boundary_and_web_integration_workstream.md +++ b/docs/design/webgpu_ep_extraction/plugin_boundary_and_web_integration/plugin_boundary_and_web_integration_workstream.md @@ -1,6 +1,7 @@ # Workstream `plugin-boundary`: Plugin Boundary and Web/Wasm Integration -Status: Working plan +Status: In progress. Generic static plugin EP registration is implemented in PR #32395, and WebGPU can be built +through it with `--use_webgpu static_plugin`. ORT Web migration and the remaining lifetime work are outstanding. [WebGPU EP extraction overview](../webgpu_ep_extraction.md) @@ -31,8 +32,8 @@ does not own WebGPU kernels, Dawn, shader tooling, or the provider's standalone ## Generic static plugin registration -Add an ORT facility that accepts statically linked plugin factory entry points. It should reuse the existing dynamic -plugin path after library loading and symbol lookup. +The static plugin facility accepts linked factory entry points and reuses the existing dynamic plugin path after +library loading and symbol lookup. The design must address: @@ -44,7 +45,11 @@ The design must address: - Dead-code elimination for statically linked providers. - Diagnostics for incompatible or duplicate registrations. -The facility must be generic and validated with at least one non-WebGPU test plugin where practical. +The facility must be generic. Validation with a non-WebGPU test plugin is deferred: WebGPU is currently the only +statically linked plugin EP, so this is tracked as a test gap rather than a completed criterion. + +The detailed design and validation summary are in +[Static Plugin EP Registration](static_plugin_ep_registration_design.md). ## Process-global ownership and teardown @@ -115,8 +120,8 @@ Likely investigation areas include: The adapter's own `Missing parts` section in `onnxruntime/core/providers/webgpu/ep/README.md` is authoritative input to this inventory rather than speculation. It records two gaps: WebGPU cleanup, which the process-global ownership -and teardown work covers, and EP default configuration, which is missing for both static and shared library builds -and sketches an `OrtApi` addition for it. +and teardown work covers, and EP default configuration, which is missing for hosts that do not expose environment +configuration. ## Browser/Wasm bridge @@ -136,15 +141,19 @@ The bridge should not expose unrelated ORT private implementation details. ## Work packages -1. **Size and latency baselines:** measure `onnxruntime-web` WebAssembly size and inference latency, and native - inference latency, on the non-plugin path while it still exists, since the size and latency completion criteria - compare the plugin path against those numbers. -2. **Static registration core:** implement and contract-test generic static factory registration. -3. **Global lifetime contract:** inventory process-global state and implement safe ownership and teardown rules. -4. **Emscripten prototype:** compile the plugin path statically and run a small model. -5. **Gap inventory triage:** convert private-dependency findings into public API or provider-owned actions. -6. **Browser bridge:** specify and prototype object and lifetime exchange. -7. **Parity and retirement:** run the existing suite through the plugin path and remove the legacy path. +1. **Size and latency baselines — In progress:** measure `onnxruntime-web` WebAssembly size and inference latency, + and native inference latency, on the non-plugin path while it still exists. +2. **Static registration core — Implemented in PR #32395:** contract-test generic static factory registration. + Validation with more than one static plugin remains outstanding. +3. **Global lifetime contract — In progress:** inventory process-global state and implement safe ownership and + teardown rules. +4. **Emscripten prototype — Done:** compile the plugin path statically and run it in a browser. Production ORT Web + migration remains separate work. +5. **Gap inventory triage — Not started:** convert private-dependency findings into public API or provider-owned + actions. +6. **Browser bridge — Not started:** specify and prototype object and lifetime exchange. +7. **Parity and retirement — In progress:** run the existing suite through the plugin path and remove the legacy + path after ORT Web parity is demonstrated. Packages 1 through 6 can proceed largely in parallel, and legacy-path removal waits for their convergence. That ordering matters for the baselines in particular: the non-plugin path is the comparison, so once package 7 removes it @@ -180,8 +189,11 @@ the numbers can no longer be captured. the native shared-library plugin. - The direct `IExecutionProvider` WebGPU path is removed. -## Open questions +## Resolved decisions -- Should static factories be registered before environment creation or through environment construction options? -- What is the stable representation of JavaScript-owned WebGPU objects at the C API boundary? +- Should static factories be registered before environment creation, or through environment construction options? + Neither. ORT core registers them during `OrtEnv` creation, after the environment is constructed and published. + See [Static Plugin EP Registration](static_plugin_ep_registration_design.md). + +## Open questions - Which private-dependency findings require a public plugin EP API addition rather than a provider-owned replacement? diff --git a/docs/design/webgpu_ep_extraction/plugin_boundary_and_web_integration/static_plugin_ep_registration_design.md b/docs/design/webgpu_ep_extraction/plugin_boundary_and_web_integration/static_plugin_ep_registration_design.md new file mode 100644 index 0000000000000..9bd889abf640d --- /dev/null +++ b/docs/design/webgpu_ep_extraction/plugin_boundary_and_web_integration/static_plugin_ep_registration_design.md @@ -0,0 +1,225 @@ +# Static Plugin EP Registration + +Status: Implemented in PR #32395. ORT Web's migration onto this path remains follow-up work. This design +supports [Plugin Boundary and Web/Wasm Integration](plugin_boundary_and_web_integration_workstream.md). + +## Context and scope + +A provider linked into the host could not previously use the plugin EP path. `EpLibraryPlugin` discovers factory +entry points in a shared library, while `EpLibraryInternal` wraps an in-tree `IExecutionProvider`. Neither accepts +linked factory entry points. + +This design adds that missing discovery path. Static and dynamic plugins then share the same factory lifecycle and +public API boundary. WebGPU is the first consumer because Emscripten cannot use its shared-module build. + +This design does not add a public static-registration API, support minimal builds, remove the EP API adapters, or +remove WebGPU's existing `EpLibraryInternal` path. Those changes remain follow-up work. + +## Decision summary + +| Concern | Decision | +| --- | --- | +| D1 Discovery | ORT core owns a hand-written, build-guarded static registry | +| D2 Symbol names | Shared builds use standard names; static builds use provider-prefixed names | +| D3 Process globals | Ownership is selected at build time; no new optional teardown ABI | +| D4 Build configuration | `onnxruntime_WEBGPU_STATIC_PLUGIN` selects linked-in plugin mode | +| D5 Test infrastructure | The plugin library path becomes optional | +| D6 Registration timing | Register after publishing `OrtEnv`; environment creation stays serialized | +| D7 C++ API initialization | Linked-in plugins use the host binary's normal initialization mode | + +Both linkage modes converge on one factory lifecycle; only discovery differs: + +```mermaid +flowchart LR + Shared[Shared plugin module] --> Load[Load library, resolve symbols] + Static[Static plugin registry] --> Direct[Use linked entry points] + Load --> Factories[Shared factory lifecycle] + Direct --> Factories + Factories --> Devices[Register factories and devices in Environment] +``` + +## Static plugin contract + +- A statically linked provider exposes the standard `CreateEpFactories` and `ReleaseEpFactory` entry points under a + provider-specific prefix that is unique within the host binary. Their signatures and factory ownership rules are + unchanged from the shared-plugin ABI. +- ORT core discovers those entry points through its build-guarded registry. Applications do not register linked-in + providers explicitly. +- Registration occurs during process-singleton `OrtEnv` creation, after the environment is published but before + other threads can obtain it. Direct callers of `onnxruntime::Environment::Create` do not receive static plugin + registration. +- `GetSupportedDevices` may call environment-resolving `OrtApi` functions on the creating thread. It runs before the + current provider's devices are committed, so `GetEpDevices` does not include those devices and may include only + static providers registered earlier in the registry. +- ORT releases factories through `ReleaseEpFactory` during environment teardown. A statically linked provider must + not release process-global state owned by the host. +- Minimal builds reject static plugin EP configuration until the required plugin infrastructure is available there. + +### Failure handling + +`EpLibraryStaticPlugin` releases factories created before `CreateEpFactories` reports a failure. Existing +`Environment::RegisterExecutionProviderLibrary` diagnostics cover duplicate names and factory or device errors. +Registration stages factories, devices, allocators, and data transfers and rolls back dependent objects before +releasing their factory on failure. If automatic registration fails, `OrtEnv::GetOrCreateInstance` unpublishes and +destroys the partial singleton before returning the error. Other threads therefore cannot observe incomplete static +registration. + +## Decisions + +### D1: Registration mechanism is a link-time registry in ORT core + +ORT core holds a hand-written, `#if`-guarded list of linked factory entry points. `EpLibraryStaticPlugin` accepts +those entry points directly and reuses `EpLibraryPlugin` after its library-loading and symbol-resolution steps. +This gives every host the provider without adding a host-specific registration call. + +Alternatives considered: + +- A CMake-generated registry adds machinery that is not justified while providers remain in-tree. +- A public registration API requires every host to call it and creates a period in which `GetEpDevices` has no + static devices. Revisit this option when an out-of-tree provider needs static linking. + +### D2: Entry point names depend on the linkage mode + +Shared plugins continue to export `CreateEpFactories` and `ReleaseEpFactory`, the names resolved by +`EpLibraryPlugin::Load`. Static plugins use provider-prefixed names such as `WebGpu_CreateEpFactories`, allowing +multiple providers to coexist in one binary. `ORT_PLUGIN_EP_STATICALLY_LINKED` selects the names in provider code. + +### D3: No new EP library ABI hook for teardown + +Factory teardown remains in `ReleaseEpFactory`. Host-owned process-global teardown, currently +`google::protobuf::ShutdownProtobufLibrary()`, is compiled out when `ORT_PLUGIN_EP_STATICALLY_LINKED` is defined. A +shared module owns the state it initializes; a static plugin shares that state with the host, which owns its lifetime. + +The existing WebGPU cleanup in `ort_env.cc` remains for the internal-EP build. It already compiles out for the static +plugin configuration because that configuration defines `ORT_USE_EP_API_ADAPTERS`. + +Static linking has no library unload, so process-global state survives unregistration. A complete regression test +must register, run, tear down the environment, recreate it, and run again. + +### D4: One new durable build option + +`onnxruntime_WEBGPU_STATIC_PLUGIN` selects linked-in plugin mode, and `build.py` exposes it as +`--use_webgpu static_plugin`. The derived `onnxruntime_WEBGPU_LINKED_INTO_HOST` controls linkage sites. + +The option requires WebGPU and the EP API adapters. It rejects minimal builds because they exclude the plugin EP +registration code while still linking the provider, which would otherwise create a binary in which WebGPU is present +but never registered. Emscripten and build-cache restrictions continue to apply only to the shared-module mode. + +`onnxruntime_USE_EP_API_ADAPTERS` remains transitional. Provider sources and the CUDA plugin configuration are +unchanged. + +### D5: Test infrastructure treats the library path as optional + +`InitializationConfig::ep_library_path` becomes optional. When it is absent, tests skip library registration because +ORT core has already registered the static plugin. The existing RAII handle supports this empty state; device +selection, de-duplication, and factory creation are unchanged. + +Virtual devices are enabled through the `allow_virtual_devices` environment configuration entry +(`kOrtEnvAllowVirtualDevices`) supplied at environment creation, rather than the `.virtual` registration-name suffix. +The suffix is unavailable because ORT core chooses the registration name for statically linked providers. + +A shared CMake helper applies the static-plugin test definitions to both `onnxruntime_test_all` and +`onnxruntime_provider_test`, where the operator tests run. + +### D6: Static plugin EPs are registered after the environment is published + +**Problem.** The natural insertion point is next to `CreateAndRegisterInternalEps` inside `Environment::Initialize`. +Registration enumerates devices, which calls provider code: + +``` +OrtEnv::GetOrCreateInstance() + lock(m_) non-recursive + Environment::Create() -> Initialize() + Environment::CreateAndRegisterStaticPluginEps() + Environment::RegisterExecutionProviderLibrary() + EpInfo::Create -> factory GetSupportedDevices() + webgpu::ep::Factory::GetSupportedDevices() provider code + Api().ep.GetEnvConfigEntries() + OrtEnv::TryGetInstance() + lock(m_) self-deadlock +``` + +The internal WebGPU factory avoids this hazard by capturing configuration at construction. A plugin factory cannot: +its only access is through `OrtApi`, and any environment-resolving API can re-enter the same mutex. + +**Decision.** `OrtEnv::GetOrCreateInstance` publishes `p_instance_` and takes its own reference *before* invoking +`Environment::CreateAndRegisterStaticPluginEps`, and `OrtEnv::m_` becomes a `std::recursive_mutex`. Registration +therefore runs against a fully constructed and published environment. + +Guarantees: + +- Environment-resolving API calls can re-enter on the creating thread and find the published instance. +- Other threads remain blocked until registration completes. +- The creating thread's reference prevents destruction during registration. +- Provider code and its `GetSupportedDevices` contract remain the same for both linkage modes. + +Costs and limits: + +- Registration failure must explicitly tear down the published instance. +- `OrtEnv::m_` becomes recursive solely to permit same-thread re-entry during static registration. +- During `GetSupportedDevices`, `GetEpDevices` cannot include the current provider and can include only static + providers registered earlier. Dynamic registration has the same current-provider limitation. +- Direct callers of `onnxruntime::Environment::Create` deliberately bypass singleton registration, often to control + logging or threading in tests, and do not receive static plugin EPs. +- Provider code that starts a thread which calls `CreateEnv` and then joins it will deadlock. This is already true of + any code running inside `Environment::Create`. + +`CreateAndRegisterStaticPluginEps` therefore runs from the `OrtEnv` singleton wrapper, not from +`Environment::Initialize`. `CreateAndRegisterInternalEps` remains in `Environment::Initialize`, so directly created +environments retain internal EP devices. + +Alternatives considered: + +- A thread-local environment pointer fixes only selected APIs, not the full `OrtApi` surface. +- Deferred device enumeration reports registration failures from `GetEpDevices` instead of environment creation. +- Registration outside the lock lets other threads observe incomplete registration unless another barrier is added. + +### D7: The statically linked build does not use manual C++ API initialization + +Shared plugins use `ORT_API_MANUAL_INIT` because they must initialize the C++ API from the host-supplied `OrtApi`. +MSVC requires every translation unit in one binary to agree on that mode, so using it in a statically linked plugin +causes `LNK2038` against ORT core and test code. + +For a static plugin, `ORT_PLUGIN_EP_STATICALLY_LINKED` leaves `onnxruntime_cxx_api.h` in its normal mode and skips +`Ort::InitApi`. `OrtGetApiBase()` resolves in the same binary and returns the same-version API. The EP-specific +`ApiPtrs` are still populated from the `OrtApiBase*` passed to `CreateEpFactories`. + +The macro is generic rather than WebGPU-specific because `ep/api.h` is shared with the CUDA plugin EP. + +## Limitations and follow-ups + +| Item | State | +| --- | --- | +| Environment recreation | Untested. Existing unit-test binaries retain the process-singleton `OrtEnv`; a dedicated process is needed to exercise register, run, tear down, recreate, and run again. | +| Multiple static plugin EPs | Untested. WebGPU is the only registry entry, so prefix uniqueness and registration ordering are not covered with two providers. | +| Minimal builds | Unsupported and rejected at configure time because they exclude plugin EP infrastructure. | +| ORT Web production migration | Deferred. The shipping Wasm configuration still uses the internal EP path. | +| Dawn shared-library mode | Remains incompatible with the EP API adapter configuration. | +| Binary size and dead-code elimination | Deferred to ORT Web migration, where the shipped Wasm artifact can be measured. | +| Internal WebGPU cleanup | Remove `allow_virtual_devices` plumbing from `CreateAndRegisterInternalEps` after retiring the internal WebGPU path. | + +## Validation summary + +| Configuration | Result | Evidence and remaining gap | +| --- | --- | --- | +| Windows static plugin, real GPU | Build and tests pass | The static-plugin and internal-EP builds have matching results for their common provider tests. | +| Windows shared plugin | Build and tests pass | The DLL continues to export the unprefixed entry points required by `EpLibraryPlugin`. | +| Linux static plugin, GCC | Build passes | Build-only CI covers GCC and Python wheel linkage; GPU execution is not exercised. | +| Emscripten static plugin | Build passes | Build-only CI and symbol inspection verify linked entry points. A local prototype passed the browser WebGPU suite, but shipping artifacts and blocking CI still use the internal EP path. | +| Minimal build | Rejected at configure time | Prevents a provider from being linked into a binary that omits its registration infrastructure. | + +The adapter configuration excludes operator cases that depend on internal WebGPU test infrastructure, in addition +to white-box and disabled tests. + +## Appendix: implementation map + +| Decision | Owning code | +| --- | --- | +| D1 static registry | `core/session/plugin_ep/ep_static_plugins.cc`, `ep_library_static_plugin.h` | +| D1 shared factory lifecycle | `core/session/plugin_ep/ep_library_plugin.cc` | +| D2 entry point prefixing | `core/providers/webgpu/ep/api.cc`, `include/onnxruntime/ep/api.h` | +| D3 process-global ownership | `core/providers/webgpu/ep/api.cc` `ReleaseEpFactory` | +| D4 build option | `cmake/CMakeLists.txt`, `cmake/onnxruntime_providers_webgpu.cmake`, `tools/ci_build/build.py` | +| D5 optional library path | `test/unittest_util/test_dynamic_plugin_ep.cc` | +| D6 registration timing | `core/session/ort_env.cc`, `Environment::CreateAndRegisterStaticPluginEps` | +| D7 C++ API initialization | `include/onnxruntime/ep/api.h` | diff --git a/docs/design/webgpu_ep_extraction/webgpu_ep_extraction.md b/docs/design/webgpu_ep_extraction/webgpu_ep_extraction.md index 06b1f8e3a9868..66fd707f25071 100644 --- a/docs/design/webgpu_ep_extraction/webgpu_ep_extraction.md +++ b/docs/design/webgpu_ep_extraction/webgpu_ep_extraction.md @@ -47,10 +47,13 @@ The effort is divided into four workstreams: | Identifier | Workstream | Primary outcome | | --- | --- | --- | | `plugin-boundary` | [Plugin boundary and Web/Wasm integration](plugin_boundary_and_web_integration/plugin_boundary_and_web_integration_workstream.md) | Static and dynamic builds use the same public plugin EP boundary, including the ORT Web browser bridge | -| `provider-isolation` | [Provider isolation and repository migration](provider_isolation_and_repository_migration/provider_isolation_and_repository_migration_workstream.md) | WebGPU-owned code, dependencies, tests, and existing plugin packaging move under `plugin-ep-webgpu/` and then to an independent repository | +| `provider-isolation` | [Provider isolation and repository migration](provider_isolation_and_repository_migration/provider_isolation_and_repository_migration_workstream.md) | WebGPU-owned code, dependencies, tests, and the Python and NuGet plugin packaging are consolidated under `plugin-ep-webgpu/` and then move to an independent repository | | `test-conformance` | [Test ownership and operator conformance](test_ownership_and_conformance/test_ownership_and_conformance_workstream.md) | Existing coverage is preserved, every test has an owner, and portable conformance coverage protects the external provider | | `node-migration` | [Node plugin migration](node_plugin_migration/node_plugin_migration_workstream.md) | Existing bundled Node WebGPU support is replaced by an explicitly consumable plugin without regressing current users | +The detailed static plugin EP registration design is in +[Static Plugin EP Registration](plugin_boundary_and_web_integration/static_plugin_ep_registration_design.md). + The detailed reusable conformance-suite design is in [Execution Provider Operator Conformance Suite](test_ownership_and_conformance/ep_operator_conformance_design.md). @@ -77,13 +80,16 @@ provider being extracted. This is handled in Every consumer that receives WebGPU today needs a recorded disposition before the built-in implementation is removed from ORT. Platform and architecture details remain to be inventoried in the individual workstreams. -| Consumer or package | Disposition | Extraction requirement | -| --- | --- | --- | -| `onnxruntime-web` | Consume a pinned external WebGPU source revision through static plugin registration | Required | -| Python WebGPU plugin package | Move the existing optional plugin package and release pipeline | Required | -| WebGPU NuGet plugin package | Move the existing optional plugin package and release pipeline | Required | -| Node WebGPU support | Provide a tested replacement for the WebGPU implementation currently bundled in `onnxruntime-node`; final package naming is open | Required before bundled support is removed | -| `onnxruntime-webgpu` on PyPI | Publication has already stopped; do not resurrect it or convert it into a plugin-dependent package | Confirm whether any other retired package needs the same treatment | +| Consumer or package | Disposition | +| --- | --- | +| `onnxruntime-web` | Consume a pinned external WebGPU source revision through static plugin registration | +| Python WebGPU plugin package | Move the existing plugin package and packaging pipeline | +| WebGPU NuGet plugin package | Move the existing plugin package and packaging pipeline | +| Node WebGPU support | Provide a tested replacement for the WebGPU implementation currently bundled in `onnxruntime-node`; final package naming is open | +| Android AAR | Not yet decided. The default full AAR build enables WebGPU | +| Java packages | Not yet decided. `OrtProvider.WEBGPU` and `SessionOptions.addWebGPU()` are public API, and the JAR carries the Dawn and DXC native libraries | +| macOS native packages | Not yet decided. macOS packaging builds with `--use_webgpu`, so the released `onnxruntime-osx-*` tarball has WebGPU in `libonnxruntime.dylib`, and that tarball also supplies the native NuGet package. The Apple `.framework` build does not enable WebGPU | +| Python `onnxruntime-webgpu` package | Publication has already stopped. Do not resurrect it or convert it into a plugin-dependent package | Hosts that do not ship WebGPU support today are outside this table. Adding one is a separate decision and is not an extraction prerequisite. @@ -121,7 +127,9 @@ The other convergence points between workstreams: - Test classification determines which tests move with the provider and which remain in ORT. Source transfer waits on it. - The Node workstream consumes generic plugin loading from ORT and native WebGPU artifacts from the external - provider. It does not gate source transfer, but bundled Node WebGPU cannot be removed until it lands. + provider. It does not gate source transfer, but bundled Node WebGPU cannot be removed until it lands. If transfer + comes first, the bundled Node build has to consume the pinned external source until the replacement package ships; + the alternative is to delay transfer until it does. ## Success criteria diff --git a/include/onnxruntime/core/session/environment.h b/include/onnxruntime/core/session/environment.h index 338dbf85ef664..503c3d2571b5c 100644 --- a/include/onnxruntime/core/session/environment.h +++ b/include/onnxruntime/core/session/environment.h @@ -140,6 +140,28 @@ class Environment { Status RegisterExecutionProviderLibrary(const std::string& registration_name, const ORTCHAR_T* lib_path); Status UnregisterExecutionProviderLibrary(const std::string& registration_name); + /** + * Passkey that restricts CreateAndRegisterStaticPluginEps() to OrtEnv, which is the only caller able to satisfy + * that function's ordering and locking requirements. Only OrtEnv can construct one. + */ + class StaticPluginEpRegistrationToken { + private: + StaticPluginEpRegistrationToken() = default; + ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(StaticPluginEpRegistrationToken); + friend struct ::OrtEnv; + }; + + /** + * Register the plugin execution providers that are statically linked into the ORT binary. + * + * This must be called after the OrtEnv singleton that owns this Environment has been published, because a + * statically linked plugin EP uses the public ORT API, and any OrtEnv API it calls must be able to find the + * instance. + * + * The passkey parameter limits the set of possible callers to OrtEnv. + */ + Status CreateAndRegisterStaticPluginEps(StaticPluginEpRegistrationToken); + // convert an OrtEpFactory* to EpFactoryInternal* if possible. EpFactoryInternal* GetEpFactoryInternal(OrtEpFactory* factory) const { // we're comparing pointers so the reinterpret_cast should be safe @@ -254,8 +276,6 @@ class Environment { // providing a CPU allocator. std::unique_ptr default_cpu_ort_allocator_; - using OrtAllocatorUniquePtr = std::unique_ptr>; - #if !defined(ORT_MINIMAL_BUILD) // register EPs that are built into the ORT binary so they can take part in AutoEP selection // added to ep_libraries @@ -266,9 +286,10 @@ class Environment { const std::vector& internal_factories = {}); struct EpInfo { - // calls EpLibrary::Load - // for each factory gets the OrtEpDevice instances and adds to execution_devices - // internal_factory is set if this is an internal EP + // Calls EpLibrary::Load. + // For each factory, gets the OrtEpDevice instances and adds to `out.execution_devices`. + // Provide `internal_factories` if this is an internal EP. + // If successful, `out` is set to the created instance. static Status Create(std::unique_ptr library_in, std::unique_ptr& out, const std::vector& internal_factories = {}); diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index 4dd086c62cf96..8c7f6e4389363 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -2394,7 +2394,9 @@ struct OrtApi { * * If the `size` parameter is less than the actual string attribute's size and `out` * is not nullptr, the value of `size` is set to the true size of the string attribute - * and a failure status is returned.) + * and a failure status is returned. + * + * The true size of the string attribute includes the trailing null character. * * \param[in] info ::OrtKernelInfo instance * \param[in] name Null terminated string of the name of the attribute diff --git a/include/onnxruntime/core/session/onnxruntime_ep_c_api.h b/include/onnxruntime/core/session/onnxruntime_ep_c_api.h index 0424a95c2d2c1..999cd495f354e 100644 --- a/include/onnxruntime/core/session/onnxruntime_ep_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_ep_c_api.h @@ -2360,11 +2360,14 @@ struct OrtEp { * The OrtMemoryInfo instance will match one of the values set in the OrtEpDevice using EpDevice_AddAllocatorInfo. * Any allocator specific options should be read from the session options. * - * If nullptr OrtEpFactory::CreateAllocator will be used. + * Implementation of this function is optional. If nullptr, OrtEpFactory::CreateAllocator will be used if + * implemented. If neither callback is implemented, the EP does not provide a preferred allocator for the memory + * location. For default CPU memory, this allows ORT to use its default CPU allocator. * - * \param[in] this_ptr The OrtEpFactory instance. + * \param[in] this_ptr The OrtEp instance. * \param[in] memory_info The OrtMemoryInfo to create the allocator for. May be nullptr. - * \param[out] allocator The created OrtAllocator instance. Set to nullptr if the default CPU allocator is used. + * \param[out] allocator The created OrtAllocator instance. Set to nullptr if the EP does not provide a preferred + * allocator for the memory location. * * \snippet{doc} snippets.dox OrtStatus Return Value * @@ -2795,7 +2798,17 @@ struct OrtEpFactory { * \param[in] max_ep_devices The maximum number of OrtEpDevices that can be added to ep_devices. * Current default is 8. This can be increased if needed. * \param[out] num_ep_devices The number of EP devices added to ep_devices. - * \return true if the factory can create an execution provider that uses `device`. + * + * \snippet{doc} snippets.dox OrtStatus Return Value + * + * \remarks Called during execution provider library registration. The OrtEpDevice instances returned are added to + * the environment after this returns. An implementation should not depend on other OrtEpDevices returned by + * OrtApi::GetEpDevices since those depend on the execution provider library registration order. + * + * Environment configuration entries are populated before any library is registered, so + * OrtEpApi::GetEnvConfigEntries may be called from this function, but it must be called synchronously on + * the thread invoking GetSupportedDevices. The implementation must not wait for another thread that calls + * OrtEpApi::GetEnvConfigEntries. * * \since Version 1.22. */ @@ -2901,10 +2914,15 @@ struct OrtEpFactory { * The factory that creates the EP is responsible for providing the allocators required by the EP. * The OrtMemoryInfo instance will match one of the values set in the OrtEpDevice using EpDevice_AddAllocatorInfo. * + * Implementation of this function is optional. It is required if ORT needs to create a shared allocator for an + * OrtMemoryInfo advertised by the EP. + * * \param[in] this_ptr The OrtEpFactory instance. * \param[in] memory_info The OrtMemoryInfo to create the allocator for. May be nullptr. * \param[in] allocator_options Optional key-value pairs for allocator options, can be nullptr. - * \param[out] allocator The created OrtAllocator instance. Set to nullptr if the default CPU allocator is used. + * \param[out] allocator The created OrtAllocator instance. When called to create an allocator for a session, this + * may be set to nullptr if the EP does not provide a preferred allocator for the memory + * location. When called to create a shared allocator, this must be set to a non-null value. * * \snippet{doc} snippets.dox OrtStatus Return Value * @@ -2916,6 +2934,9 @@ struct OrtEpFactory { _Outptr_result_maybenull_ OrtAllocator** allocator); /** \brief Release an OrtAllocator created by the factory. + * + * Implementation of this function is required if OrtEpFactory::CreateAllocator or OrtEp::CreateAllocator is + * implemented. * * \since Version 1.23. */ diff --git a/include/onnxruntime/ep/adapter/op_kernel_info.h b/include/onnxruntime/ep/adapter/op_kernel_info.h index 417ebd4adf7a2..49fb78ca6aaad 100644 --- a/include/onnxruntime/ep/adapter/op_kernel_info.h +++ b/include/onnxruntime/ep/adapter/op_kernel_info.h @@ -9,11 +9,16 @@ #include #include +#include +#include +#include +#include "core/common/common.h" #include "core/common/inlined_containers.h" #include "core/common/narrow.h" #include "core/common/status.h" #include "core/framework/config_options.h" +#include "core/framework/error_code_helper.h" #include "core/framework/tensor_shape.h" #include "core/framework/tensor.h" @@ -154,7 +159,7 @@ struct OpKernelInfo { template [[nodiscard]] T GetAttrOrDefault(const std::string& name, const T& default_value) const { - T tmp; + T tmp{}; return GetAttr(name, &tmp).IsOK() ? tmp : default_value; } template @@ -164,27 +169,17 @@ struct OpKernelInfo { } template [[nodiscard]] T GetAttr(const std::string& name) const { - T value; + T value{}; ORT_THROW_IF_ERROR(GetAttr(name, &value)); return value; } template Status GetAttr(const std::string& name, T* value) const { - try { - *value = info_.GetAttribute(name.c_str()); - return Status::OK(); - } catch (const Ort::Exception& ex) { - return Status(onnxruntime::common::ONNXRUNTIME, ex.GetOrtErrorCode(), ex.what()); - } + return GetAttrImpl(cache_->kernel_info_, name.c_str(), value); } template Status GetAttrs(const std::string& name, std::vector& values) const { - try { - values = info_.GetAttributes(name.c_str()); - return Status::OK(); - } catch (const Ort::Exception& ex) { - return Status(onnxruntime::common::ONNXRUNTIME, ex.GetOrtErrorCode(), ex.what()); - } + return GetAttrsImpl(cache_->kernel_info_, name.c_str(), values); } Status GetAttrs(const std::string& name, TensorShapeVector& out) const { @@ -210,6 +205,90 @@ struct OpKernelInfo { } private: + static Status GetAttrImpl(const OrtKernelInfo* info, const char* name, float* out) { + return ToStatusAndRelease(Ort::GetApi().KernelInfoGetAttribute_float(info, name, out)); + } + + static Status GetAttrImpl(const OrtKernelInfo* info, const char* name, int64_t* out) { + return ToStatusAndRelease(Ort::GetApi().KernelInfoGetAttribute_int64(info, name, out)); + } + + static Status GetAttrImpl(const OrtKernelInfo* info, const char* name, std::string* out) { + size_t size = 0; + // Feed nullptr for the data buffer to query the true size of the string attribute. + ORT_RETURN_IF_ERROR(ToStatusAndRelease(Ort::GetApi().KernelInfoGetAttribute_string(info, name, nullptr, &size))); + + std::string value; + value.resize(size); + ORT_RETURN_IF_ERROR( + ToStatusAndRelease(Ort::GetApi().KernelInfoGetAttribute_string(info, name, value.data(), &size))); + value.resize(size - 1); // remove the terminating character '\0' + *out = std::move(value); + return Status::OK(); + } + + static Status GetAttrsImpl(const OrtKernelInfo* info, const char* name, std::vector& out) { + size_t size = 0; + ORT_RETURN_IF_ERROR( + ToStatusAndRelease(Ort::GetApi().KernelInfoGetAttributeArray_float(info, name, nullptr, &size))); + + std::vector values(size); + ORT_RETURN_IF_ERROR( + ToStatusAndRelease(Ort::GetApi().KernelInfoGetAttributeArray_float(info, name, values.data(), &size))); + out.swap(values); + return Status::OK(); + } + + static Status GetAttrsImpl(const OrtKernelInfo* info, const char* name, std::vector& out) { + size_t size = 0; + ORT_RETURN_IF_ERROR( + ToStatusAndRelease(Ort::GetApi().KernelInfoGetAttributeArray_int64(info, name, nullptr, &size))); + + std::vector values(size); + ORT_RETURN_IF_ERROR( + ToStatusAndRelease(Ort::GetApi().KernelInfoGetAttributeArray_int64(info, name, values.data(), &size))); + out.swap(values); + return Status::OK(); + } + + static Status GetAttrsImpl(const OrtKernelInfo* info, const char* name, std::vector& out) { + OrtAllocator* allocator = nullptr; + ORT_RETURN_IF_ERROR(ToStatusAndRelease(Ort::GetApi().GetAllocatorWithDefaultOptions(&allocator))); + size_t size = 0; + ORT_RETURN_IF_ERROR( + ToStatusAndRelease(Ort::GetApi().KernelInfoGetAttributeArray_string(info, name, allocator, nullptr, &size))); + if (size == 0) { + out.clear(); + return Status::OK(); + } + + char** raw_values = nullptr; + ORT_RETURN_IF_ERROR( + ToStatusAndRelease( + Ort::GetApi().KernelInfoGetAttributeArray_string(info, name, allocator, &raw_values, &size))); + + // The allocator owns both the array and every string in it, so release them on all exit paths: constructing the + // std::string copies below can throw. Free through the OrtAllocator function pointer rather than + // Ort::AllocatorWithDefaultOptions::Free(), which throws on failure and so must not run in this deleter. + auto free_raw_values = [allocator, size](char** values_to_free) { + for (size_t i = 0; i < size; ++i) { + if (values_to_free[i] != nullptr) { + allocator->Free(allocator, values_to_free[i]); + } + } + allocator->Free(allocator, values_to_free); + }; + std::unique_ptr raw_values_guard{raw_values, std::move(free_raw_values)}; + + std::vector values; + values.reserve(size); + for (size_t i = 0; i < size; ++i) { + values.emplace_back(raw_values[i] != nullptr ? raw_values[i] : ""); + } + out.swap(values); + return Status::OK(); + } + const Ort::ConstKernelInfo info_; std::shared_ptr cache_; }; diff --git a/include/onnxruntime/ep/api.h b/include/onnxruntime/ep/api.h index 5a6da2250a9cd..93425ca5779e6 100644 --- a/include/onnxruntime/ep/api.h +++ b/include/onnxruntime/ep/api.h @@ -10,11 +10,26 @@ #include #include +// Users of this header can explicitly define ORT_PLUGIN_EP_SKIP_API_MANUAL_INIT to skip manual ORT C++ API +// initialization. +// +// A plugin EP built as its own shared library must not call OrtGetApiBase() itself. It receives the OrtApiBase* from +// ORT and initializes the C++ API manually, so ORT_API_MANUAL_INIT is forced on for onnxruntime_cxx_api.h. This is +// the default because it is the safe choice for a separately built plugin. +// +// A build that links the plugin EP into the ORT binary defines ORT_PLUGIN_EP_SKIP_API_MANUAL_INIT: OrtGetApiBase() is +// then available in-process and the rest of the binary is compiled without ORT_API_MANUAL_INIT. MSVC's detect_mismatch +// guard in onnxruntime_cxx_api.h requires every translation unit in a binary to agree on ORT_API_MANUAL_INIT, so in +// that case leave it alone and let the C++ API initialize itself. +#if !defined(ORT_PLUGIN_EP_SKIP_API_MANUAL_INIT) #pragma push_macro("ORT_API_MANUAL_INIT") #undef ORT_API_MANUAL_INIT #define ORT_API_MANUAL_INIT #include "onnxruntime_cxx_api.h" #pragma pop_macro("ORT_API_MANUAL_INIT") +#else +#include "onnxruntime_cxx_api.h" +#endif namespace onnxruntime { namespace ep { @@ -141,8 +156,11 @@ inline void ApiInit(const OrtApiBase* ort_api_base, const char* min_ort_version throw std::runtime_error("Failed to initialize EP API: GetEpApi or GetModelEditorApi returned null."); } - // Manual init for the C++ API +#if !defined(ORT_PLUGIN_EP_SKIP_API_MANUAL_INIT) + // Manual init for the C++ API. When the EP is linked into the ORT binary, the C++ API initializes itself from + // OrtGetApiBase() with ORT_API_VERSION, which is the same API pointer because the EP and ORT are the same binary. Ort::InitApi(ort_api); +#endif // Initialize globals detail::g_api_ptrs.emplace(*ort_api, *ep_api, *model_editor_api); diff --git a/js/eslint.config.mjs b/js/eslint.config.mjs index 9a5edd2e6761e..9ae9f36362dba 100644 --- a/js/eslint.config.mjs +++ b/js/eslint.config.mjs @@ -273,6 +273,7 @@ export default [ '_OrtAddRunConfigEntry', '_OrtAddSessionConfigEntry', '_OrtAppendExecutionProvider', + '_OrtAppendExecutionProviderV2', '_OrtBindInput', '_OrtBindOutput', '_OrtClearBoundOutputs', @@ -282,7 +283,9 @@ export default [ '_OrtCreateSessionOptions', '_OrtCreateTensor', '_OrtEndProfiling', + '_OrtEpDevice_EpName', '_OrtFree', + '_OrtGetEpDevices', '_OrtGetInputName', '_OrtGetInputOutputCount', '_OrtGetInputOutputMetadata', diff --git a/js/web/lib/wasm/session-options.ts b/js/web/lib/wasm/session-options.ts index 4ba0e1c432cb8..30f1b12185c58 100644 --- a/js/web/lib/wasm/session-options.ts +++ b/js/web/lib/wasm/session-options.ts @@ -70,6 +70,52 @@ const appendEpOption = (epOptions: Array<[number, number]>, key: string, value: epOptions.push([keyDataOffset, valueDataOffset]); }; +// OrtErrorCode values are part of the public C API ABI. +const ORT_NOT_IMPLEMENTED = 9; + +/** + * Get the OrtEpDevice instances registered with the environment, grouped by execution provider name. + * + * An execution provider may register more than one device, e.g. one per GPU, and all of them share its name. + * + * @returns a map of execution provider name to the OrtEpDevice pointers registered under it, or undefined if the + * build does not support the EP device API. + */ +const getEpDevicesByEpName = (): Map | undefined => { + const wasm = getInstance(); + const stack = wasm.stackSave(); + try { + const epDevicesPtr = wasm.stackAlloc(wasm.PTR_SIZE); + const numEpDevicesPtr = wasm.stackAlloc(wasm.PTR_SIZE); + const errorCode = wasm._OrtGetEpDevices(epDevicesPtr, numEpDevicesPtr); + if (errorCode === ORT_NOT_IMPLEMENTED) { + return undefined; + } + if (errorCode !== 0) { + checkLastError("Can't get execution provider devices."); + } + // The array is owned by the environment, so only the pointers are read out here. + // getValue() with '*' returns a BigInt in a wasm64 build, so convert before doing pointer arithmetic. + const epDevices = Number(wasm.getValue(epDevicesPtr, '*')); + const numEpDevices = Number(wasm.getValue(numEpDevicesPtr, '*')); + + const epDevicesByEpName = new Map(); + for (let i = 0; i < numEpDevices; i++) { + const epDevice = Number(wasm.getValue(epDevices + i * wasm.PTR_SIZE, '*')); + const epName = wasm.UTF8ToString(wasm._OrtEpDevice_EpName(epDevice)); + const devices = epDevicesByEpName.get(epName); + if (devices) { + devices.push(epDevice); + } else { + epDevicesByEpName.set(epName, [epDevice]); + } + } + return epDevicesByEpName; + } finally { + wasm.stackRestore(stack); + } +}; + const setExecutionProviders = async ( sessionOptionsHandle: number, sessionOptions: InferenceSession.SessionOptions, @@ -79,6 +125,7 @@ const setExecutionProviders = async ( for (const ep of executionProviders) { let epName = typeof ep === 'string' ? ep : ep.name; const epOptions: Array<[number, number]> = []; + let selectedEpDevices: number[] | undefined; // check EP name switch (epName) { @@ -100,6 +147,17 @@ const setExecutionProviders = async ( case 'webgpu': if (!BUILD_DEFS.DISABLE_WEBGPU) { epName = 'WebGPU'; + const epDevicesByEpName = getEpDevicesByEpName(); + if (epDevicesByEpName !== undefined) { + // Non-minimal builds select by OrtEpDevice under the EP's canonical name. Minimal builds do not support + // the EP device API and continue to use the built-in EP name table. + epName = 'WebGpuExecutionProvider'; + selectedEpDevices = epDevicesByEpName.get(epName); + if (selectedEpDevices === undefined) { + const registered = [...epDevicesByEpName.keys()].join(', ') || '(none)'; + throw new Error(`no execution provider device is registered for: ${epName}. registered: ${registered}.`); + } + } let customDevice: GPUDevice | undefined; if (typeof ep !== 'string') { @@ -193,7 +251,6 @@ const setExecutionProviders = async ( throw new Error(`not supported execution provider: ${epName}`); } - const epNameDataOffset = allocWasmString(epName, allocs); const epOptionsCount = epOptions.length; let keysOffset = 0; let valuesOffset = 0; @@ -207,15 +264,46 @@ const setExecutionProviders = async ( getInstance().setValue(valuesOffset + i * getInstance().PTR_SIZE, epOptions[i][1], '*'); } } - if ( - (await getInstance()._OrtAppendExecutionProvider( + + let appendErrorCode: number; + if (selectedEpDevices !== undefined) { + // Select a single OrtEpDevice, even though more than one may match the EP name. + // + // Necessary: the WebGPU EP backs a session with one Dawn device, so its factory rejects any device + // count other than 1 (see WebGpuEpFactory::CreateIExecutionProvider for the built-in EP and + // Factory::CreateEpImpl in core/providers/webgpu/ep/factory.cc for the plugin EP). + // + // Safe to take the first: the factory reads nothing from the device except whether it is virtual, and + // Emscripten device discovery reports a single GPU entry derived from navigator.gpu, so there is at + // most one real candidate. Which GPU is used is decided by the adapter ORT Web hands to Dawn, not by + // this device (see env.webgpu.adapter and the WebGPU EP `device` option). A second entry only appears + // when virtual devices are enabled, and the factory registers those after the real ones, so the first + // is still a real device. Should that order ever change, EP creation fails with an explicit + // "selected on a virtual GPU device" error rather than silently running on the wrong device. + const epDevicesOffset = getInstance()._malloc(getInstance().PTR_SIZE); + allocs.push(epDevicesOffset); + getInstance().setValue(epDevicesOffset, selectedEpDevices[0], '*'); + + appendErrorCode = await getInstance()._OrtAppendExecutionProviderV2( + sessionOptionsHandle, + epDevicesOffset, + 1, + keysOffset, + valuesOffset, + epOptionsCount, + ); + } else { + const epNameDataOffset = allocWasmString(epName, allocs); + appendErrorCode = await getInstance()._OrtAppendExecutionProvider( sessionOptionsHandle, epNameDataOffset, keysOffset, valuesOffset, epOptionsCount, - )) !== 0 - ) { + ); + } + + if (appendErrorCode !== 0) { checkLastError(`Can't append execution provider: ${epName}.`); } } diff --git a/js/web/lib/wasm/wasm-types.ts b/js/web/lib/wasm/wasm-types.ts index 82d6808196a5f..02e1a0051c6ff 100644 --- a/js/web/lib/wasm/wasm-types.ts +++ b/js/web/lib/wasm/wasm-types.ts @@ -413,6 +413,16 @@ export interface OrtInferenceAPIs { providerOptionsValues: number, numKeys: number, ): Promise; + _OrtAppendExecutionProviderV2( + sessionOptionsHandle: number, + epDevices: number, + numEpDevices: number, + providerOptionsKeys: number, + providerOptionsValues: number, + numKeys: number, + ): Promise; + _OrtGetEpDevices(epDevices: number, numEpDevices: number): number; + _OrtEpDevice_EpName(epDevice: number): number; _OrtAddFreeDimensionOverride(sessionOptionsHandle: number, name: number, dim: number): number; _OrtAddSessionConfigEntry(sessionOptionsHandle: number, configKey: number, configValue: number): number; _OrtReleaseSessionOptions(sessionOptionsHandle: number): number; diff --git a/onnxruntime/contrib_ops/webgpu/quantization/matmul_nbits_mlp.cc b/onnxruntime/contrib_ops/webgpu/quantization/matmul_nbits_mlp.cc index f9891860a92c5..3b94812c481f9 100644 --- a/onnxruntime/contrib_ops/webgpu/quantization/matmul_nbits_mlp.cc +++ b/onnxruntime/contrib_ops/webgpu/quantization/matmul_nbits_mlp.cc @@ -11,6 +11,7 @@ #include "contrib_ops/webgpu/quantization/dp4a_matmul_nbits.h" #include "contrib_ops/webgpu/bert/skip_layer_norm.h" #include "contrib_ops/webgpu/webgpu_contrib_kernels.h" +#include "core/common/make_string.h" #include "core/providers/cpu/math/matmul_helper.h" #include "core/providers/webgpu/nn/layer_norm.h" #include "core/providers/webgpu/shader_helper.h" @@ -44,7 +45,8 @@ std::string EmitGateActivationExpr(MlpActivationKind kind, std::string_view gate switch (kind) { case MlpActivationKind::Silu: // SiLU(x) = x * sigmoid(x) - return std::string{gate_var} + " * (one / (one + exp(-" + std::string{gate_var} + ")))"; + // MakeString avoids a GCC 14 -Warray-bounds false positive from std::string concatenation here. + return MakeString(gate_var, " * (one / (one + exp(-", gate_var, ")))"); } ORT_THROW("MatMulNBitsMlp: unhandled MlpActivationKind ", static_cast(kind)); } diff --git a/onnxruntime/core/framework/plugin_data_transfer.cc b/onnxruntime/core/framework/plugin_data_transfer.cc index 8a6d37a4c30ce..9b80d5e24e7ec 100644 --- a/onnxruntime/core/framework/plugin_data_transfer.cc +++ b/onnxruntime/core/framework/plugin_data_transfer.cc @@ -44,8 +44,8 @@ Status DataTransfer::CopyTensors(const std::vector& src_dst_pairs) c streams.push_back(reinterpret_cast(src_dst_pairs[i].src_stream)); } - auto* status = impl_.CopyTensors(&impl_, src_values.data(), dst_values.data(), streams.data(), - src_dst_pairs.size()); + auto* status = impl_->CopyTensors(impl_.get(), src_values.data(), dst_values.data(), streams.data(), + src_dst_pairs.size()); return ToStatusAndRelease(status); } @@ -59,7 +59,7 @@ Status DataTransfer::CopyTensorImpl(const Tensor& src_tensor, Tensor& dst_tensor const OrtValue* src_ptr = &src; OrtValue* dst_ptr = &dst; OrtSyncStream* stream_ptr = reinterpret_cast(stream); - auto* status = impl_.CopyTensors(&impl_, &src_ptr, &dst_ptr, &stream_ptr, 1); + auto* status = impl_->CopyTensors(impl_.get(), &src_ptr, &dst_ptr, &stream_ptr, 1); return ToStatusAndRelease(status); } diff --git a/onnxruntime/core/framework/plugin_data_transfer.h b/onnxruntime/core/framework/plugin_data_transfer.h index e8ad29f4c0609..23ef3db98390c 100644 --- a/onnxruntime/core/framework/plugin_data_transfer.h +++ b/onnxruntime/core/framework/plugin_data_transfer.h @@ -2,6 +2,9 @@ // Licensed under the MIT License. #pragma once +#include +#include + #include "core/framework/data_transfer.h" #include "core/framework/error_code_helper.h" #include "core/framework/ort_value.h" @@ -11,21 +14,35 @@ namespace onnxruntime { namespace plugin_ep { +struct OrtDataTransferImplDeleter { + void operator()(OrtDataTransferImpl* data_transfer) const noexcept { + if (data_transfer != nullptr && data_transfer->Release != nullptr) { + data_transfer->Release(data_transfer); + } + } +}; + +using OrtDataTransferImplUniquePtr = std::unique_ptr; + /// /// Class to implement IDataTransfer for plugin execution providers. /// It uses the OrtDataTransferImpl from the plugin EP factory to implement the data transfer functionality. /// class DataTransfer : public IDataTransfer { public: - DataTransfer(OrtDataTransferImpl& impl) - : impl_{impl} { + explicit DataTransfer(OrtDataTransferImplUniquePtr impl) + : impl_{std::move(impl)} { + ORT_ENFORCE(impl_ != nullptr, "OrtDataTransferImpl must not be null."); + ORT_ENFORCE(impl_->Release != nullptr, "OrtDataTransferImpl must provide a Release function."); + ORT_ENFORCE(impl_->CanCopy != nullptr, "OrtDataTransferImpl must provide a CanCopy function."); + ORT_ENFORCE(impl_->CopyTensors != nullptr, "OrtDataTransferImpl must provide a CopyTensors function."); } bool CanCopy(const OrtDevice& src_device, const OrtDevice& dst_device) const override { const OrtMemoryDevice* src_memory_device = static_cast(&src_device); const OrtMemoryDevice* dst_memory_device = static_cast(&dst_device); - return impl_.CanCopy(&impl_, src_memory_device, dst_memory_device); + return impl_->CanCopy(impl_.get(), src_memory_device, dst_memory_device); } Status CopyTensor(const Tensor& src, Tensor& dst) const override { @@ -38,14 +55,10 @@ class DataTransfer : public IDataTransfer { Status CopyTensors(const std::vector& src_dst_pairs) const override; - ~DataTransfer() override { - impl_.Release(&impl_); - } - private: Status CopyTensorImpl(const Tensor& src, Tensor& dst, onnxruntime::Stream* stream = nullptr) const; - OrtDataTransferImpl& impl_; + OrtDataTransferImplUniquePtr impl_; }; } // namespace plugin_ep } // namespace onnxruntime diff --git a/onnxruntime/core/platform/emscripten/device_discovery.cc b/onnxruntime/core/platform/emscripten/device_discovery.cc new file mode 100644 index 0000000000000..fd11ce9ea00e9 --- /dev/null +++ b/onnxruntime/core/platform/emscripten/device_discovery.cc @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/platform/device_discovery.h" + +#include + +namespace onnxruntime { + +namespace { + +// Returns true if the JS environment exposes a WebGPU entry point. +// +// This is a capability check rather than a hardware enumeration, because a browser does not offer anything +// better. A GPUAdapter is only obtainable through the asynchronous navigator.gpu.requestAdapter(), which cannot +// be called from here, and even once obtained GPUAdapterInfo masks vendor details for fingerprinting +// resistance. The presence of navigator.gpu is therefore the strongest signal available at discovery time. It is +// also the same signal ORT Web itself uses to decide whether WebGPU is usable (see initEp() in +// js/web/lib/wasm/wasm-core-impl.ts). +// +// navigator is defined in both window and worker scopes, so this works on either. Hosts without navigator.gpu, +// including Node.js by default, report no GPU. +bool IsWebGpuAvailable() { + // clang-format off + // The body below is JavaScript, not C++; clang-format would otherwise rewrite `!==` as `!= =`. + return EM_ASM_INT({ + return (typeof navigator !== 'undefined' && navigator.gpu) ? 1 : 0; + }) != 0; + // clang-format on +} + +} // namespace + +std::unordered_set DeviceDiscovery::DiscoverDevicesForPlatform() { + std::unordered_set devices{}; + + devices.emplace(GetCpuDeviceFromCPUIDInfo()); + + // Report a GPU when the host exposes WebGPU. There is no OS-level device enumeration under Emscripten and a + // browser will not identify the GPU it is running on, but what matters for EP selection is that a GPU exists + // and is reachable, which is exactly what navigator.gpu establishes. + // + // vendor_id and device_id are left at 0 because they are unknown, not because the device is synthetic: this + // entry describes real hardware, so it is deliberately not marked with kOrtHardwareDevice_MetadataKey_IsVirtual + // and is a valid device to back an inference session. + if (IsWebGpuAvailable()) { + OrtHardwareDevice gpu_device{}; + gpu_device.type = OrtHardwareDeviceType_GPU; + gpu_device.vendor_id = 0; + gpu_device.device_id = 0; + gpu_device.metadata.Add("Description", "WebGPU"); + + devices.emplace(std::move(gpu_device)); + } + + return devices; +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/platform/windows/debug_alloc.cc b/onnxruntime/core/platform/windows/debug_alloc.cc index 14eb06a9cfa39..5107b720b1f5e 100644 --- a/onnxruntime/core/platform/windows/debug_alloc.cc +++ b/onnxruntime/core/platform/windows/debug_alloc.cc @@ -28,9 +28,13 @@ constexpr int c_callstack_limit = 32; // Maximum depth of callstack in leak tra #pragma warning(disable : 26490) // type.1 Don't use reinterpret_cast #pragma warning(disable : 26493) // type.4 Don't use C-style casts -#include -#include +#include +#include #include +#include +#include + +#include #include "debug_alloc.h" #include #pragma comment(lib, "Dbghelp.lib") @@ -228,34 +232,45 @@ Memory_LeakCheck::~Memory_LeakCheck() { const std::string string = message.str(); - // Google test has memory leaks that they haven't fixed. One such issue is tracked here: https://github.com/google/googletest/issues/692 - // - // In gtest-port.cc in function: static ThreadIdToThreadLocals* GetThreadLocalsMapLocked() - // static ThreadIdToThreadLocals* map = new ThreadIdToThreadLocals; - // - // In gtest-port.cc in Mutex::~Mutex() there is this comment: - // "Static mutexes are leaked intentionally. It is not thread-safe to try to clean them up." - // Which explains this leak inside of: void Mutex::ThreadSafeLazyInit() - // critical_section_ = new CRITICAL_SECTION; - // - // in google/re2 re2.cc initializes leaking singletons - // std::call_once(empty_once, []() { - // empty_string = new string; - // empty_named_groups = new std::map; - // empty_group_names = new std::map; }); - if (string.find("RtlRunOnceExecuteOnce") == std::string::npos && - string.find("re2::RE2::Init") == std::string::npos && - string.find("dynamic initializer for 'FLAGS_") == std::string::npos && - string.find("AbslFlagDefaultGenForgtest_") == std::string::npos && - string.find("AbslFlagDefaultGenForundefok::Gen") == std::string::npos && - string.find("::SetProgramUsageMessage") == std::string::npos && - string.find("testing::internal::ParseGoogleTestFlagsOnly") == std::string::npos && - string.find("testing::internal::Mutex::ThreadSafeLazyInit") == std::string::npos && - string.find("testing::internal::ThreadLocalRegistryImpl::GetThreadLocalsMapLocked") == std::string::npos && - string.find("testing::internal::ThreadLocalRegistryImpl::GetValueOnCurrentThread") == std::string::npos && - string.find("PyInit_onnxruntime_pybind11_state") == std::string::npos && - string.find("google::protobuf::internal::InitProtobufDefaultsSlow") == std::string::npos && - string.find("EtwEventWriteNoRegistration") == std::string::npos) { + constexpr auto allow_list = std::to_array({ + "RtlRunOnceExecuteOnce", + // In google/re2 re2.cc initializes leaking singletons + // std::call_once(empty_once, []() { + // empty_string = new string; + // empty_named_groups = new std::map; + // empty_group_names = new std::map; }); + "re2::RE2::Init", + "dynamic initializer for 'FLAGS_", + "AbslFlagDefaultGenForgtest_", + "AbslFlagDefaultGenForundefok::Gen", + "::SetProgramUsageMessage", + // Google test has memory leaks that they haven't fixed. One such issue is tracked here: + // https://github.com/google/googletest/issues/692 + // + // In gtest-port.cc in function: static ThreadIdToThreadLocals* GetThreadLocalsMapLocked() + // static ThreadIdToThreadLocals* map = new ThreadIdToThreadLocals; + // + // In gtest-port.cc in Mutex::~Mutex() there is this comment: + // "Static mutexes are leaked intentionally. It is not thread-safe to try to clean them up." + // Which explains this leak inside of: void Mutex::ThreadSafeLazyInit() + // critical_section_ = new CRITICAL_SECTION; + "testing::internal::ParseGoogleTestFlagsOnly", + "testing::internal::Mutex::ThreadSafeLazyInit", + "testing::internal::ThreadLocalRegistryImpl::GetThreadLocalsMapLocked", + "testing::internal::ThreadLocalRegistryImpl::GetValueOnCurrentThread", + "PyInit_onnxruntime_pybind11_state", + "google::protobuf::internal::InitProtobufDefaultsSlow", + "EtwEventWriteNoRegistration", +#if defined(USE_WEBGPU) + // In onnxruntime/core/providers/webgpu/webgpu_context_dawn_platform.cc:GetDawnPlatform(), there is an + // intentional leak of a function-local static: + // static DawnPlatform* platform = new DawnPlatform(); + "onnxruntime::webgpu::`anonymous namespace'::GetDawnPlatform", +#endif // defined(USE_WEBGPU), + }); + + if (std::none_of(allow_list.begin(), allow_list.end(), + [&string](std::string_view allowed) { return string.find(allowed) != std::string::npos; })) { if (leaked_bytes == 0) DebugPrint("\n-----Starting Heap Trace-----\n\n"); diff --git a/onnxruntime/core/providers/webgpu/ep/api.cc b/onnxruntime/core/providers/webgpu/ep/api.cc index 18b1c658e5726..c0b1550d70cfd 100644 --- a/onnxruntime/core/providers/webgpu/ep/api.cc +++ b/onnxruntime/core/providers/webgpu/ep/api.cc @@ -1,21 +1,31 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#define ORT_API_MANUAL_INIT -#include "onnxruntime_cxx_api.h" -#undef ORT_API_MANUAL_INIT - #include #include #include #include +#include "ep/api.h" + #include "core/platform/env_var.h" #include "core/providers/webgpu/ep/factory.h" #include "core/session/onnxruntime_env_config_keys.h" -// To make symbols visible on macOS/iOS -#ifdef __APPLE__ +// When this EP is statically linked into the host binary instead of being built as a separate shared library, the +// entry points below are given a `WebGpu_` prefix. +// +// The shared library build must leave the entry points unprefixed, because they are resolved by exact name. +#if defined(ORT_PLUGIN_EP_STATICALLY_LINKED) +#define ORT_PLUGIN_EP_ENTRY_POINT_CONCAT_IMPL(prefix, name) prefix##name +#define ORT_PLUGIN_EP_ENTRY_POINT_CONCAT(prefix, name) ORT_PLUGIN_EP_ENTRY_POINT_CONCAT_IMPL(prefix, name) +#define ORT_PLUGIN_EP_ENTRY_POINT(name) ORT_PLUGIN_EP_ENTRY_POINT_CONCAT(WebGpu_, name) +#else +#define ORT_PLUGIN_EP_ENTRY_POINT(name) name +#endif + +// To make symbols visible on macOS/iOS. Only needed when this EP is built as a separate shared library. +#if defined(__APPLE__) && !defined(ORT_PLUGIN_EP_STATICALLY_LINKED) #define EXPORT_SYMBOL __attribute__((visibility("default"))) #else #define EXPORT_SYMBOL @@ -28,11 +38,13 @@ void CleanupKernelRegistries(); } // namespace webgpu } // namespace onnxruntime +#if !defined(ORT_PLUGIN_EP_STATICALLY_LINKED) namespace google { namespace protobuf { void ShutdownProtobufLibrary(); } // namespace protobuf } // namespace google +#endif namespace { // Set to "1" to allow WebGPU to be advertised against a CPU hardware device when no GPU hardware device is available. @@ -43,9 +55,10 @@ extern "C" { // // Public symbols // -EXPORT_SYMBOL OrtStatus* CreateEpFactories(const char* /*registration_name*/, const OrtApiBase* ort_api_base, - const OrtLogger* default_logger, - OrtEpFactory** factories, size_t max_factories, size_t* num_factories) noexcept { +EXPORT_SYMBOL OrtStatus* ORT_PLUGIN_EP_ENTRY_POINT(CreateEpFactories)( + const char* /*registration_name*/, const OrtApiBase* ort_api_base, + const OrtLogger* default_logger, + OrtEpFactory** factories, size_t max_factories, size_t* num_factories) noexcept { { // Note: We can't use the EXCEPTION_TO_RETURNED_STATUS_BEGIN/EXCEPTION_TO_RETURNED_STATUS_END macros around the // call to `onnxruntime::ep::ApiInit()` because they depend on the API to create `OrtStatus`. We need to create an @@ -117,7 +130,7 @@ EXPORT_SYMBOL OrtStatus* CreateEpFactories(const char* /*registration_name*/, co EXCEPTION_TO_RETURNED_STATUS_END } -EXPORT_SYMBOL OrtStatus* ReleaseEpFactory(OrtEpFactory* factory) noexcept { +EXPORT_SYMBOL OrtStatus* ORT_PLUGIN_EP_ENTRY_POINT(ReleaseEpFactory)(OrtEpFactory* factory) noexcept { EXCEPTION_TO_RETURNED_STATUS_BEGIN // STEP.1 - Release the factory delete static_cast(factory); @@ -131,8 +144,12 @@ EXPORT_SYMBOL OrtStatus* ReleaseEpFactory(OrtEpFactory* factory) noexcept { // STEP.4 - Destroy the global default logger wrapper ::onnxruntime::ep::adapter::LoggingManager::DestroyDefaultLogger(); - // STEP.5 - Shutdown protobuf library +#if !defined(ORT_PLUGIN_EP_STATICALLY_LINKED) + // STEP.5 - Shutdown protobuf library. + // Only do this when this EP owns the process-global state, i.e. when it is built as a separate library. + // When statically linked into the host binary, protobuf is shared with the host, which owns its lifetime. google::protobuf::ShutdownProtobufLibrary(); +#endif return nullptr; EXCEPTION_TO_RETURNED_STATUS_END diff --git a/onnxruntime/core/providers/webgpu/math/gemm.h b/onnxruntime/core/providers/webgpu/math/gemm.h index bf97ff0f2b994..fcffc523cfd58 100644 --- a/onnxruntime/core/providers/webgpu/math/gemm.h +++ b/onnxruntime/core/providers/webgpu/math/gemm.h @@ -60,11 +60,11 @@ class Gemm final : public WebGpuKernel { }; Gemm(const OpKernelInfo& info) : WebGpuKernel(info) { - int64_t transA_temp; + int64_t transA_temp = 0; info.GetAttrOrDefault("transA", &transA_temp, static_cast(0)); transA_ = transA_temp != 0; - int64_t transB_temp; + int64_t transB_temp = 0; info.GetAttrOrDefault("transB", &transB_temp, static_cast(0)); transB_ = transB_temp != 0; diff --git a/onnxruntime/core/providers/webgpu/math/softmax.h b/onnxruntime/core/providers/webgpu/math/softmax.h index 532a56ff0be41..cae5581e9ff2f 100644 --- a/onnxruntime/core/providers/webgpu/math/softmax.h +++ b/onnxruntime/core/providers/webgpu/math/softmax.h @@ -15,7 +15,7 @@ class Softmax final : public WebGpuKernel { public: Softmax(const OpKernelInfo& info) : WebGpuKernel{info} { opset_ = info.node().SinceVersion(); - int64_t axis; + int64_t axis = 0; Status status = info.GetAttr("axis", &axis); if (status.IsOK()) { diff --git a/onnxruntime/core/providers/webgpu/math/top_k.h b/onnxruntime/core/providers/webgpu/math/top_k.h index 6652357ec71cc..08b5a2372c6ae 100644 --- a/onnxruntime/core/providers/webgpu/math/top_k.h +++ b/onnxruntime/core/providers/webgpu/math/top_k.h @@ -82,7 +82,7 @@ class TopK final : public WebGpuKernel { info.GetAttrOrDefault("largest", &largest_, 1); info.GetAttrOrDefault("sorted", &sorted_, 1); if (opset_ <= 9) { - int64_t k_temp; + int64_t k_temp = 0; ORT_ENFORCE(info.GetAttr("k", &k_temp).IsOK()); attr_k_ = k_temp; } diff --git a/onnxruntime/core/providers/webgpu/tensor/cast.h b/onnxruntime/core/providers/webgpu/tensor/cast.h index f86771854c2b6..6ff0e7cd40590 100644 --- a/onnxruntime/core/providers/webgpu/tensor/cast.h +++ b/onnxruntime/core/providers/webgpu/tensor/cast.h @@ -36,7 +36,7 @@ class CastProgram final : public Program { class Cast final : public WebGpuKernel { public: Cast(const OpKernelInfo& info) : WebGpuKernel(info) { - int64_t to; + int64_t to = 0; Status status = info.GetAttr("to", &to); ORT_ENFORCE(status.IsOK(), "Attribute to is not set."); to_ = onnxruntime::narrow(to); diff --git a/onnxruntime/core/session/allocator_adapters.cc b/onnxruntime/core/session/allocator_adapters.cc index 62316dd737bf1..84d0641d9a717 100644 --- a/onnxruntime/core/session/allocator_adapters.cc +++ b/onnxruntime/core/session/allocator_adapters.cc @@ -139,9 +139,8 @@ onnxruntime::AllocatorPtr OrtAllocatorImplWrappingIAllocator::GetWrappedIAllocat IAllocatorImplWrappingOrtAllocator::IAllocatorImplWrappingOrtAllocator(OrtAllocator* ort_allocator) : IAllocator(*ort_allocator->Info(ort_allocator)) { - ort_allocator_ = OrtAllocatorUniquePtr(ort_allocator, [](OrtAllocator*) { - // no-op - }); + void (*noop_deleter)(OrtAllocator*) = [](OrtAllocator*) {}; + ort_allocator_ = OrtAllocatorUniquePtr(ort_allocator, noop_deleter); } IAllocatorImplWrappingOrtAllocator::IAllocatorImplWrappingOrtAllocator(OrtAllocatorUniquePtr ort_allocator) diff --git a/onnxruntime/core/session/environment.cc b/onnxruntime/core/session/environment.cc index f09238c27e451..fbadf6da73678 100644 --- a/onnxruntime/core/session/environment.cc +++ b/onnxruntime/core/session/environment.cc @@ -3,9 +3,12 @@ #include "core/session/environment.h" +#include #include +#include #include "core/common/basic_types.h" +#include "core/common/safeint.h" #include "core/framework/allocator.h" #include "core/framework/allocator_utils.h" #include "core/framework/error_code_helper.h" @@ -18,10 +21,12 @@ #include "core/session/inference_session.h" #include "core/session/onnxruntime_env_config_keys.h" #include "core/session/onnxruntime_ep_device_ep_metadata_keys.h" +#include "core/session/plugin_ep/ep_allocator_utils.h" #include "core/session/plugin_ep/ep_factory_internal.h" #include "core/session/plugin_ep/ep_library_internal.h" #include "core/session/plugin_ep/ep_library_plugin.h" #include "core/session/plugin_ep/ep_library_provider_bridge.h" +#include "core/session/plugin_ep/ep_static_plugins.h" #include "core/session/ort_apis.h" #include "core/session/utils.h" @@ -541,14 +546,16 @@ namespace { Status CreateDataTransferForFactory(OrtEpFactory& ep_factory, std::unique_ptr& data_transfer) { OrtDataTransferImpl* data_transfer_impl = nullptr; - OrtStatus* status = ep_factory.CreateDataTransfer(&ep_factory, &data_transfer_impl); - if (status != nullptr) { + OrtStatus* ort_status = ep_factory.CreateDataTransfer(&ep_factory, &data_transfer_impl); + plugin_ep::OrtDataTransferImplUniquePtr owned_impl{data_transfer_impl}; + + if (ort_status != nullptr) { return ORT_MAKE_STATUS(ONNXRUNTIME, EP_FAIL, - "Error creating data transfer: ", ToStatusAndRelease(status).ToString()); + "Error creating data transfer: ", ToStatusAndRelease(ort_status).ToString()); } - if (data_transfer_impl != nullptr) { - data_transfer = std::make_unique(*data_transfer_impl); + if (owned_impl != nullptr) { + data_transfer = std::make_unique(std::move(owned_impl)); } return Status::OK(); @@ -568,62 +575,138 @@ Status Environment::RegisterExecutionProviderLibrary(const std::string& registra const std::vector& internal_factories) { #if defined(_WIN32) || defined(ORT_USE_TELEMETRY) const Env& env = Env::Default(); -#endif + #if defined(ORT_USE_TELEMETRY) const TimePoint tp = std::chrono::high_resolution_clock::now(); #endif + // Windows ETW needs these calls even without ORT_USE_TELEMETRY. -#if defined(_WIN32) || defined(ORT_USE_TELEMETRY) env.GetTelemetryProvider().LogRegisterEpLibraryStart(registration_name); -#endif +#endif // defined(_WIN32) || defined(ORT_USE_TELEMETRY) auto status = Status::OK(); ORT_TRY { - // Contain early Status returns so they reach the End event. status = [&]() -> Status { + bool registration_succeeded = false; + if (ep_libraries_.contains(registration_name)) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "library is already registered under ", registration_name); + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "EP library is already registered under ", registration_name); } - // create the EpInfo which loads the library if required - std::unique_ptr ep_info = nullptr; - ORT_RETURN_IF_ERROR(EpInfo::Create(std::move(ep_library), ep_info)); + std::unique_ptr ep_info; + ORT_RETURN_IF_ERROR(EpInfo::Create(std::move(ep_library), ep_info, internal_factories)); - // add the pointers to the OrtEpDevice instances to our global list - execution_devices_.reserve(execution_devices_.size() + ep_info->execution_devices.size()); - for (const auto& ed : ep_info->execution_devices) { - execution_devices_.push_back(ed.get()); - - // add shared allocators so they're available without an inference session being required. - // we don't replace an existing allocator as we just need one to exist for the OrtMemoryInfo and we don't want - // to blow away any custom allocators previously added by the user. - if (ed->device_memory_info != nullptr) { - ORT_RETURN_IF_ERROR(CreateSharedAllocatorImpl(*ed, *ed->device_memory_info, OrtDeviceAllocator, nullptr, - nullptr, /*replace_existing*/ false)); + // Register shared allocators + const size_t max_allocator_count = SafeInt(ep_info->execution_devices.size()) * 2; + std::vector registered_allocator_memory_infos; + registered_allocator_memory_infos.reserve(max_allocator_count); + + auto rollback_shared_allocator_registrations = gsl::finally([&]() { + if (registration_succeeded) { + return; } + for (auto it = registered_allocator_memory_infos.rbegin(); it != registered_allocator_memory_infos.rend(); ++it) { + ORT_IGNORE_RETURN_VALUE(UnregisterAllocatorImpl(**it, /*error_if_not_found*/ false)); + } + }); - if (ed->host_accessible_memory_info != nullptr) { - ORT_RETURN_IF_ERROR(CreateSharedAllocatorImpl(*ed, *ed->host_accessible_memory_info, OrtDeviceAllocator, - nullptr, nullptr, /*replace_existing*/ false)); + for (const auto& ed : ep_info->execution_devices) { + // Add shared allocators so they're available without an inference session being required. Do not replace an + // existing allocator: we only need one for the OrtMemoryInfo and must preserve user-provided allocators. + for (const OrtMemoryInfo* memory_info : {ed->device_memory_info, ed->host_accessible_memory_info}) { + if (memory_info == nullptr) { + continue; + } + + OrtAllocator* registered_allocator = nullptr; + ORT_RETURN_IF_ERROR(CreateSharedAllocatorImpl(*ed, *memory_info, OrtDeviceAllocator, nullptr, + ®istered_allocator, /*replace_existing*/ false)); + if (registered_allocator != nullptr) { + registered_allocator_memory_infos.push_back(memory_info); + } } } + // Register data transfers + std::vector registered_data_transfers; + registered_data_transfers.reserve(ep_info->factories.size()); + ep_info->data_transfers.reserve(ep_info->factories.size()); + + auto rollback_data_transfer_registrations = gsl::finally([&]() { + if (registration_succeeded) { + return; + } + + for (auto it = registered_data_transfers.rbegin(); it != registered_data_transfers.rend(); ++it) { + ORT_IGNORE_RETURN_VALUE(data_transfer_mgr_.UnregisterDataTransfer(*it)); + } + }); + for (auto* factory : ep_info->factories) { std::unique_ptr data_transfer; ORT_RETURN_IF_ERROR(CreateDataTransferForFactory(*factory, data_transfer)); if (data_transfer) { - ep_info->data_transfers.push_back(data_transfer.get()); // store so we can unregister in the unload + auto* data_transfer_ptr = data_transfer.get(); ORT_RETURN_IF_ERROR(data_transfer_mgr_.RegisterDataTransfer(std::move(data_transfer))); + registered_data_transfers.push_back(data_transfer_ptr); + ep_info->data_transfers.push_back(data_transfer_ptr); } } - for (const auto& internal_factory : internal_factories) { - internal_ep_factories_.insert(internal_factory); + // Register internal factories + std::vector registered_internal_factories; + registered_internal_factories.reserve(internal_factories.size()); + + auto rollback_internal_factory_registration = gsl::finally([&]() { + if (registration_succeeded) { + return; + } + + for (auto it = registered_internal_factories.rbegin(); it != registered_internal_factories.rend(); ++it) { + internal_ep_factories_.erase(*it); + } + }); + + for (auto* internal_factory : internal_factories) { + if (internal_ep_factories_.insert(internal_factory).second) { + registered_internal_factories.push_back(internal_factory); + } } - ep_libraries_[registration_name] = std::move(ep_info); + // Register EP devices + auto rollback_execution_devices_registration = gsl::finally( + [&, original_size = execution_devices_.size()]() { + if (registration_succeeded) { + return; + } + + execution_devices_.erase(execution_devices_.begin() + original_size, + execution_devices_.end()); + }); + + execution_devices_.reserve(SafeInt(execution_devices_.size()) + + ep_info->execution_devices.size()); + std::transform(ep_info->execution_devices.begin(), ep_info->execution_devices.end(), + std::back_inserter(execution_devices_), + [](const std::unique_ptr& ep_device) { + return ep_device.get(); + }); + + // Finally, register the EP library + + // Notes: + // `ep_info`'s target must stay alive for any rollback logic that may need to run. + // Directly moving into `ep_libraries_.emplace()` could cause it to be prematurely destroyed if emplace throws + // after ownership has transferred from `ep_info`. This is why we first emplace an empty unique_ptr and then + // swap into it. + // + // We ignore `emplace_result.second` since we already checked for an existing entry in `ep_libraries_` above. + const auto emplace_result = ep_libraries_.emplace(registration_name, nullptr); + emplace_result.first->second.swap(ep_info); + + registration_succeeded = true; return Status::OK(); }(); } @@ -651,8 +734,8 @@ Status Environment::RegisterExecutionProviderLibrary(const std::string& registra Status Environment::CreateAndRegisterInternalEps() { // Capture allow_virtual_devices here (lock-free) and pass it to the internal EP factories at // construction. The internal WebGPU EP factory needs it in GetSupportedDevices but cannot query the - // OrtEnv singleton there: internal EPs are registered while OrtEnv's creation mutex is already held on - // this thread, so the query would self-deadlock. + // OrtEnv singleton there: internal EPs are registered from Environment::Create, before OrtEnv + // publishes p_instance_, so there is no instance to find. const bool allow_virtual_devices = num_allow_virtual_device_uses_ > 0; auto internal_ep_libraries = EpLibraryInternal::CreateInternalEps(allow_virtual_devices); for (auto& ep_library : internal_ep_libraries) { @@ -666,6 +749,20 @@ Status Environment::CreateAndRegisterInternalEps() { return Status::OK(); } +Status Environment::CreateAndRegisterStaticPluginEps(StaticPluginEpRegistrationToken) { + // Intentionally does not take mutex_ and calls the private, non-locking RegisterExecutionProviderLibrary overload. + // This is safe because the caller holds the OrtEnv creation mutex, so no other thread can obtain a reference to this + // Environment yet. Taking mutex_ here would deadlock if a plugin EP called a locking Environment API from + // OrtEpFactory::GetSupportedDevices. + auto static_plugin_ep_libraries = CreateStaticPluginEpLibraries(); + for (auto& ep_library : static_plugin_ep_libraries) { + const std::string registration_name = ep_library->RegistrationName(); + ORT_RETURN_IF_ERROR(RegisterExecutionProviderLibrary(registration_name, std::move(ep_library))); + } + + return Status::OK(); +} + Status Environment::RegisterExecutionProviderLibrary(const std::string& registration_name, const ORTCHAR_T* lib_path) { std::lock_guard lock{mutex_}; @@ -868,66 +965,52 @@ Status Environment::CreateSharedAllocatorImpl(const OrtEpDevice& ep_device, "any arena options via the allocator options."); } - // we need to remove from shared_ort_allocators_ first in case the entry in shared_allocators_ owns the pointer in - // shared_ort_allocators_. - if (auto it = FindExistingAllocator(shared_ort_allocators_, memory_info, /*match_name*/ true); - it != shared_ort_allocators_.end()) { - if (!replace_existing) { - LOGS_DEFAULT(INFO) << "A shared allocator is already registered for " << memory_info - << ". Skipping EP allocator creation."; - return Status::OK(); - } - - shared_ort_allocators_.erase(it); - } - // we only want one shared allocator for an OrtDevice in the shared_allocators_ so that it's deterministic which // one will be used for an inference session. ignore the name so that is the case. - if (auto it = FindExistingAllocator(shared_allocators_, memory_info, /*match_name*/ false); - it != shared_allocators_.end()) { - if (!replace_existing) { - LOGS_DEFAULT(INFO) << "A shared allocator is already registered for " << memory_info - << ". Skipping EP allocator creation."; - return Status::OK(); + auto existing_ort_allocator = + FindExistingAllocator(shared_ort_allocators_, memory_info, /*match_name*/ true); + + auto existing_shared_allocator = + FindExistingAllocator(shared_allocators_, memory_info, /*match_name*/ false); + + if (!replace_existing && + (existing_ort_allocator != shared_ort_allocators_.end() || + existing_shared_allocator != shared_allocators_.end())) { + if (allocator_out != nullptr) { + *allocator_out = nullptr; } - shared_allocators_.erase(it); + LOGS_DEFAULT(INFO) << "A shared allocator is already registered for " << memory_info + << ". Skipping EP allocator creation."; + return Status::OK(); } - OrtAllocator* allocator = nullptr; - auto* ort_status = ep_device.ep_factory->CreateAllocator(ep_device.ep_factory, &memory_info, allocator_options, - &allocator); - if (ort_status != nullptr) { - return ToStatusAndRelease(ort_status); + // we need to remove from shared_ort_allocators_ first in case the entry in shared_allocators_ owns the pointer in + // shared_ort_allocators_. + if (existing_ort_allocator != shared_ort_allocators_.end()) { + shared_ort_allocators_.erase(existing_ort_allocator); } - if (allocator->Info(allocator)->alloc_type == OrtAllocatorType::OrtArenaAllocator) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, - "OrtEpFactory returned an allocator with OrtAllocatorType of OrtArenaAllocator. " - "This type is reserved for ONNX Runtime internal usage only, as any arena usage by the " - "EP library should be opaque to ORT"); + if (existing_shared_allocator != shared_allocators_.end()) { + shared_allocators_.erase(existing_shared_allocator); } - auto* ep_factory = ep_device.ep_factory; - auto ort_allocator = OrtAllocatorUniquePtr(allocator, - [ep_factory](OrtAllocator* allocator) { - ep_factory->ReleaseAllocator(ep_factory, allocator); - }); - - shared_ort_allocators_.insert(allocator); + // Create a new allocator. - // Wrap as IArena when the plugin allocator implements Shrink(), making it - // discoverable by session-level arena management (e.g. ShrinkMemoryArenas). + auto* ep_factory = ep_device.ep_factory; + OrtAllocator* raw_ort_allocator = nullptr; AllocatorPtr shared_allocator; - if (allocator->version >= 25 && allocator->Shrink != nullptr) { - shared_allocator = std::make_shared(std::move(ort_allocator)); - } else { - shared_allocator = std::make_shared(std::move(ort_allocator)); - } + ORT_RETURN_IF_ERROR(ep_allocator_utils::CreateAndWrapEpAllocator(*ep_factory, memory_info, allocator_options, + shared_allocator, &raw_ort_allocator)); + ORT_RETURN_IF_NOT(raw_ort_allocator != nullptr, "OrtEpFactory returned a null allocator."); + + shared_allocators_.reserve(SafeInt(shared_allocators_.size()) + 1); + const auto [_, inserted] = shared_ort_allocators_.insert(raw_ort_allocator); + ORT_RETURN_IF_NOT(inserted, "OrtEpFactory returned an allocator that is already registered."); shared_allocators_.push_back(std::move(shared_allocator)); if (allocator_out != nullptr) { - *allocator_out = allocator; + *allocator_out = raw_ort_allocator; } return Status::OK(); @@ -951,13 +1034,17 @@ Status Environment::EpInfo::Create(std::unique_ptr library_in, std::u return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "EpLibrary was null"); } - out.reset(new EpInfo()); // can't use make_unique with private ctor - EpInfo& instance = *out; + auto instance_unique_ptr = std::unique_ptr(new EpInfo()); // can't use make_unique with private ctor + EpInfo& instance = *instance_unique_ptr; instance.library = std::move(library_in); instance.internal_factories = internal_factories; ORT_RETURN_IF_ERROR(instance.library->Load()); - instance.factories = instance.library->GetFactories(); + + { + auto factories = instance.library->GetFactories(); + instance.factories.assign(factories.begin(), factories.end()); + } // OrtHardwareDevice instances to pass to GetSupportedDevices. const auto& sorted_devices = GetSortedHardwareDevices(); @@ -988,6 +1075,7 @@ Status Environment::EpInfo::Create(std::unique_ptr library_in, std::u } } + out = std::move(instance_unique_ptr); return Status::OK(); } diff --git a/onnxruntime/core/session/ort_env.cc b/onnxruntime/core/session/ort_env.cc index 68ecd7c2c804e..7786a051d6861 100644 --- a/onnxruntime/core/session/ort_env.cc +++ b/onnxruntime/core/session/ort_env.cc @@ -4,6 +4,7 @@ // this file contains implementations of the C API #include +#include #include "ort_env.h" #include "core/session/ort_apis.h" @@ -29,7 +30,7 @@ void CleanupWebGpuContexts(); OrtEnv* OrtEnv::p_instance_; int OrtEnv::ref_count_ = 0; -std::mutex OrtEnv::m_; +std::recursive_mutex OrtEnv::m_; OrtEnv::OrtEnv(std::unique_ptr value1) : value_(std::move(value1)) { @@ -57,7 +58,7 @@ OrtEnvPtr OrtEnv::GetOrCreateInstance(const OrtEnv::LoggingManagerConstructionIn onnxruntime::common::Status& status, const OrtThreadingOptions* tp_options, const OrtKeyValuePairs* config_entries) { - std::lock_guard lock(m_); + std::lock_guard lock(m_); if (!p_instance_) { std::unique_ptr lmgr; std::string name = lm_info.logid; @@ -89,6 +90,26 @@ OrtEnvPtr OrtEnv::GetOrCreateInstance(const OrtEnv::LoggingManagerConstructionIn // Use 'new' to allocate OrtEnv, as it will be managed by p_instance_ // and deleted in ReleaseEnv or leaked if g_is_process_shutting_down is true. p_instance_ = new OrtEnv(std::move(env)); + + // Take this caller's reference before running any execution provider code below, so that if a provider + // acquires and releases an OrtEnvPtr the instance isn't destroyed out from under us. Holding it in an OrtEnvPtr + // also unpublishes and destroys the half-built instance if anything below fails or throws. + ++ref_count_; + OrtEnvPtr instance{p_instance_, OrtEnv::Release}; + +#if !defined(ORT_MINIMAL_BUILD) + // Register statically linked plugin EPs *after* p_instance_ is published. They use the public ORT API, so any + // OrtEnv API they call while enumerating their devices must be able to find the instance. m_ is recursive so + // that such a call from this thread doesn't self-deadlock. + status = p_instance_->GetEnvironment().CreateAndRegisterStaticPluginEps( + Environment::StaticPluginEpRegistrationToken{}); + + if (!status.IsOK()) { + return OrtEnvPtr(nullptr, OrtEnv::Release); + } +#endif // !defined(ORT_MINIMAL_BUILD) + + return instance; } ++ref_count_; @@ -104,7 +125,7 @@ void OrtEnv::Release(OrtEnv* env_ptr) { OrtEnv* instance_to_delete = nullptr; { // Scope for the lock guard - std::lock_guard lock(m_); + std::lock_guard lock(m_); assert(p_instance_ == env_ptr); --ref_count_; @@ -135,7 +156,7 @@ void OrtEnv::Release(OrtEnv* env_ptr) { /*static*/ OrtEnvPtr OrtEnv::TryGetInstance() { - std::lock_guard lock(m_); + std::lock_guard lock(m_); if (p_instance_) { ++ref_count_; diff --git a/onnxruntime/core/session/ort_env.h b/onnxruntime/core/session/ort_env.h index 2f6c270e9c5e7..fbfaa574c62f3 100644 --- a/onnxruntime/core/session/ort_env.h +++ b/onnxruntime/core/session/ort_env.h @@ -80,7 +80,11 @@ struct OrtEnv { // tracking active users. It is set to nullptr when the last reference is released // (and not shutting down). static OrtEnv* p_instance_; - static std::mutex m_; + // Recursive solely to support static plugin EP registration, which happens while this mutex is held: a plugin EP + // may call an OrtEnv API (e.g. OrtEpApi::GetEnvConfigEntries) from OrtEpFactory::GetSupportedDevices, which + // re-enters TryGetInstance(). No other code path requires recursive locking, so this can revert to std::mutex if + // static plugin EPs are no longer registered here. + static std::recursive_mutex m_; static int ref_count_; std::unique_ptr value_; diff --git a/onnxruntime/core/session/plugin_ep/ep_allocator_utils.cc b/onnxruntime/core/session/plugin_ep/ep_allocator_utils.cc new file mode 100644 index 0000000000000..5772a71ded6c8 --- /dev/null +++ b/onnxruntime/core/session/plugin_ep/ep_allocator_utils.cc @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/session/plugin_ep/ep_allocator_utils.h" + +#include +#include +#include + +#include "core/common/common.h" +#include "core/framework/error_code_helper.h" +#include "core/session/allocator_adapters.h" + +namespace onnxruntime { +namespace ep_allocator_utils { + +Status CreateAndWrapEpAllocator(OrtEp* ep, + OrtEpFactory& ep_factory, + const OrtMemoryInfo& memory_info, + const OrtKeyValuePairs* ep_factory_allocator_options, + AllocatorPtr& allocator_out, + OrtAllocator** raw_allocator_out) { + const bool has_ep_create = ep != nullptr && ep->CreateAllocator != nullptr; + const bool has_factory_create = ep_factory.CreateAllocator != nullptr; + + ORT_RETURN_IF_NOT((!has_ep_create && !has_factory_create) || ep_factory.ReleaseAllocator != nullptr, + "OrtEpFactory must implement ReleaseAllocator when OrtEp::CreateAllocator or " + "OrtEpFactory::CreateAllocator is implemented."); + + if (!has_ep_create && !has_factory_create) { + allocator_out = nullptr; + if (raw_allocator_out != nullptr) { + *raw_allocator_out = nullptr; + } + return Status::OK(); + } + + std::function allocator_deleter = [&ep_factory](OrtAllocator* allocator) { + ep_factory.ReleaseAllocator(&ep_factory, allocator); + }; + + OrtAllocator* raw_allocator = nullptr; + OrtStatus* creation_ort_status = + has_ep_create + ? ep->CreateAllocator(ep, &memory_info, &raw_allocator) + : ep_factory.CreateAllocator(&ep_factory, &memory_info, ep_factory_allocator_options, &raw_allocator); + OrtAllocatorUniquePtr owned_allocator{raw_allocator, std::move(allocator_deleter)}; + + if (creation_ort_status != nullptr) { + return ToStatusAndRelease(creation_ort_status); + } + + AllocatorPtr wrapped_allocator; + if (owned_allocator != nullptr) { + const std::string_view creator = has_ep_create ? "OrtEp" : "OrtEpFactory"; + + const OrtMemoryInfo* allocator_memory_info = owned_allocator->Info(owned_allocator.get()); + ORT_RETURN_IF_NOT(allocator_memory_info != nullptr, creator, " returned an allocator with null memory info."); + + ORT_RETURN_IF_NOT(allocator_memory_info->alloc_type != OrtAllocatorType::OrtArenaAllocator, + creator, + " returned an allocator with OrtAllocatorType of OrtArenaAllocator. " + "This type is reserved for ONNX Runtime internal usage only, as any arena usage by the " + "EP library should be opaque to ORT"); + + constexpr uint32_t kOrtAllocatorShrinkMinVersion = 25; + if (owned_allocator->version >= kOrtAllocatorShrinkMinVersion && owned_allocator->Shrink != nullptr) { + wrapped_allocator = std::make_shared(std::move(owned_allocator)); + } else { + wrapped_allocator = std::make_shared(std::move(owned_allocator)); + } + } + + if (raw_allocator_out != nullptr) { + *raw_allocator_out = raw_allocator; + } + allocator_out = std::move(wrapped_allocator); + return Status::OK(); +} + +} // namespace ep_allocator_utils +} // namespace onnxruntime diff --git a/onnxruntime/core/session/plugin_ep/ep_allocator_utils.h b/onnxruntime/core/session/plugin_ep/ep_allocator_utils.h new file mode 100644 index 0000000000000..81ea11ad3b571 --- /dev/null +++ b/onnxruntime/core/session/plugin_ep/ep_allocator_utils.h @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/status.h" +#include "core/framework/allocator.h" +#include "core/session/onnxruntime_c_api.h" + +namespace onnxruntime { +namespace ep_allocator_utils { + +// Creates a plugin EP allocator using OrtEp::CreateAllocator when available, otherwise OrtEpFactory::CreateAllocator +// when available, and wraps the result as an IAllocator. Returns a null allocator if neither callback is implemented. +// Outputs are modified only on success. +Status CreateAndWrapEpAllocator(OrtEp* ep, + OrtEpFactory& ep_factory, + const OrtMemoryInfo& memory_info, + const OrtKeyValuePairs* ep_factory_allocator_options, + AllocatorPtr& allocator_out, + OrtAllocator** raw_allocator_out = nullptr); + +// Creates a plugin EP allocator using OrtEpFactory::CreateAllocator when available and wraps the result as an +// IAllocator. Returns a null allocator if the callback is not implemented. +// Outputs are modified only on success. +inline Status CreateAndWrapEpAllocator(OrtEpFactory& ep_factory, + const OrtMemoryInfo& memory_info, + const OrtKeyValuePairs* ep_factory_allocator_options, + AllocatorPtr& allocator_out, + OrtAllocator** raw_allocator_out = nullptr) { + return CreateAndWrapEpAllocator(/*ep*/ nullptr, ep_factory, memory_info, ep_factory_allocator_options, + allocator_out, raw_allocator_out); +} + +} // namespace ep_allocator_utils +} // namespace onnxruntime diff --git a/onnxruntime/core/session/plugin_ep/ep_factory_webgpu.h b/onnxruntime/core/session/plugin_ep/ep_factory_webgpu.h index dcb0b1cef6ee1..f8542b4db485a 100644 --- a/onnxruntime/core/session/plugin_ep/ep_factory_webgpu.h +++ b/onnxruntime/core/session/plugin_ep/ep_factory_webgpu.h @@ -13,9 +13,8 @@ namespace onnxruntime { class WebGpuEpFactory : public EpFactoryInternalImpl { public: // allow_virtual_devices is captured at construction (from the OrtEnv config "allow_virtual_devices") - // rather than queried from the OrtEnv singleton inside GetSupportedDevices: internal EPs are - // registered while the OrtEnv creation mutex is already held on this thread, so querying the singleton - // there would self-deadlock. + // rather than queried from the OrtEnv singleton inside GetSupportedDevices: internal EPs are registered + // from Environment::Create, before OrtEnv publishes p_instance_, so there is no instance to find. explicit WebGpuEpFactory(bool allow_virtual_devices) : EpFactoryInternalImpl(kWebGpuExecutionProvider, "Microsoft", OrtDevice::VendorIds::MICROSOFT), allow_virtual_devices_{allow_virtual_devices} { diff --git a/onnxruntime/core/session/plugin_ep/ep_library.h b/onnxruntime/core/session/plugin_ep/ep_library.h index fed9eb072c704..d99b9ca008069 100644 --- a/onnxruntime/core/session/plugin_ep/ep_library.h +++ b/onnxruntime/core/session/plugin_ep/ep_library.h @@ -3,6 +3,8 @@ #pragma once +#include +#include #include #include "core/common/status.h" #include "core/framework/provider_options.h" @@ -10,7 +12,6 @@ #include "core/session/onnxruntime_c_api.h" namespace onnxruntime { - /// /// EpLibrary is the base class for implementing support for execution provider libraries that provide /// OrtEpFactory instances. @@ -18,15 +19,22 @@ namespace onnxruntime { class EpLibrary { public: EpLibrary() = default; + virtual ~EpLibrary() = default; virtual const char* RegistrationName() const = 0; virtual const std::filesystem::path* LibraryPath() const { return nullptr; } virtual Status Load() { return Status::OK(); } - virtual const std::vector& GetFactories() = 0; // valid after Load() virtual Status Unload() { return Status::OK(); } - virtual ~EpLibrary() = default; + std::ranges::view auto GetFactories() const { + return std::views::iota(size_t{0}, GetFactoryCount()) | + std::views::transform([this](size_t index) { return GetFactory(index); }); + } ORT_DISALLOW_COPY_AND_ASSIGNMENT(EpLibrary); + + private: + virtual size_t GetFactoryCount() const = 0; + virtual OrtEpFactory* GetFactory(size_t index) const = 0; }; } // namespace onnxruntime diff --git a/onnxruntime/core/session/plugin_ep/ep_library_internal.h b/onnxruntime/core/session/plugin_ep/ep_library_internal.h index 8d246e5539cbb..248f6bab8d511 100644 --- a/onnxruntime/core/session/plugin_ep/ep_library_internal.h +++ b/onnxruntime/core/session/plugin_ep/ep_library_internal.h @@ -21,17 +21,13 @@ namespace onnxruntime { class EpLibraryInternal : public EpLibrary { public: EpLibraryInternal(std::unique_ptr factory) - : factory_{std::move(factory)}, factory_ptrs_{factory_.get()} { + : factory_{std::move(factory)} { } const char* RegistrationName() const override { return factory_->GetName(); // same as EP name for internally registered libraries } - const std::vector& GetFactories() override { - return factory_ptrs_; - } - // there's only ever one currently EpFactoryInternal& GetInternalFactory() { return *factory_; @@ -44,6 +40,15 @@ class EpLibraryInternal : public EpLibrary { static std::vector> CreateInternalEps(bool allow_virtual_devices); private: + size_t GetFactoryCount() const override { + return 1; + } + + OrtEpFactory* GetFactory(size_t index) const override { + ORT_ENFORCE(index == 0); + return factory_.get(); + } + static std::unique_ptr CreateCpuEp(); #if defined(USE_COREML) && defined(__APPLE__) static std::unique_ptr CreateCoreMLEp(); @@ -56,7 +61,6 @@ class EpLibraryInternal : public EpLibrary { #endif std::unique_ptr factory_; // all internal EPs register a single factory currently - std::vector factory_ptrs_; // for convenience }; } // namespace onnxruntime diff --git a/onnxruntime/core/session/plugin_ep/ep_library_plugin.cc b/onnxruntime/core/session/plugin_ep/ep_library_plugin.cc index 2f2d504b5406b..7ad1662f07aad 100644 --- a/onnxruntime/core/session/plugin_ep/ep_library_plugin.cc +++ b/onnxruntime/core/session/plugin_ep/ep_library_plugin.cc @@ -6,6 +6,7 @@ #include "core/common/logging/logging.h" #include "core/framework/error_code_helper.h" #include "core/session/environment.h" +#include "core/session/plugin_ep/ep_library_plugin_utils.h" namespace onnxruntime { Status EpLibraryPlugin::Load() { @@ -27,19 +28,7 @@ Status EpLibraryPlugin::Load() { ORT_RETURN_IF_ERROR(Env::Default().GetSymbolFromLibrary(handle_, "ReleaseEpFactory", reinterpret_cast(&release_fn_))); - // allocate buffer for EP to add factories to. library can add up to 4 factories. - std::vector factories{4, nullptr}; - - size_t num_factories = 0; - ORT_RETURN_IF_ERROR(ToStatusAndRelease(create_fn_(registration_name_.c_str(), OrtGetApiBase(), - logging::LoggingManager::DefaultLogger().ToExternal(), - factories.data(), factories.size(), &num_factories))); - - for (size_t i = 0; i < num_factories; ++i) { - factories_.push_back(factories[i]); - } - - return Status::OK(); + return ep_library_plugin_utils::CreateFactories(create_fn_, release_fn_, registration_name_, factories_); }(); } } @@ -67,42 +56,8 @@ Status EpLibraryPlugin::Load() { Status EpLibraryPlugin::Unload() { std::lock_guard lock{mutex_}; - // Call ReleaseEpFactory for all factories and unload the library. - // Current implementation assumes any error is permanent so does not leave pieces around to re-attempt Unload. if (handle_) { - if (!factories_.empty()) { - ORT_TRY { - for (size_t idx = 0, end = factories_.size(); idx < end; ++idx) { - auto* factory = factories_[idx]; - if (factory == nullptr) { - continue; - } - - auto status = ToStatusAndRelease(release_fn_(factory)); - if (!status.IsOK()) { - // log it and treat it as released - LOGS_DEFAULT(ERROR) << "ReleaseEpFactory failed for: " << library_path_ << " with error: " - << status.ErrorMessage(); - } - - factories_[idx] = nullptr; // clear the pointer in case there's a failure before all are released - } - - factories_.clear(); - } - ORT_CATCH(const std::exception& ex) { - ORT_HANDLE_EXCEPTION([&]() { - LOGS_DEFAULT(ERROR) << "Failed releasing EP factories from " << library_path_ << ": " << ex.what(); - }); - } - } - - // TODO: Is there a better way? Is it worth worrying about? - if (!factories_.empty()) { - LOGS_DEFAULT(ERROR) << "Unloading " << library_path_ << ". " << factories_.size() - << " factories were not released due to errors. This may cause memory leaks. " - "Please check the error details in the log."; - } + factories_.clear(); ORT_RETURN_IF_ERROR(Env::Default().UnloadDynamicLibrary(handle_)); } diff --git a/onnxruntime/core/session/plugin_ep/ep_library_plugin.h b/onnxruntime/core/session/plugin_ep/ep_library_plugin.h index cce52b3d5d282..501c887c3f663 100644 --- a/onnxruntime/core/session/plugin_ep/ep_library_plugin.h +++ b/onnxruntime/core/session/plugin_ep/ep_library_plugin.h @@ -7,6 +7,7 @@ #include #include "core/session/plugin_ep/ep_library.h" +#include "core/session/plugin_ep/ep_library_plugin_utils.h" namespace onnxruntime { /// @@ -31,20 +32,24 @@ class EpLibraryPlugin : public EpLibrary { Status Load() override; - const std::vector& GetFactories() override { - return factories_; - } - Status Unload() override; ORT_DISALLOW_COPY_AND_ASSIGNMENT(EpLibraryPlugin); private: + size_t GetFactoryCount() const override { + return factories_.size(); + } + + OrtEpFactory* GetFactory(size_t index) const override { + return factories_[index].get(); + } + std::mutex mutex_; const std::string registration_name_; const std::filesystem::path library_path_; void* handle_{}; - std::vector factories_{}; + std::vector factories_{}; CreateEpApiFactoriesFn create_fn_{nullptr}; ReleaseEpApiFactoryFn release_fn_{nullptr}; }; diff --git a/onnxruntime/core/session/plugin_ep/ep_library_plugin_utils.cc b/onnxruntime/core/session/plugin_ep/ep_library_plugin_utils.cc new file mode 100644 index 0000000000000..f7e62649b1bad --- /dev/null +++ b/onnxruntime/core/session/plugin_ep/ep_library_plugin_utils.cc @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/session/plugin_ep/ep_library_plugin_utils.h" + +#include +#include + +#include "core/common/common.h" +#include "core/common/logging/logging.h" +#include "core/common/safeint.h" +#include "core/framework/error_code_helper.h" + +namespace onnxruntime { +namespace ep_library_plugin_utils { + +void OrtEpFactoryDeleter::operator()(OrtEpFactory* factory) const noexcept { + if (factory == nullptr) { + return; + } + + auto status = ToStatusAndRelease(release_fn(factory)); + if (!status.IsOK()) { + LOGS_DEFAULT(ERROR) << "ReleaseEpFactory failed with error: " << status.ErrorMessage(); + } +} + +Status CreateFactories(CreateEpApiFactoriesFn create_fn, ReleaseEpApiFactoryFn release_fn, + const std::string& registration_name, + std::vector& factories) { + constexpr size_t kMaxFactories = 4; + std::array raw_factories{}; + std::array new_factories{}; + + // Make adoption non-throwing before provider code transfers any ownership to us. + const auto adopt_factories = [&]() noexcept { + for (size_t i = 0; i < raw_factories.size(); ++i) { + if (raw_factories[i] != nullptr) { + new_factories[i] = OrtEpFactoryUniquePtr{raw_factories[i], OrtEpFactoryDeleter{release_fn}}; + raw_factories[i] = nullptr; + } + } + }; + + size_t num_factories = 0; + OrtStatus* ort_status = create_fn(registration_name.c_str(), OrtGetApiBase(), + logging::LoggingManager::DefaultLogger().ToExternal(), + raw_factories.data(), raw_factories.size(), &num_factories); + adopt_factories(); + + if (ort_status != nullptr) { + return ToStatusAndRelease(ort_status); + } + + ORT_RETURN_IF_NOT(num_factories <= new_factories.size(), + "CreateEpFactories returned ", num_factories, " factories but the supplied capacity was ", + new_factories.size(), "."); + + for (size_t i = 0; i < new_factories.size(); ++i) { + if ((i < num_factories) != (new_factories[i] != nullptr)) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, + "CreateEpFactories returned factory outputs inconsistent with num_factories."); + } + } + + factories.reserve(SafeInt(factories.size()) + num_factories); + factories.insert(factories.end(), + std::make_move_iterator(new_factories.begin()), + std::make_move_iterator(new_factories.begin() + num_factories)); + + return Status::OK(); +} + +} // namespace ep_library_plugin_utils +} // namespace onnxruntime diff --git a/onnxruntime/core/session/plugin_ep/ep_library_plugin_utils.h b/onnxruntime/core/session/plugin_ep/ep_library_plugin_utils.h new file mode 100644 index 0000000000000..01aecadcc7a8d --- /dev/null +++ b/onnxruntime/core/session/plugin_ep/ep_library_plugin_utils.h @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include + +#include "core/common/status.h" +#include "core/session/onnxruntime_c_api.h" + +namespace onnxruntime { +namespace ep_library_plugin_utils { + +// Shared implementation for EpLibrary types that obtain OrtEpFactory instances from a plugin EP's +// CreateEpFactories and ReleaseEpFactory entry points. The entry points are resolved at load time for a +// dynamically loaded library and at link time for a statically linked one, but the lifetime rules are the same either +// way. + +struct OrtEpFactoryDeleter { + ReleaseEpApiFactoryFn release_fn{}; + + void operator()(OrtEpFactory* factory) const noexcept; +}; + +using OrtEpFactoryUniquePtr = std::unique_ptr; + +// Calls the CreateEpFactories entry point and appends the factories it produces to `factories`. +// `factories` is left unmodified and all returned factories are released if creation or validation fails. +Status CreateFactories(CreateEpApiFactoriesFn create_fn, ReleaseEpApiFactoryFn release_fn, + const std::string& registration_name, + std::vector& factories); + +} // namespace ep_library_plugin_utils +} // namespace onnxruntime diff --git a/onnxruntime/core/session/plugin_ep/ep_library_provider_bridge.cc b/onnxruntime/core/session/plugin_ep/ep_library_provider_bridge.cc index da94a9f12ba9d..ace8b273213e2 100644 --- a/onnxruntime/core/session/plugin_ep/ep_library_provider_bridge.cc +++ b/onnxruntime/core/session/plugin_ep/ep_library_provider_bridge.cc @@ -28,11 +28,10 @@ Status EpLibraryProviderBridge::Load() { // we also need to update any returned OrtEpDevice instances to swap the wrapper EpFactoryInternal in so that we can // call Provider::CreateIExecutionProvider in EpFactoryInternal::CreateIExecutionProvider. - for (const auto& factory : ep_library_plugin_->GetFactories()) { + for (OrtEpFactory* factory : ep_library_plugin_->GetFactories()) { auto factory_impl = std::make_unique(*factory, *provider_library_, library_path_); auto internal_factory = std::make_unique(std::move(factory_impl)); - factory_ptrs_.push_back(internal_factory.get()); internal_factory_ptrs_.push_back(internal_factory.get()); factories_.push_back(std::move(internal_factory)); } @@ -44,7 +43,6 @@ Status EpLibraryProviderBridge::Unload() { std::lock_guard lock{mutex_}; internal_factory_ptrs_.clear(); - factory_ptrs_.clear(); factories_.clear(); // we loaded ep_library_plugin_ after provider_library_ in LoadPluginOrProviderBridge so do the reverse order here. diff --git a/onnxruntime/core/session/plugin_ep/ep_library_provider_bridge.h b/onnxruntime/core/session/plugin_ep/ep_library_provider_bridge.h index f3147794bc823..46eecfd657cc7 100644 --- a/onnxruntime/core/session/plugin_ep/ep_library_provider_bridge.h +++ b/onnxruntime/core/session/plugin_ep/ep_library_provider_bridge.h @@ -36,10 +36,6 @@ class EpLibraryProviderBridge : public EpLibrary { return &library_path_; } - const std::vector& GetFactories() override { - return factory_ptrs_; - } - // Provider bridge EPs are 'internal' as they can provide an IExecutionProvider instance directly. const std::vector& GetInternalFactories() { return internal_factory_ptrs_; @@ -51,6 +47,14 @@ class EpLibraryProviderBridge : public EpLibrary { ORT_DISALLOW_COPY_AND_ASSIGNMENT(EpLibraryProviderBridge); private: + size_t GetFactoryCount() const override { + return factories_.size(); + } + + OrtEpFactory* GetFactory(size_t index) const override { + return factories_[index].get(); + } + std::mutex mutex_; std::unique_ptr provider_library_; // provider bridge EP library @@ -63,7 +67,6 @@ class EpLibraryProviderBridge : public EpLibrary { std::filesystem::path library_path_; std::vector> factories_; - std::vector factory_ptrs_; // for convenience std::vector internal_factory_ptrs_; // for convenience }; } // namespace onnxruntime diff --git a/onnxruntime/core/session/plugin_ep/ep_library_static_plugin.cc b/onnxruntime/core/session/plugin_ep/ep_library_static_plugin.cc new file mode 100644 index 0000000000000..077d01abfe937 --- /dev/null +++ b/onnxruntime/core/session/plugin_ep/ep_library_static_plugin.cc @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/session/plugin_ep/ep_library_static_plugin.h" + +#include "core/common/common.h" +#include "core/session/plugin_ep/ep_library_plugin_utils.h" + +namespace onnxruntime { +Status EpLibraryStaticPlugin::Load() { + auto status = Status::OK(); + + ORT_TRY { + std::lock_guard lock{mutex_}; + if (factories_.empty()) { + status = ep_library_plugin_utils::CreateFactories(create_fn_, release_fn_, registration_name_, factories_); + } + } + ORT_CATCH(const std::exception& ex) { + ORT_HANDLE_EXCEPTION([&]() { + status = ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, + "Failed to create factories for statically linked execution " + "provider '", + registration_name_, "' with error: ", ex.what()); + }); + } + + return status; +} + +Status EpLibraryStaticPlugin::Unload() { + std::lock_guard lock{mutex_}; + + factories_.clear(); + + return Status::OK(); +} +} // namespace onnxruntime diff --git a/onnxruntime/core/session/plugin_ep/ep_library_static_plugin.h b/onnxruntime/core/session/plugin_ep/ep_library_static_plugin.h new file mode 100644 index 0000000000000..a974e8b88ccfd --- /dev/null +++ b/onnxruntime/core/session/plugin_ep/ep_library_static_plugin.h @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include + +#include "core/session/plugin_ep/ep_library.h" +#include "core/session/plugin_ep/ep_library_plugin_utils.h" + +namespace onnxruntime { +/// +/// EpLibraryStaticPlugin supports a plugin execution provider that is statically linked into the ORT binary. +/// +/// The provider implements the same CreateEpFactories and ReleaseEpFactory entry points that a dynamically loaded +/// plugin library exports, and uses the same public EP API, but the entry points are resolved at link time instead +/// of being looked up by name. There is no library to load or unload, so the provider's process-global state +/// outlives unregistration. +/// +class EpLibraryStaticPlugin : public EpLibrary { + public: + EpLibraryStaticPlugin(std::string registration_name, + CreateEpApiFactoriesFn create_fn, + ReleaseEpApiFactoryFn release_fn) + : registration_name_{std::move(registration_name)}, + create_fn_{create_fn}, + release_fn_{release_fn} { + } + + const char* RegistrationName() const override { + return registration_name_.c_str(); + } + + // LibraryPath() intentionally uses the base class implementation, which returns nullptr. There is no library + // file backing a statically linked provider. + + Status Load() override; + + Status Unload() override; + + ORT_DISALLOW_COPY_AND_ASSIGNMENT(EpLibraryStaticPlugin); + + private: + size_t GetFactoryCount() const override { + return factories_.size(); + } + + OrtEpFactory* GetFactory(size_t index) const override { + return factories_[index].get(); + } + + std::mutex mutex_; + const std::string registration_name_; + const CreateEpApiFactoriesFn create_fn_; + const ReleaseEpApiFactoryFn release_fn_; + std::vector factories_{}; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/session/plugin_ep/ep_plugin_provider_interfaces.cc b/onnxruntime/core/session/plugin_ep/ep_plugin_provider_interfaces.cc index f0e86ec79c46c..edce64e330ac2 100644 --- a/onnxruntime/core/session/plugin_ep/ep_plugin_provider_interfaces.cc +++ b/onnxruntime/core/session/plugin_ep/ep_plugin_provider_interfaces.cc @@ -25,6 +25,7 @@ #include "core/session/abi_logger.h" #include "core/session/abi_session_options_impl.h" #include "core/session/allocator_adapters.h" +#include "core/session/plugin_ep/ep_allocator_utils.h" #include "core/session/plugin_ep/ep_kernel_registration.h" #include "core/session/plugin_ep/ep_event_profiling.h" #include "core/session/ort_apis.h" @@ -861,16 +862,17 @@ std::unique_ptr PluginExecutionProvider::GetDataTran if (ep_factory_.CreateDataTransfer != nullptr) { OrtStatus* status = ep_factory_.CreateDataTransfer(&ep_factory_, &data_transfer_impl); + plugin_ep::OrtDataTransferImplUniquePtr owned_impl{data_transfer_impl}; if (status != nullptr) { ORT_THROW("Error creating data transfer: ", ToStatusAndRelease(status).ToString()); } - } - if (data_transfer_impl == nullptr) { - return {}; + if (owned_impl != nullptr) { + return std::make_unique(std::move(owned_impl)); + } } - return std::make_unique(*data_transfer_impl); + return {}; } std::vector PluginExecutionProvider::CreatePreferredAllocators() { @@ -879,53 +881,15 @@ std::vector PluginExecutionProvider::CreatePreferredAllocators() { const OrtKeyValuePairs* allocator_options = session_arena_options_ ? &*session_arena_options_ : nullptr; - for (const auto* memory_info : allocator_mem_infos_) { - OrtAllocator* ort_allocator_ptr = nullptr; - - if (!ort_ep_->CreateAllocator && !ep_factory_.CreateAllocator) { - ORT_THROW("The OrtEpDevice requires the EP library to implement an allocator, but none were found."); - } - - // prefer OrtEp function if available, otherwise fall back to using the OrtEpFactory implementation. - OrtStatus* ort_status = ort_ep_->CreateAllocator - ? ort_ep_->CreateAllocator(ort_ep_.get(), memory_info, &ort_allocator_ptr) - : ep_factory_.CreateAllocator(&ep_factory_, memory_info, allocator_options, - &ort_allocator_ptr); - - // throw or log? start with throw - if (ort_status != nullptr) { - ORT_THROW("Error creating allocator: ", ToStatusAndRelease(ort_status).ToString()); - } - - // A successful callback may return nullptr to request ORT's default CPU allocator. - if (ort_allocator_ptr == nullptr) { - continue; - } - - if (ort_allocator_ptr->Info(ort_allocator_ptr)->alloc_type == OrtAllocatorType::OrtArenaAllocator) { - ORT_THROW( - "OrtEpFactory returned an allocator with OrtAllocatorType of OrtArenaAllocator. " - "This type is reserved for ONNX Runtime internal usage only, as any arena usage by the " - "EP library should be opaque to ORT"); - } + auto* ep_factory = &ep_factory_; - auto* ep_factory = &ep_factory_; - auto ort_allocator = OrtAllocatorUniquePtr( - ort_allocator_ptr, - [ep_factory](OrtAllocator* allocator) { - ep_factory->ReleaseAllocator(ep_factory, allocator); - }); - - // Use the arena wrapper when the allocator supports Shrink(), matching - // the logic in Environment::CreateSharedAllocatorImpl. This ensures - // per-session plugin arenas are visible to ShrinkMemoryArenas. + for (const auto* memory_info : allocator_mem_infos_) { AllocatorPtr alloc_ptr; - if (ort_allocator->version >= 25 && ort_allocator->Shrink != nullptr) { - alloc_ptr = std::make_shared(std::move(ort_allocator)); - } else { - alloc_ptr = std::make_shared(std::move(ort_allocator)); + ORT_THROW_IF_ERROR(ep_allocator_utils::CreateAndWrapEpAllocator(ort_ep_.get(), *ep_factory, *memory_info, + allocator_options, alloc_ptr)); + if (alloc_ptr != nullptr) { + allocators.push_back(std::move(alloc_ptr)); } - allocators.push_back(std::move(alloc_ptr)); } return allocators; diff --git a/onnxruntime/core/session/plugin_ep/ep_static_plugins.cc b/onnxruntime/core/session/plugin_ep/ep_static_plugins.cc new file mode 100644 index 0000000000000..b09b6f72a40a8 --- /dev/null +++ b/onnxruntime/core/session/plugin_ep/ep_static_plugins.cc @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/session/plugin_ep/ep_static_plugins.h" + +#include "core/graph/constants.h" +#include "core/session/plugin_ep/ep_library_static_plugin.h" + +// Entry points of the plugin execution providers that are statically linked into this build. +// +// A statically linked provider compiles its CreateEpFactories and ReleaseEpFactory entry points with a provider +// specific `WebGpu_` style prefix (see ORT_PLUGIN_EP_STATICALLY_LINKED in the provider's api.cc) so that several +// providers can be linked into one binary without their entry points colliding. Declare them here rather than +// including a provider header, so that ORT core does not take a compile time dependency on provider internals. +#if defined(USE_WEBGPU) && defined(ORT_WEBGPU_STATIC_PLUGIN) +extern "C" { +OrtStatus* WebGpu_CreateEpFactories(const char* registration_name, const OrtApiBase* ort_api_base, + const OrtLogger* default_logger, OrtEpFactory** factories, + size_t max_factories, size_t* num_factories) noexcept; +OrtStatus* WebGpu_ReleaseEpFactory(OrtEpFactory* factory) noexcept; +} +#endif + +namespace onnxruntime { +std::vector> CreateStaticPluginEpLibraries() { + std::vector> ep_libraries; + +#if defined(USE_WEBGPU) && defined(ORT_WEBGPU_STATIC_PLUGIN) + // Register under the EP name, matching what an internal EP does. ORT core picks the registration name because + // there is no application call site to supply one. + ep_libraries.push_back(std::make_unique(kWebGpuExecutionProvider, + &WebGpu_CreateEpFactories, + &WebGpu_ReleaseEpFactory)); +#endif + + return ep_libraries; +} +} // namespace onnxruntime diff --git a/onnxruntime/core/session/plugin_ep/ep_static_plugins.h b/onnxruntime/core/session/plugin_ep/ep_static_plugins.h new file mode 100644 index 0000000000000..7107bedcb3861 --- /dev/null +++ b/onnxruntime/core/session/plugin_ep/ep_static_plugins.h @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include + +#include "core/session/plugin_ep/ep_library.h" + +namespace onnxruntime { +/// +/// Creates an EpLibrary for each plugin execution provider that is statically linked into this build. +/// The returned libraries have not been loaded yet. +/// +std::vector> CreateStaticPluginEpLibraries(); +} // namespace onnxruntime diff --git a/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.cc b/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.cc index 6c140adcbecef..7d18fc290903e 100644 --- a/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.cc +++ b/onnxruntime/test/autoep/library/example_plugin_ep/ep_factory.cc @@ -11,6 +11,7 @@ #include "ep_arena.h" #include "ep_data_transfer.h" #include "ep_stream_support.h" +#include "ep_test_hooks.h" #include "core/session/onnxruntime_ep_device_ep_metadata_keys.h" #include "core/session/onnxruntime_session_options_config_keys.h" @@ -441,6 +442,10 @@ OrtStatus* ORT_API_CALL ExampleEpFactory::CreateDataTransferImpl(OrtEpFactory* t auto& factory = *static_cast(this_ptr); *data_transfer = factory.data_transfer_impl_.get(); + if (ShouldFailCreateDataTransfer()) { + return factory.ort_api.CreateStatus(ORT_FAIL, "injected data transfer creation failure"); + } + return nullptr; } diff --git a/onnxruntime/test/autoep/library/example_plugin_ep/ep_test_hooks.cc b/onnxruntime/test/autoep/library/example_plugin_ep/ep_test_hooks.cc index f09ed22869d10..675085f0c4dd6 100644 --- a/onnxruntime/test/autoep/library/example_plugin_ep/ep_test_hooks.cc +++ b/onnxruntime/test/autoep/library/example_plugin_ep/ep_test_hooks.cc @@ -8,6 +8,7 @@ std::atomic g_sync_count{0}; std::atomic g_preallocated_output_query_result{-1}; std::atomic g_preallocated_output_bad_index_rejected{-1}; +std::atomic g_create_data_transfer_should_fail{false}; extern "C" void ExampleEpTestHooks_ResetSyncCount() { g_sync_count.store(0); } extern "C" uint64_t ExampleEpTestHooks_GetSyncCount() { return g_sync_count.load(); } @@ -21,9 +22,15 @@ extern "C" int ExampleEpTestHooks_GetPreallocatedOutputQueryResult() { extern "C" int ExampleEpTestHooks_GetPreallocatedOutputBadIndexRejected() { return g_preallocated_output_bad_index_rejected.load(); } +extern "C" void ExampleEpTestHooks_SetCreateDataTransferFailure(int enabled) { + g_create_data_transfer_should_fail.store(enabled != 0); +} void RecordPreallocatedOutputQueryResult(int has_preallocated_output) { g_preallocated_output_query_result.store(has_preallocated_output); } void RecordPreallocatedOutputBadIndexRejected(int rejected) { g_preallocated_output_bad_index_rejected.store(rejected); } +bool ShouldFailCreateDataTransfer() { + return g_create_data_transfer_should_fail.load(); +} diff --git a/onnxruntime/test/autoep/library/example_plugin_ep/ep_test_hooks.h b/onnxruntime/test/autoep/library/example_plugin_ep/ep_test_hooks.h index 47168f0f5fce0..9a2ba88cde5cb 100644 --- a/onnxruntime/test/autoep/library/example_plugin_ep/ep_test_hooks.h +++ b/onnxruntime/test/autoep/library/example_plugin_ep/ep_test_hooks.h @@ -23,8 +23,10 @@ EXPORT_SYMBOL uint64_t ExampleEpTestHooks_GetSyncCount(); EXPORT_SYMBOL void ExampleEpTestHooks_ResetPreallocatedOutputQuery(); EXPORT_SYMBOL int ExampleEpTestHooks_GetPreallocatedOutputQueryResult(); EXPORT_SYMBOL int ExampleEpTestHooks_GetPreallocatedOutputBadIndexRejected(); +EXPORT_SYMBOL void ExampleEpTestHooks_SetCreateDataTransferFailure(int enabled); } // Internal to the library; not exported. void RecordPreallocatedOutputQueryResult(int has_preallocated_output); void RecordPreallocatedOutputBadIndexRejected(int rejected); +bool ShouldFailCreateDataTransfer(); diff --git a/onnxruntime/test/autoep/library/example_plugin_ep/example_plugin_ep_library.def b/onnxruntime/test/autoep/library/example_plugin_ep/example_plugin_ep_library.def index f3f87edbdb27b..59e909c86d84f 100644 --- a/onnxruntime/test/autoep/library/example_plugin_ep/example_plugin_ep_library.def +++ b/onnxruntime/test/autoep/library/example_plugin_ep/example_plugin_ep_library.def @@ -2,4 +2,3 @@ LIBRARY "example_plugin_ep.dll" EXPORTS CreateEpFactories @1 ReleaseEpFactory @2 - \ No newline at end of file diff --git a/onnxruntime/test/autoep/library/example_plugin_ep/example_plugin_ep_library.lds b/onnxruntime/test/autoep/library/example_plugin_ep/example_plugin_ep_library.lds index 5117e6f124b2d..bed5d9956be7b 100644 --- a/onnxruntime/test/autoep/library/example_plugin_ep/example_plugin_ep_library.lds +++ b/onnxruntime/test/autoep/library/example_plugin_ep/example_plugin_ep_library.lds @@ -7,6 +7,7 @@ VERS_1.0.0 { ExampleEpTestHooks_ResetPreallocatedOutputQuery; ExampleEpTestHooks_GetPreallocatedOutputQueryResult; ExampleEpTestHooks_GetPreallocatedOutputBadIndexRejected; + ExampleEpTestHooks_SetCreateDataTransferFailure; local: *; }; diff --git a/onnxruntime/test/autoep/test_autoep_utils.cc b/onnxruntime/test/autoep/test_autoep_utils.cc index 5e372b3dc58ee..d031f4504c349 100644 --- a/onnxruntime/test/autoep/test_autoep_utils.cc +++ b/onnxruntime/test/autoep/test_autoep_utils.cc @@ -92,6 +92,8 @@ void Utils::LoadExampleEpHooks(const ExamplePluginInfo& ep_info, hooks.get_preallocated_output_bad_index_rejected = reinterpret_cast( GetProcAddress(lib, "ExampleEpTestHooks_GetPreallocatedOutputBadIndexRejected")); + hooks.set_create_data_transfer_failure = reinterpret_cast( + GetProcAddress(lib, "ExampleEpTestHooks_SetCreateDataTransferFailure")); #else void* lib = dlopen(ep_info.library_path.c_str(), RTLD_LAZY | RTLD_LOCAL); ASSERT_NE(lib, nullptr); @@ -108,6 +110,8 @@ void Utils::LoadExampleEpHooks(const ExamplePluginInfo& ep_info, hooks.get_preallocated_output_bad_index_rejected = reinterpret_cast( dlsym(lib, "ExampleEpTestHooks_GetPreallocatedOutputBadIndexRejected")); + hooks.set_create_data_transfer_failure = reinterpret_cast( + dlsym(lib, "ExampleEpTestHooks_SetCreateDataTransferFailure")); #endif example_ep_hooks = LoadExampleEpHooksPtr( diff --git a/onnxruntime/test/autoep/test_autoep_utils.h b/onnxruntime/test/autoep/test_autoep_utils.h index c57f68a1404cf..059f133a31166 100644 --- a/onnxruntime/test/autoep/test_autoep_utils.h +++ b/onnxruntime/test/autoep/test_autoep_utils.h @@ -43,12 +43,14 @@ struct Utils { using ResetPreallocatedOutputQueryFn = void (*)(); using GetPreallocatedOutputQueryResultFn = int (*)(); using GetPreallocatedOutputBadIndexRejectedFn = int (*)(); + using SetCreateDataTransferFailureFn = void (*)(int enabled); ResetSyncCountFn reset_sync_count{}; GetSyncCountFn get_sync_count{}; ResetPreallocatedOutputQueryFn reset_preallocated_output_query{}; GetPreallocatedOutputQueryResultFn get_preallocated_output_query_result{}; GetPreallocatedOutputBadIndexRejectedFn get_preallocated_output_bad_index_rejected{}; + SetCreateDataTransferFailureFn set_create_data_transfer_failure{}; }; using LoadExampleEpHooksPtr = std::unique_ptr>; diff --git a/onnxruntime/test/autoep/test_registration.cc b/onnxruntime/test/autoep/test_registration.cc index 99539ff73ee94..8eff2e55e4d5b 100644 --- a/onnxruntime/test/autoep/test_registration.cc +++ b/onnxruntime/test/autoep/test_registration.cc @@ -2,6 +2,7 @@ // Licensed under the MIT License. #include +#include #include #include @@ -114,6 +115,37 @@ TEST(OrtEpLibrary, LoadUnloadPluginLibraryCxxApi) { ort_env->UnregisterExecutionProviderLibrary(registration_name.c_str()); } +TEST(OrtEpLibrary, FailedRegistrationLeavesEnvironmentUnchanged) { + const std::filesystem::path& library_path = Utils::example_ep_info.library_path; + const std::string& registration_name = Utils::example_ep_info.registration_name; + const size_t initial_device_count = ort_env->GetEpDevices().size(); + + Utils::LoadExampleEpHooksPtr hooks; + ASSERT_NO_FATAL_FAILURE(Utils::LoadExampleEpHooks(Utils::example_ep_info, hooks)); + ASSERT_NE(hooks->set_create_data_transfer_failure, nullptr); + + { + // Fail registration with a data transfer creation failure. + hooks->set_create_data_transfer_failure(1); + auto reset_failure = gsl::finally([&] { hooks->set_create_data_transfer_failure(0); }); + + Ort::Status status{Ort::GetApi().RegisterExecutionProviderLibrary( + *ort_env, registration_name.c_str(), library_path.c_str())}; + ASSERT_FALSE(status.IsOK()); + + // The failed library must not contribute any EP devices to the environment. + EXPECT_EQ(ort_env->GetEpDevices().size(), initial_device_count); + } + + // The same registration name must remain available after the failed attempt. + ort_env->RegisterExecutionProviderLibrary(registration_name.c_str(), library_path.c_str()); + EXPECT_EQ(ort_env->GetEpDevices().size(), initial_device_count + 1); + ort_env->UnregisterExecutionProviderLibrary(registration_name.c_str()); + + // The successful registration must still support a normal unregister lifecycle. + EXPECT_EQ(ort_env->GetEpDevices().size(), initial_device_count); +} + // Test loading example_plugin_ep_virt_gpu and its associated OrtEpDevice/OrtHardwareDevice. // This EP creates a new OrtHardwareDevice instance that represents a virtual GPU and gives to ORT. TEST(OrtEpLibrary, LoadUnloadPluginVirtGpuLibraryCxxApi) { diff --git a/onnxruntime/test/framework/data_transfer_manager_test.cc b/onnxruntime/test/framework/data_transfer_manager_test.cc index 90da354cbc449..f50ba7c00656c 100644 --- a/onnxruntime/test/framework/data_transfer_manager_test.cc +++ b/onnxruntime/test/framework/data_transfer_manager_test.cc @@ -19,7 +19,12 @@ TEST(DataTransferManagerTest, PluginCopiesForwardStreams) { struct TestDataTransfer final : OrtDataTransferImpl { TestDataTransfer() : OrtDataTransferImpl{} { ort_version_supported = ORT_API_VERSION; - Release = [](OrtDataTransferImpl*) noexcept {}; + Release = [](OrtDataTransferImpl* impl) noexcept { + delete static_cast(impl); + }; + CanCopy = [](const OrtDataTransferImpl*, const OrtMemoryDevice*, const OrtMemoryDevice*) noexcept { + return true; + }; CopyTensors = [](OrtDataTransferImpl* impl, const OrtValue**, OrtValue**, OrtSyncStream** streams, size_t num_tensors) noexcept -> OrtStatus* { auto& self = *static_cast(impl); @@ -33,9 +38,11 @@ TEST(DataTransferManagerTest, PluginCopiesForwardStreams) { size_t copied_tensors = 0; OrtSyncStream* last_stream = nullptr; - } impl; + }; - plugin_ep::DataTransfer data_transfer{impl}; + auto impl = std::make_unique(); + auto* impl_ptr = impl.get(); + plugin_ep::DataTransfer data_transfer{plugin_ep::OrtDataTransferImplUniquePtr{impl.release()}}; auto allocator = TestCPUExecutionProvider()->CreatePreferredAllocators()[0]; Tensor source{DataTypeImpl::GetType(), TensorShape{4}, allocator}; Tensor destination{DataTypeImpl::GetType(), TensorShape{4}, allocator}; @@ -43,16 +50,16 @@ TEST(DataTransferManagerTest, PluginCopiesForwardStreams) { Stream stream{nullptr, device}; ASSERT_STATUS_OK(data_transfer.CopyTensorAsync(source, destination, stream)); - EXPECT_EQ(impl.copied_tensors, 1U); - EXPECT_EQ(impl.last_stream, reinterpret_cast(&stream)); + EXPECT_EQ(impl_ptr->copied_tensors, 1U); + EXPECT_EQ(impl_ptr->last_stream, reinterpret_cast(&stream)); ASSERT_STATUS_OK(data_transfer.CopyTensor(source, destination)); - EXPECT_EQ(impl.copied_tensors, 2U); - EXPECT_EQ(impl.last_stream, nullptr); + EXPECT_EQ(impl_ptr->copied_tensors, 2U); + EXPECT_EQ(impl_ptr->last_stream, nullptr); ASSERT_STATUS_OK(data_transfer.CopyTensors({{source, destination, &stream}})); - EXPECT_EQ(impl.copied_tensors, 3U); - EXPECT_EQ(impl.last_stream, reinterpret_cast(&stream)); + EXPECT_EQ(impl_ptr->copied_tensors, 3U); + EXPECT_EQ(impl_ptr->last_stream, reinterpret_cast(&stream)); } // DataTransferManager::CopyTensors should validate sizes match before calling the IDataTransfer implementation diff --git a/onnxruntime/test/framework/dynamic_plugin_ep_test.cc b/onnxruntime/test/framework/dynamic_plugin_ep_test.cc index 3f6d1ec1654b2..12a83f41c8629 100644 --- a/onnxruntime/test/framework/dynamic_plugin_ep_test.cc +++ b/onnxruntime/test/framework/dynamic_plugin_ep_test.cc @@ -72,10 +72,27 @@ TEST(DynamicPluginEpInfraTest, ParseInitializationConfigDefaultsUnsetOptionalFie EXPECT_TRUE(config.tests_to_skip.empty()); } +TEST(DynamicPluginEpInfraTest, ParseInitializationConfigAllowsMissingEpLibraryPath) { + // `ep_library_path` may be omitted when the EP library is already registered, e.g. a statically linked plugin EP. + constexpr std::string_view kConfigJson = R"json( +{ + "ep_library_registration_name": "WebGpuExecutionProvider", + "selected_ep_name": "WebGpuExecutionProvider" +} +)json"; + + dynamic_plugin_ep_test_infra::InitializationConfig config{}; + ASSERT_STATUS_OK(dynamic_plugin_ep_test_infra::ParseInitializationConfig(kConfigJson, config)); + + EXPECT_EQ(config.ep_library_registration_name, "WebGpuExecutionProvider"); + EXPECT_TRUE(config.ep_library_path.empty()); + EXPECT_EQ(config.selected_ep_name, "WebGpuExecutionProvider"); +} + TEST(DynamicPluginEpInfraTest, ParseInitializationConfigRejectsMissingRequiredFields) { constexpr std::string_view kConfigJson = R"json( { - "ep_library_registration_name": "CUDAExecutionProvider" + "ep_library_path": "/tmp/libonnxruntime_providers_cuda.so" } )json"; diff --git a/onnxruntime/test/framework/ep_library_plugin_utils_test.cc b/onnxruntime/test/framework/ep_library_plugin_utils_test.cc new file mode 100644 index 0000000000000..fa013d927c96a --- /dev/null +++ b/onnxruntime/test/framework/ep_library_plugin_utils_test.cc @@ -0,0 +1,110 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/session/plugin_ep/ep_library_plugin_utils.h" + +#include "gtest/gtest.h" + +#include "core/session/onnxruntime_cxx_api.h" + +namespace onnxruntime::test { + +namespace { + +size_t released_factory_count = 0; + +OrtEpFactory* CreateTestFactory() { + return new OrtEpFactory{}; +} + +OrtStatus* ReleaseTestFactory(OrtEpFactory* factory) { + ++released_factory_count; + delete factory; + return nullptr; +} + +OrtStatus* CreateOneFactory(const char*, const OrtApiBase*, const OrtLogger*, + OrtEpFactory** factories, size_t, size_t* num_factories) { + factories[0] = CreateTestFactory(); + *num_factories = 1; + return nullptr; +} + +OrtStatus* CreateOneFactoryThenFail(const char*, const OrtApiBase*, const OrtLogger*, + OrtEpFactory** factories, size_t, size_t* num_factories) { + factories[0] = CreateTestFactory(); + *num_factories = 1; + return Ort::GetApi().CreateStatus(ORT_FAIL, "injected factory creation failure"); +} + +OrtStatus* ReturnOversizedFactoryCount(const char*, const OrtApiBase*, const OrtLogger*, + OrtEpFactory** factories, size_t max_factories, size_t* num_factories) { + factories[0] = CreateTestFactory(); + *num_factories = max_factories + 1; + return nullptr; +} + +OrtStatus* ReturnInconsistentFactoryOutputs(const char*, const OrtApiBase*, const OrtLogger*, + OrtEpFactory** factories, size_t, size_t* num_factories) { + factories[1] = CreateTestFactory(); + *num_factories = 1; + return nullptr; +} + +} // namespace + +class EpLibraryPluginUtilsTest : public ::testing::Test { + protected: + void SetUp() override { + released_factory_count = 0; + } +}; + +TEST_F(EpLibraryPluginUtilsTest, CreateFactoriesAdoptsFactoriesOnSuccess) { + std::vector factories; + + ASSERT_TRUE(ep_library_plugin_utils::CreateFactories( + CreateOneFactory, ReleaseTestFactory, "test plugin", factories) + .IsOK()); + ASSERT_EQ(factories.size(), 1u); + EXPECT_EQ(released_factory_count, 0u); + + factories.clear(); + EXPECT_TRUE(factories.empty()); + EXPECT_EQ(released_factory_count, 1u); +} + +TEST_F(EpLibraryPluginUtilsTest, CreateFactoriesReleasesFactoryWhenCreationFails) { + std::vector factories; + + const auto status = ep_library_plugin_utils::CreateFactories( + CreateOneFactoryThenFail, ReleaseTestFactory, "test plugin", factories); + + EXPECT_FALSE(status.IsOK()); + EXPECT_TRUE(factories.empty()); + EXPECT_EQ(released_factory_count, 1u); +} + +TEST_F(EpLibraryPluginUtilsTest, CreateFactoriesRejectsOversizedFactoryCount) { + std::vector factories; + + const auto status = ep_library_plugin_utils::CreateFactories( + ReturnOversizedFactoryCount, ReleaseTestFactory, "test plugin", factories); + + EXPECT_FALSE(status.IsOK()); + EXPECT_TRUE(factories.empty()); + EXPECT_EQ(released_factory_count, 1u); +} + +TEST_F(EpLibraryPluginUtilsTest, CreateFactoriesRejectsInconsistentFactoryOutputs) { + std::vector factories; + + const auto status = ep_library_plugin_utils::CreateFactories( + ReturnInconsistentFactoryOutputs, ReleaseTestFactory, "test plugin", factories); + + EXPECT_FALSE(status.IsOK()); + EXPECT_TRUE(factories.empty()); + EXPECT_EQ(released_factory_count, 1u); +} + +} // namespace onnxruntime::test diff --git a/onnxruntime/test/framework/ep_plugin_provider_test.cc b/onnxruntime/test/framework/ep_plugin_provider_test.cc index 4820f4e5c8898..8677ffe822712 100644 --- a/onnxruntime/test/framework/ep_plugin_provider_test.cc +++ b/onnxruntime/test/framework/ep_plugin_provider_test.cc @@ -18,6 +18,7 @@ #include "core/framework/config_options.h" #include "core/framework/kernel_def_builder.h" #include "core/framework/op_kernel.h" +#include "core/framework/plugin_data_transfer.h" #include "core/framework/resource_accountant.h" #include "core/graph/constants.h" #include "core/graph/graph_viewer.h" @@ -26,6 +27,7 @@ #include "core/session/abi_devices.h" #include "core/session/onnxruntime_cxx_api.h" #include "core/session/onnxruntime_session_options_config_keys.h" +#include "core/session/plugin_ep/ep_allocator_utils.h" #include "test/util/include/api_asserts.h" #include "test/util/include/asserts.h" #include "test/util/include/test_environment.h" @@ -149,14 +151,15 @@ struct MakeTestOrtEpResult { // `setup` runs on the raw TestOrtEp before PluginExecutionProvider is constructed -- // callbacks consulted at construction time (e.g., GetDefaultMemoryDevice seeding // default_device_) must be configured here. -MakeTestOrtEpResult MakeTestOrtEp(std::vector ep_devices = {}, +MakeTestOrtEpResult MakeTestOrtEp(OrtEpFactory& ep_factory, + std::vector ep_devices, std::function setup = nullptr) { // Default OrtHardwareDevice and OrtEpDevice used if the caller does not explicitly provide ep_devices. static std::unique_ptr ort_hw_device = MakeTestOrtHardwareDevice(OrtHardwareDeviceType_CPU); static std::unique_ptr ort_ep_device = MakeTestOrtEpDevice(ort_hw_device.get()); auto ort_ep_raw = std::make_unique().release(); - auto ort_ep = UniqueOrtEp(ort_ep_raw, OrtEpDeleter{g_test_ort_ep_factory}); + auto ort_ep = UniqueOrtEp(ort_ep_raw, OrtEpDeleter{ep_factory}); if (setup) { setup(*ort_ep_raw); } @@ -169,7 +172,7 @@ MakeTestOrtEpResult MakeTestOrtEp(std::vector ep_devices = { auto& logging_manager = DefaultLoggingManager(); auto ep = std::make_unique(std::move(ort_ep), *static_cast(ort_session_options), - g_test_ort_ep_factory, + ep_factory, ep_devices, /*kernel_registry*/ nullptr, logging_manager.DefaultLogger()); @@ -178,6 +181,11 @@ MakeTestOrtEpResult MakeTestOrtEp(std::vector ep_devices = { return result; } +MakeTestOrtEpResult MakeTestOrtEp(std::vector ep_devices = {}, + std::function setup = nullptr) { + return MakeTestOrtEp(g_test_ort_ep_factory, std::move(ep_devices), std::move(setup)); +} + using LookUpKernelFunc = std::function; class MockKernelLookup : public IExecutionProvider::IKernelLookup { @@ -1405,6 +1413,7 @@ static FakeArenaOrtAllocator MakeFakeArenaAllocator(OrtMemoryInfo* mem_info, boo // Namespace-level storage so C function pointers can access the fake allocator. static OrtAllocator* g_fake_allocator_for_test = nullptr; +static int g_fake_allocator_release_count = 0; static OrtStatus* ORT_API_CALL FakeCreateAllocator(OrtEp*, const OrtMemoryInfo*, OrtAllocator** out) noexcept { @@ -1412,10 +1421,32 @@ static OrtStatus* ORT_API_CALL FakeCreateAllocator(OrtEp*, const OrtMemoryInfo*, return nullptr; } +static OrtStatus* ORT_API_CALL FakeCreateAllocatorWithError(OrtEp*, const OrtMemoryInfo*, + OrtAllocator** out) noexcept { + *out = g_fake_allocator_for_test; + return Ort::GetApi().CreateStatus(ORT_FAIL, "injected allocator creation failure"); +} + +static OrtStatus* ORT_API_CALL FakeCreateNullAllocator(OrtEp*, const OrtMemoryInfo*, + OrtAllocator** out) noexcept { + *out = nullptr; + return nullptr; +} + +static OrtStatus* ORT_API_CALL UnexpectedFactoryCreateAllocator(OrtEpFactory*, const OrtMemoryInfo*, + const OrtKeyValuePairs*, OrtAllocator** out) noexcept { + *out = nullptr; + return Ort::GetApi().CreateStatus(ORT_FAIL, "Unexpected call to OrtEpFactory::CreateAllocator"); +} + static void ORT_API_CALL FakeReleaseAllocator(OrtEpFactory*, OrtAllocator*) noexcept { // No-op: tests own the fake allocator lifetime. } +static void ORT_API_CALL FakeReleaseAllocatorCounting(OrtEpFactory*, OrtAllocator*) noexcept { + ++g_fake_allocator_release_count; +} + } // namespace TEST(PluginExecutionProviderTest, CreatePreferredAllocators_ShrinkCapableAllocatorExposedAsArena) { @@ -1436,6 +1467,7 @@ TEST(PluginExecutionProviderTest, CreatePreferredAllocators_ShrinkCapableAllocat g_fake_allocator_for_test = fake_alloc_ptr; ort_ep->CreateAllocator = FakeCreateAllocator; + test_plugin_ep::g_test_ort_ep_factory.CreateAllocator = UnexpectedFactoryCreateAllocator; test_plugin_ep::g_test_ort_ep_factory.ReleaseAllocator = FakeReleaseAllocator; auto allocators = ep->CreatePreferredAllocators(); @@ -1468,6 +1500,7 @@ TEST(PluginExecutionProviderTest, CreatePreferredAllocators_NonShrinkAllocatorNo g_fake_allocator_for_test = fake_alloc_ptr; ort_ep->CreateAllocator = FakeCreateAllocator; + test_plugin_ep::g_test_ort_ep_factory.CreateAllocator = UnexpectedFactoryCreateAllocator; test_plugin_ep::g_test_ort_ep_factory.ReleaseAllocator = FakeReleaseAllocator; auto allocators = ep->CreatePreferredAllocators(); @@ -1478,6 +1511,147 @@ TEST(PluginExecutionProviderTest, CreatePreferredAllocators_NonShrinkAllocatorNo << "Non-Shrink allocator must not be exposed as IArena"; } +#if !defined(ORT_NO_EXCEPTIONS) +TEST(PluginExecutionProviderTest, CreatePreferredAllocators_ReleasesReturnedAllocatorOnError) { + auto ort_device = test_plugin_ep::MakeTestOrtDevice(OrtDevice::GPU, OrtDevice::MemType::DEFAULT); + auto ort_memory_info = std::make_unique("FakeGPU", OrtAllocatorType::OrtDeviceAllocator, + ort_device, OrtMemTypeDefault); + auto fake_allocator = MakeFakeArenaAllocator(ort_memory_info.get()); + auto ort_hw_device = test_plugin_ep::MakeTestOrtHardwareDevice(OrtHardwareDeviceType_GPU); + auto ort_ep_device = test_plugin_ep::MakeTestOrtEpDevice(ort_hw_device.get(), ort_memory_info.get()); + std::vector ep_devices{ort_ep_device.get()}; + auto [ep, ort_ep] = test_plugin_ep::MakeTestOrtEp(ep_devices); + + g_fake_allocator_for_test = &fake_allocator; + g_fake_allocator_release_count = 0; + ort_ep->CreateAllocator = FakeCreateAllocatorWithError; + test_plugin_ep::g_test_ort_ep_factory.CreateAllocator = UnexpectedFactoryCreateAllocator; + test_plugin_ep::g_test_ort_ep_factory.ReleaseAllocator = FakeReleaseAllocatorCounting; + + try { + static_cast(ep->CreatePreferredAllocators()); + FAIL() << "Expected allocator creation to fail"; + } catch (const OnnxRuntimeException& ex) { + EXPECT_THAT(ex.what(), ::testing::HasSubstr("injected allocator creation failure")); + } + EXPECT_EQ(g_fake_allocator_release_count, 1); +} +#endif // !defined(ORT_NO_EXCEPTIONS) + +TEST(PluginExecutionProviderTest, CreatePreferredAllocators_AllowsNullAllocator) { + auto ort_device = test_plugin_ep::MakeTestOrtDevice(OrtDevice::GPU, OrtDevice::MemType::DEFAULT); + auto ort_memory_info = std::make_unique("FakeGPU", OrtAllocatorType::OrtDeviceAllocator, + ort_device, OrtMemTypeDefault); + auto ort_hw_device = test_plugin_ep::MakeTestOrtHardwareDevice(OrtHardwareDeviceType_GPU); + auto ort_ep_device = test_plugin_ep::MakeTestOrtEpDevice(ort_hw_device.get(), ort_memory_info.get()); + std::vector ep_devices{ort_ep_device.get()}; + auto [ep, ort_ep] = test_plugin_ep::MakeTestOrtEp(ep_devices); + + g_fake_allocator_release_count = 0; + ort_ep->CreateAllocator = FakeCreateNullAllocator; + test_plugin_ep::g_test_ort_ep_factory.CreateAllocator = UnexpectedFactoryCreateAllocator; + test_plugin_ep::g_test_ort_ep_factory.ReleaseAllocator = FakeReleaseAllocatorCounting; + + EXPECT_TRUE(ep->CreatePreferredAllocators().empty()); + EXPECT_EQ(g_fake_allocator_release_count, 0); +} + +TEST(PluginExecutionProviderTest, CreatePreferredAllocators_AllowsNullFactoryCreateAllocator) { + test_plugin_ep::TestOrtEpFactory ep_factory; + auto ort_device = test_plugin_ep::MakeTestOrtDevice(OrtDevice::GPU, OrtDevice::MemType::DEFAULT); + auto ort_memory_info = std::make_unique("FakeGPU", OrtAllocatorType::OrtDeviceAllocator, + ort_device, OrtMemTypeDefault); + auto fake_allocator = MakeFakeArenaAllocator(ort_memory_info.get()); + auto ort_hw_device = test_plugin_ep::MakeTestOrtHardwareDevice(OrtHardwareDeviceType_GPU); + auto ort_ep_device = + test_plugin_ep::MakeTestOrtEpDevice(ort_hw_device.get(), ep_factory, ort_memory_info.get(), nullptr); + std::vector ep_devices{ort_ep_device.get()}; + auto [ep, ort_ep] = test_plugin_ep::MakeTestOrtEp(ep_factory, ep_devices); + + g_fake_allocator_for_test = &fake_allocator; + ort_ep->CreateAllocator = FakeCreateAllocator; + ep_factory.CreateAllocator = nullptr; + ep_factory.ReleaseAllocator = FakeReleaseAllocator; + + auto allocators = ep->CreatePreferredAllocators(); + ASSERT_EQ(allocators.size(), 1u); +} + +TEST(PluginExecutionProviderTest, CreatePreferredAllocators_AllowsNoCreateAllocatorCallbacks) { + test_plugin_ep::TestOrtEpFactory ep_factory; + auto ort_device = test_plugin_ep::MakeTestOrtDevice(OrtDevice::GPU, OrtDevice::MemType::DEFAULT); + auto ort_memory_info = std::make_unique("FakeGPU", OrtAllocatorType::OrtDeviceAllocator, + ort_device, OrtMemTypeDefault); + auto ort_hw_device = test_plugin_ep::MakeTestOrtHardwareDevice(OrtHardwareDeviceType_GPU); + auto ort_ep_device = + test_plugin_ep::MakeTestOrtEpDevice(ort_hw_device.get(), ep_factory, ort_memory_info.get(), nullptr); + std::vector ep_devices{ort_ep_device.get()}; + auto [ep, ort_ep] = test_plugin_ep::MakeTestOrtEp(ep_factory, ep_devices); + + ort_ep->CreateAllocator = nullptr; + ep_factory.CreateAllocator = nullptr; + ep_factory.ReleaseAllocator = nullptr; + + EXPECT_TRUE(ep->CreatePreferredAllocators().empty()); +} + +TEST(PluginExecutionProviderTest, CreateAndWrapEpAllocator_AllowsNoCreateAllocatorCallbacks) { + test_plugin_ep::TestOrtEpFactory ep_factory; + auto ort_device = test_plugin_ep::MakeTestOrtDevice(OrtDevice::GPU, OrtDevice::MemType::DEFAULT); + OrtMemoryInfo ort_memory_info{"FakeGPU", OrtAllocatorType::OrtDeviceAllocator, ort_device, OrtMemTypeDefault}; + + AllocatorPtr allocator; + OrtAllocator* raw_allocator = nullptr; + ASSERT_STATUS_OK(ep_allocator_utils::CreateAndWrapEpAllocator( + ep_factory, ort_memory_info, nullptr, allocator, &raw_allocator)); + EXPECT_EQ(allocator, nullptr); + EXPECT_EQ(raw_allocator, nullptr); +} + +TEST(PluginExecutionProviderTest, CreateAndWrapEpAllocator_RejectsMissingReleaseAllocator) { + test_plugin_ep::TestOrtEpFactory ep_factory; + auto ort_device = test_plugin_ep::MakeTestOrtDevice(OrtDevice::GPU, OrtDevice::MemType::DEFAULT); + OrtMemoryInfo ort_memory_info{"FakeGPU", OrtAllocatorType::OrtDeviceAllocator, ort_device, OrtMemTypeDefault}; + test_plugin_ep::TestOrtEp ort_ep; + + ort_ep.CreateAllocator = FakeCreateAllocator; + ep_factory.CreateAllocator = nullptr; + ep_factory.ReleaseAllocator = nullptr; + + AllocatorPtr allocator; + const auto status = ep_allocator_utils::CreateAndWrapEpAllocator( + &ort_ep, ep_factory, ort_memory_info, nullptr, allocator); + ASSERT_FALSE(status.IsOK()); + EXPECT_THAT(status.ErrorMessage(), ::testing::HasSubstr("OrtEpFactory must implement ReleaseAllocator")); +} + +#if !defined(ORT_NO_EXCEPTIONS) +TEST(PluginExecutionProviderTest, CreatePreferredAllocators_RejectsAllocatorWithNullMemoryInfo) { + auto ort_device = test_plugin_ep::MakeTestOrtDevice(OrtDevice::GPU, OrtDevice::MemType::DEFAULT); + auto ort_memory_info = std::make_unique("FakeGPU", OrtAllocatorType::OrtDeviceAllocator, + ort_device, OrtMemTypeDefault); + auto fake_allocator = MakeFakeArenaAllocator(nullptr); + auto ort_hw_device = test_plugin_ep::MakeTestOrtHardwareDevice(OrtHardwareDeviceType_GPU); + auto ort_ep_device = test_plugin_ep::MakeTestOrtEpDevice(ort_hw_device.get(), ort_memory_info.get()); + std::vector ep_devices{ort_ep_device.get()}; + auto [ep, ort_ep] = test_plugin_ep::MakeTestOrtEp(ep_devices); + + g_fake_allocator_for_test = &fake_allocator; + g_fake_allocator_release_count = 0; + ort_ep->CreateAllocator = FakeCreateAllocator; + test_plugin_ep::g_test_ort_ep_factory.CreateAllocator = UnexpectedFactoryCreateAllocator; + test_plugin_ep::g_test_ort_ep_factory.ReleaseAllocator = FakeReleaseAllocatorCounting; + + try { + static_cast(ep->CreatePreferredAllocators()); + FAIL() << "Expected allocator validation to fail"; + } catch (const OnnxRuntimeException& ex) { + EXPECT_THAT(ex.what(), ::testing::HasSubstr("OrtEp returned an allocator with null memory info.")); + } + EXPECT_EQ(g_fake_allocator_release_count, 1); +} +#endif // !defined(ORT_NO_EXCEPTIONS) + TEST(PluginExecutionProviderTest, IsGraphCaptureEnabled) { auto [ep, ort_ep] = test_plugin_ep::MakeTestOrtEp(); @@ -1862,4 +2036,66 @@ TEST(PluginExecutionProviderTest, OnSessionInitializationEnd_OldVersionFallback) ASSERT_STATUS_OK(ep->OnSessionInitializationEnd()); } +TEST(PluginDataTransferTest, OwnsAndReleasesImplementation) { + struct TestDataTransferImpl : OrtDataTransferImpl { + explicit TestDataTransferImpl(size_t& release_count) : release_count{release_count} { + Release = [](OrtDataTransferImpl* this_ptr) noexcept { + auto* impl = static_cast(this_ptr); + ++impl->release_count; + delete impl; + }; + CanCopy = [](const OrtDataTransferImpl*, const OrtMemoryDevice*, const OrtMemoryDevice*) noexcept { + return false; + }; + CopyTensors = [](OrtDataTransferImpl*, const OrtValue**, OrtValue**, OrtSyncStream**, size_t) noexcept { + return static_cast(nullptr); + }; + } + + size_t& release_count; + }; + + size_t release_count = 0; + { + plugin_ep::DataTransfer data_transfer{ + plugin_ep::OrtDataTransferImplUniquePtr{new TestDataTransferImpl{release_count}}}; + } + + EXPECT_EQ(release_count, 1u); +} + +#if !defined(ORT_NO_EXCEPTIONS) +TEST(PluginDataTransferTest, RejectsImplementationWithMissingRequiredFunction) { + auto make_data_transfer_impl = []() { + OrtDataTransferImpl impl{}; + impl.Release = [](OrtDataTransferImpl*) noexcept {}; + impl.CanCopy = [](const OrtDataTransferImpl*, const OrtMemoryDevice*, const OrtMemoryDevice*) noexcept { + return false; + }; + impl.CopyTensors = [](OrtDataTransferImpl*, const OrtValue**, OrtValue**, OrtSyncStream**, size_t) noexcept { + return static_cast(nullptr); + }; + return impl; + }; + + { + auto impl = make_data_transfer_impl(); + impl.Release = nullptr; + EXPECT_THROW(plugin_ep::DataTransfer{plugin_ep::OrtDataTransferImplUniquePtr{&impl}}, OnnxRuntimeException); + } + + { + auto impl = make_data_transfer_impl(); + impl.CanCopy = nullptr; + EXPECT_THROW(plugin_ep::DataTransfer{plugin_ep::OrtDataTransferImplUniquePtr{&impl}}, OnnxRuntimeException); + } + + { + auto impl = make_data_transfer_impl(); + impl.CopyTensors = nullptr; + EXPECT_THROW(plugin_ep::DataTransfer{plugin_ep::OrtDataTransferImplUniquePtr{&impl}}, OnnxRuntimeException); + } +} +#endif + } // namespace onnxruntime::test diff --git a/onnxruntime/test/framework/inference_session_test.cc b/onnxruntime/test/framework/inference_session_test.cc index c47511954e051..a4d7d2b407562 100644 --- a/onnxruntime/test/framework/inference_session_test.cc +++ b/onnxruntime/test/framework/inference_session_test.cc @@ -835,8 +835,10 @@ TEST(InferenceSessionTests, WebGpuCompileOnlyUsesNoOpAllocator) { // End-to-end via the public V2 API in the *plugin* WebGPU build: select the virtual WebGPU OrtEpDevice and run a // compile-only session. // -// Relies on test_main.cc registering the WebGPU plugin EP under a ".virtual" name, whose suffix auto-enables the env -// config "allow_virtual_devices" so the factory surfaces a virtual GPU OrtEpDevice. +// Relies on test_main.cc arranging for the factory to surface a virtual GPU OrtEpDevice: in the shared-library +// plugin build by registering the EP library under a ".virtual" name, whose suffix auto-enables the env config +// "allow_virtual_devices"; in the static plugin build by setting that env config entry directly at environment +// creation (ORT core, not the test, registers the statically linked EP, so there is no registration name to suffix). // // It exercises the accepted (device-free) path even on a host that has a real GPU: device-free is driven by // session.compile_only, not by which device is selected, so no Dawn device is created. @@ -855,9 +857,10 @@ TEST(InferenceSessionTests, WebGpuVirtualDeviceCompileOnlyEndToEnd) { break; } } - // A virtual device must be present in this build (test_main.cc's ".virtual" registration enables it). + // A virtual device must be present in this build (see the comment above this test for how test_main.cc enables it). ASSERT_FALSE(selected.empty()) - << "Expected a virtual WebGPU EP device from test_main.cc's .virtual registration, but none was surfaced."; + << "Expected a virtual WebGPU EP device from test_main.cc's virtual device configuration, " + "but none was surfaced."; Ort::SessionOptions session_options; // session-level compile_only (NOT an EP option) -> drives the device-free context and stop-before-finalize. @@ -874,7 +877,7 @@ TEST(InferenceSessionTests, WebGpuVirtualDeviceCompileOnlyEndToEnd) { // internal factory): selecting the virtual WebGPU device for a normal (non-compile-only) session must be rejected // up front by the *adapter* factory's CreateEp with ORT_INVALID_ARGUMENT, rather than proceeding into Dawn to fail // obscurely with no real GPU behind the virtual device. Exercises the adapter factory's copy of the enforcement -// through the public V2 API. Depends on the same test_main.cc ".virtual" registration as +// through the public V2 API. Depends on the same test_main.cc virtual device configuration as // WebGpuVirtualDeviceCompileOnlyEndToEnd above (that's what surfaces the virtual device to select). TEST(InferenceSessionTests, WebGpuVirtualDeviceRejectedWithoutCompileOnly) { std::vector selected; @@ -889,10 +892,11 @@ TEST(InferenceSessionTests, WebGpuVirtualDeviceRejectedWithoutCompileOnly) { break; } } - // See WebGpuVirtualDeviceCompileOnlyEndToEnd: a virtual device must be present from test_main.cc's .virtual - // registration in this build, so fail (not skip) if none was surfaced. + // See WebGpuVirtualDeviceCompileOnlyEndToEnd: a virtual device must be present in this build, so fail (not skip) + // if none was surfaced. ASSERT_FALSE(selected.empty()) - << "Expected a virtual WebGPU EP device from test_main.cc's .virtual registration, but none was surfaced."; + << "Expected a virtual WebGPU EP device from test_main.cc's virtual device configuration, " + "but none was surfaced."; // Deliberately NOT setting session.compile_only -> a runnable session on a virtual device, which must be rejected. Ort::SessionOptions session_options; diff --git a/onnxruntime/test/platform/device_discovery_test.cc b/onnxruntime/test/platform/device_discovery_test.cc index 13b515697ad8c..c3a66dadaa568 100644 --- a/onnxruntime/test/platform/device_discovery_test.cc +++ b/onnxruntime/test/platform/device_discovery_test.cc @@ -33,8 +33,13 @@ TEST(DeviceDiscoveryTest, GpuDevicesHaveValidProperties) { // GPU detection should not crash. If GPUs are present, validate their properties. for (const auto& gpu_device : gpu_devices) { +#if !defined(__wasm__) EXPECT_NE(gpu_device.vendor_id, 0u); - // Note: device_id may be 0 on some platforms (e.g., Apple Silicon) where it is not populated. +#else + // WASM GPU device discovery is limited so vendor ID is unknown. + EXPECT_EQ(gpu_device.vendor_id, 0u); +#endif + // Note: gpu_device.device_id may be 0 on some platforms (e.g., Apple Silicon) where it is not populated. } } diff --git a/onnxruntime/test/providers/webgpu/plugin/test_webgpu_allocators.cc b/onnxruntime/test/providers/webgpu/plugin/test_webgpu_allocators.cc index c460bc39a5887..35dbd0d7a7025 100644 --- a/onnxruntime/test/providers/webgpu/plugin/test_webgpu_allocators.cc +++ b/onnxruntime/test/providers/webgpu/plugin/test_webgpu_allocators.cc @@ -19,14 +19,22 @@ #include "core/graph/onnx_protobuf.h" #include "core/platform/env_var_utils.h" #include "core/session/onnxruntime_cxx_api.h" +#include "core/session/onnxruntime_ep_device_ep_metadata_keys.h" #include "core/session/onnxruntime_session_options_config_keys.h" #include "test/providers/webgpu/plugin/webgpu_plugin_test_utils.h" #include "test/util/include/api_asserts.h" #if defined(ORT_UNIT_TEST_HAS_WEBGPU_PLUGIN_EP) && !defined(USE_EXTERNAL_DAWN) && !defined(BUILD_DAWN_SHARED_LIBRARY) +#if defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wstrict-aliasing" +#endif #include "dawn/dawn_proc.h" #include "dawn/native/DawnNative.h" +#if defined(__GNUC__) +#pragma GCC diagnostic pop +#endif #endif extern std::unique_ptr ort_env; @@ -145,8 +153,17 @@ class WebGpuPluginSharedAllocatorTest : public ::testing::Test { protected: void SetUp() override { registration_.emplace(*ort_env, "webgpu_allocator_test"); + + const auto is_virtual_device = [](const Ort::ConstEpDevice& device) { + const auto metadata = device.Device().Metadata().GetKeyValuePairs(); + const auto is_virtual = metadata.find(kOrtHardwareDevice_MetadataKey_IsVirtual); + return is_virtual != metadata.end() && is_virtual->second == "1"; + }; + for (const auto& device : registration_->GetEpDevices()) { - ep_devices_.push_back(device); + if (!is_virtual_device(device)) { + ep_devices_.push_back(device); + } } ASSERT_FALSE(ep_devices_.empty()); } diff --git a/onnxruntime/test/providers/webgpu/plugin/test_webgpu_concurrency.cc b/onnxruntime/test/providers/webgpu/plugin/test_webgpu_concurrency.cc index 136879b5a6208..adbe254075770 100644 --- a/onnxruntime/test/providers/webgpu/plugin/test_webgpu_concurrency.cc +++ b/onnxruntime/test/providers/webgpu/plugin/test_webgpu_concurrency.cc @@ -57,13 +57,6 @@ class FirstError { std::string message_; }; -void ThrowOnError(OrtStatus* status_ptr) { - Ort::Status status{status_ptr}; - if (!status.IsOK()) { - throw std::runtime_error(status.GetErrorMessage()); - } -} - constexpr int kThreads = 4; constexpr int kIterations = 20; constexpr size_t kElements = 6; @@ -116,15 +109,15 @@ void CopyTensorRoundTrip(Allocator& allocator, float value, bool verify_zero_ini if (verify_zero_initialization) { output_data.fill(-1.0f); - ThrowOnError(ort_env->CopyTensor(gpu_tensor, cpu_output, nullptr)); + Ort::ThrowOnError(ort_env->CopyTensor(gpu_tensor, cpu_output, nullptr)); if (output_data != std::array{}) { throw std::runtime_error("Streamless allocation was not zero initialized"); } } - ThrowOnError(ort_env->CopyTensor(cpu_input, gpu_tensor, nullptr)); - ThrowOnError(ort_env->CopyTensor(gpu_tensor, gpu_copy, nullptr)); - ThrowOnError(ort_env->CopyTensor(gpu_copy, cpu_output, nullptr)); + Ort::ThrowOnError(ort_env->CopyTensor(cpu_input, gpu_tensor, nullptr)); + Ort::ThrowOnError(ort_env->CopyTensor(gpu_tensor, gpu_copy, nullptr)); + Ort::ThrowOnError(ort_env->CopyTensor(gpu_copy, cpu_output, nullptr)); if (output_data != input_data) { throw std::runtime_error("CopyTensor round trip returned incorrect data"); } @@ -176,7 +169,7 @@ class PluginEpWebGpuConcurrency : public ::testing::Test { Ort::Allocator allocator(session, Device().GetMemoryInfo(OrtDeviceMemoryType_DEFAULT)); auto gpu_input = Ort::Value::CreateTensor(allocator, kShape.data(), kShape.size()); auto gpu_output = Ort::Value::CreateTensor(allocator, kShape.data(), kShape.size()); - ThrowOnError(ort_env->CopyTensor(cpu_input, gpu_input, nullptr)); + Ort::ThrowOnError(ort_env->CopyTensor(cpu_input, gpu_input, nullptr)); Ort::IoBinding io_binding(session); io_binding.BindInput("X", gpu_input); @@ -185,7 +178,7 @@ class PluginEpWebGpuConcurrency : public ::testing::Test { session.Run(Ort::RunOptions{nullptr}, io_binding); io_binding.SynchronizeOutputs(); - ThrowOnError(ort_env->CopyTensor(gpu_output, cpu_output, nullptr)); + Ort::ThrowOnError(ort_env->CopyTensor(gpu_output, cpu_output, nullptr)); VerifyOutput(output_data, value); } @@ -222,7 +215,7 @@ class PluginEpWebGpuConcurrency : public ::testing::Test { std::array output_data{}; auto cpu_output = Ort::Value::CreateTensor( cpu_memory_info, output_data.data(), output_data.size(), kShape.data(), kShape.size()); - ThrowOnError(ort_env->CopyTensor(gpu_output, cpu_output, nullptr)); + Ort::ThrowOnError(ort_env->CopyTensor(gpu_output, cpu_output, nullptr)); VerifyOutput(output_data, value); } @@ -234,7 +227,7 @@ class PluginEpWebGpuConcurrency : public ::testing::Test { cpu_memory_info, input_data.data(), input_data.size(), kShape.data(), kShape.size()); Ort::Allocator allocator(session, Device().GetMemoryInfo(OrtDeviceMemoryType_DEFAULT)); auto gpu_input = Ort::Value::CreateTensor(allocator, kShape.data(), kShape.size()); - ThrowOnError(ort_env->CopyTensor(cpu_input, gpu_input, nullptr)); + Ort::ThrowOnError(ort_env->CopyTensor(cpu_input, gpu_input, nullptr)); const std::array input_names{"X"}; const std::array output_names{"Y"}; @@ -331,7 +324,7 @@ class PluginEpWebGpuConcurrency : public ::testing::Test { cpu_node_data[index] = static_cast(100 + iteration * kElements + index); } if (use_gpu_feed) { - ThrowOnError(ort_env->CopyTensor(cpu_node_source, feeds[reverse_feeds ? 0 : 1], nullptr)); + Ort::ThrowOnError(ort_env->CopyTensor(cpu_node_source, feeds[reverse_feeds ? 0 : 1], nullptr)); } if (bind_cpu_output_to_gpu) { for (size_t index = 0; index < feeds.size(); ++index) { @@ -351,7 +344,7 @@ class PluginEpWebGpuConcurrency : public ::testing::Test { Ort::Value::CreateTensor(allocator, kShape.data(), kShape.size()), Ort::Value::CreateTensor(allocator, kShape.data(), kShape.size())}; for (auto& cached_tensor : cached_tensors) { - ThrowOnError(ort_env->CopyTensor(feeds[reverse_feeds ? 1 : 0], cached_tensor, nullptr)); + Ort::ThrowOnError(ort_env->CopyTensor(feeds[reverse_feeds ? 1 : 0], cached_tensor, nullptr)); } } @@ -367,7 +360,7 @@ class PluginEpWebGpuConcurrency : public ::testing::Test { ASSERT_EQ(outputs[1].GetTensorMemoryInfo().GetDeviceType(), OrtMemoryInfoDeviceType_GPU); auto cpu_output = Ort::Value::CreateTensor( cpu_memory, cpu_output_data.data(), cpu_output_data.size(), kShape.data(), kShape.size()); - ThrowOnError(ort_env->CopyTensor(outputs[1], cpu_output, nullptr)); + Ort::ThrowOnError(ort_env->CopyTensor(outputs[1], cpu_output, nullptr)); outputs[1] = std::move(cpu_output); binding.ClearBoundOutputs(); } else { @@ -429,7 +422,9 @@ class PluginEpWebGpuConcurrency : public ::testing::Test { options.SetGraphOptimizationLevel(GraphOptimizationLevel::ORT_DISABLE_ALL); options.AddConfigEntry(kOrtSessionOptionsDisableCPUEPFallback, "1"); options.EnableProfiling(ORT_TSTR("webgpu_copy_regression")); - options.AppendExecutionProvider_V2(*ort_env, {Device()}, {{"maxNumPendingDispatches", "4096"}}); + Ort::KeyValuePairs ep_options; + ep_options.Add("maxNumPendingDispatches", "4096"); + options.AppendExecutionProvider_V2(*ort_env, {Device()}, ep_options); const auto bytes = model.SerializeAsString(); Ort::Session session(*ort_env, bytes.data(), bytes.size(), options); for (const auto& device : session.GetEpDeviceForOutputs()) { @@ -446,7 +441,7 @@ class PluginEpWebGpuConcurrency : public ::testing::Test { auto gpu_producer = Ort::Value::CreateTensor(allocator, shape.data(), shape.size()); auto gpu_copy = Ort::Value::CreateTensor(allocator, shape.data(), shape.size()); ASSERT_NE(gpu_producer.GetTensorMutableData(), gpu_copy.GetTensorMutableData()); - ThrowOnError(ort_env->CopyTensor(input, gpu_input, nullptr)); + Ort::ThrowOnError(ort_env->CopyTensor(input, gpu_input, nullptr)); Ort::IoBinding binding(session); binding.BindInput("X", gpu_input); // Preallocated, distinct graph outputs force the no-op kernel's actual copy branch, @@ -461,7 +456,7 @@ class PluginEpWebGpuConcurrency : public ::testing::Test { std::array output_data{}; auto output = Ort::Value::CreateTensor(cpu_memory, output_data.data(), output_data.size(), shape.data(), shape.size()); - ThrowOnError(ort_env->CopyTensor(gpu_copy, output, nullptr)); + Ort::ThrowOnError(ort_env->CopyTensor(gpu_copy, output, nullptr)); for (size_t i = 0; i < input_data.size(); ++i) { EXPECT_EQ(output_data[i], -input_data[i]); } @@ -933,7 +928,7 @@ TEST_F(PluginEpWebGpuConcurrency, DedicatedSessionAllocatorFeedsConcurrentSessio auto input = Ort::Value::CreateTensor( cpu_memory, input_data.data(), input_data.size(), kShape.data(), kShape.size()); auto gpu_input = Ort::Value::CreateTensor(allocator, kShape.data(), kShape.size()); - ThrowOnError(ort_env->CopyTensor(input, gpu_input, nullptr)); + Ort::ThrowOnError(ort_env->CopyTensor(input, gpu_input, nullptr)); auto outputs = sessions[thread_id]->Run(Ort::RunOptions{nullptr}, input_names.data(), &gpu_input, 1, output_names.data(), output_names.size()); std::array output_data{}; @@ -958,8 +953,8 @@ TEST_F(PluginEpWebGpuConcurrency, SharedGpuCopyIsSubmittedBeforeSessionRun) { kShape.data(), kShape.size()); auto source = Ort::Value::CreateTensor(allocator, kShape.data(), kShape.size()); auto destination = Ort::Value::CreateTensor(allocator, kShape.data(), kShape.size()); - ThrowOnError(ort_env->CopyTensor(input, source, nullptr)); - ThrowOnError(ort_env->CopyTensor(source, destination, nullptr)); + Ort::ThrowOnError(ort_env->CopyTensor(input, source, nullptr)); + Ort::ThrowOnError(ort_env->CopyTensor(source, destination, nullptr)); auto result = session->Run(Ort::RunOptions{nullptr}, input_names.data(), &destination, 1, output_names.data(), output_names.size()); std::array output{}; diff --git a/onnxruntime/test/unittest_main/test_main.cc b/onnxruntime/test/unittest_main/test_main.cc index 2693f0a93f050..4940d15df4f8b 100644 --- a/onnxruntime/test/unittest_main/test_main.cc +++ b/onnxruntime/test/unittest_main/test_main.cc @@ -42,6 +42,11 @@ #include "test/util/include/skipping_test_listener.h" #endif // defined(TEST_MAIN_ENABLE_DYNAMIC_PLUGIN_EP_USAGE) +#if defined(TEST_MAIN_ENABLE_DYNAMIC_PLUGIN_EP_USAGE) && defined(ORT_UNIT_TEST_HAS_WEBGPU_STATIC_PLUGIN_EP) +#define TEST_MAIN_ENABLE_WEBGPU_STATIC_PLUGIN_EP_USAGE +#include "core/session/onnxruntime_env_config_keys.h" +#endif + std::unique_ptr ort_env; // define environment variable name constants here @@ -82,7 +87,27 @@ extern "C" void ortenv_setup() { log_level = static_cast(*log_level_override); } +#if defined(TEST_MAIN_ENABLE_WEBGPU_STATIC_PLUGIN_EP_USAGE) + // The WebGPU plugin EP is statically linked into the ORT binary and is registered by ORT core while the + // environment is created, so "allow_virtual_devices" must be set here rather than being derived from a + // ".virtual" EP library registration name (there is no library registration call to name). This lets the + // WebGPU factory surface a virtual GPU device, which gives dynamic_plugin_ep_infra::Initialize() below a + // selectable WebGPU device so the binary doesn't exit at startup when GetEpDevices() would otherwise be + // empty (no real GPU). It does not make non-compile-only WebGPU tests runnable without a device. + Ort::KeyValuePairs env_config_entries{}; + env_config_entries.Add(kOrtEnvAllowVirtualDevices, "1"); + + OrtEnvCreationOptions env_creation_options{}; + env_creation_options.version = ORT_API_VERSION; + env_creation_options.logging_severity_level = static_cast(log_level); + env_creation_options.log_id = "Default"; + env_creation_options.threading_options = &tpo; + env_creation_options.config_entries = env_config_entries.GetConst(); + + ort_env.reset(new Ort::Env(&env_creation_options)); +#else ort_env.reset(new Ort::Env(&tpo, log_level, "Default")); +#endif #if defined(TEST_MAIN_ENABLE_DYNAMIC_PLUGIN_EP_USAGE) { @@ -142,6 +167,22 @@ extern "C" void ortenv_setup() { } #endif +#if defined(TEST_MAIN_ENABLE_WEBGPU_STATIC_PLUGIN_EP_USAGE) + if (!dynamic_plugin_ep_config_json.has_value()) { + // No "ep_library_path": ORT core already registered the statically linked WebGPU plugin EP when the + // environment above was created. + dynamic_plugin_ep_config_json.emplace( + "{\n" + " \"ep_library_registration_name\": \"WebGpuExecutionProvider\",\n" + // Enable int64 so the generic (EP-agnostic) int64 operator tests exercise the WebGPU + // int64 kernels on the plugin path too (mirrors DefaultWebGpuExecutionProvider on the + // internal EP path). The key is the un-prefixed provider option name for plugin EPs. + " \"default_ep_options\": { \"enableInt64\": \"1\" },\n" + " \"selected_ep_name\": \"WebGpuExecutionProvider\"\n" + "}"); + } +#endif + if (dynamic_plugin_ep_config_json.has_value()) { std::cout << "Initializing dynamic plugin EP infrastructure with configuration:\n" << *dynamic_plugin_ep_config_json << "\n"; diff --git a/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.cc b/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.cc index 3e4913e44fb86..4fb57635d22b0 100644 --- a/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.cc +++ b/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.cc @@ -84,9 +84,11 @@ Status ParseInitializationConfig(std::string_view json_str, InitializationConfig // required keys parsed_json.at("ep_library_registration_name").get_to(config.ep_library_registration_name); - parsed_json.at("ep_library_path").get_to(config.ep_library_path); // optional keys + // `ep_library_path` is not needed when the EP library is already registered, e.g. a plugin EP that is + // statically linked into the ORT binary and registered by ORT core at environment creation. + config.ep_library_path = parsed_json.value("ep_library_path", {}); config.default_ep_options = parsed_json.value("default_ep_options", {}); config.selected_ep_name = parsed_json.value("selected_ep_name", {}); config.selected_ep_device_indices = @@ -116,8 +118,13 @@ Status ParseInitializationConfig(std::string_view json_str, InitializationConfig Status Initialize(Ort::Env& env, InitializationConfig config) { ORT_RETURN_IF(IsInitialized(), "Already initialized."); - auto ep_library_registration_handle = RegisterPluginEpLibrary(env, config.ep_library_registration_name, - ToPathString(config.ep_library_path)); + // An empty `ep_library_path` means the EP library is already registered and this infrastructure should not + // register or unregister it. That is the case for a plugin EP that is statically linked into the ORT binary. + auto ep_library_registration_handle = + config.ep_library_path.empty() + ? PluginEpLibraryRegistrationHandle{nullptr, [](void*) {}} + : RegisterPluginEpLibrary(env, config.ep_library_registration_name, + ToPathString(config.ep_library_path)); ORT_RETURN_IF(config.selected_ep_device_indices.empty() == config.selected_ep_name.empty(), "Exactly one of selected_ep_device_indices or selected_ep_name should be specified."); diff --git a/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h b/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h index c3543962b80ef..6ece30801b316 100644 --- a/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h +++ b/onnxruntime/test/unittest_util/test_dynamic_plugin_ep.h @@ -40,6 +40,8 @@ inline constexpr std::string_view kWebGpuExecutionProviderPluginName{"WebGpuExec // Configuration for initializing the dynamic plugin EP infrastructure. struct InitializationConfig { std::string ep_library_registration_name{}; + // Path to the EP library to register. May be empty if the EP does not need to be registered by the test + // infrastructure, e.g. a plugin EP that is statically linked into the ORT binary and registered by ORT core. std::string ep_library_path{}; // Note: Exactly one of `selected_ep_name` or `selected_ep_device_indices` should be set. @@ -59,6 +61,8 @@ struct InitializationConfig { // Parses `InitializationConfig` from JSON. // The configuration JSON object should have keys and values that match the `InitializationConfig` fields. +// `ep_library_path` may be omitted for a plugin EP that is already registered (e.g. statically linked into the ORT +// binary and registered by ORT core). // E.g.: // { // "ep_library_registration_name": "example_plugin_ep", diff --git a/onnxruntime/wasm/api.cc b/onnxruntime/wasm/api.cc index 3717674bdb854..67d13c884c62e 100644 --- a/onnxruntime/wasm/api.cc +++ b/onnxruntime/wasm/api.cc @@ -16,6 +16,7 @@ WGPUDevice GetDevice(int); } // namespace onnxruntime #endif +#include #include #include #include @@ -180,6 +181,24 @@ int OrtAppendExecutionProvider(ort_session_options_handle_t session_options, return CHECK_STATUS(SessionOptionsAppendExecutionProvider, session_options, name, provider_options_keys, provider_options_values, num_keys); } +int OrtGetEpDevices(const ort_ep_device_handle_t** ep_devices, size_t* num_ep_devices) { + return CHECK_STATUS(GetEpDevices, g_env, ep_devices, num_ep_devices); +} + +const char* OrtEpDevice_EpName(ort_ep_device_handle_t ep_device) { + return Ort::GetApi().EpDevice_EpName(ep_device); +} + +int OrtAppendExecutionProviderV2(ort_session_options_handle_t session_options, + ort_ep_device_handle_t* ep_devices, + size_t num_ep_devices, + const char* const* provider_options_keys, + const char* const* provider_options_values, + size_t num_keys) { + return CHECK_STATUS(SessionOptionsAppendExecutionProvider_V2, session_options, g_env, + ep_devices, num_ep_devices, provider_options_keys, provider_options_values, num_keys); +} + int OrtAddFreeDimensionOverride(ort_session_options_handle_t session_options, const char* dim_param_name, int dim_value) { diff --git a/onnxruntime/wasm/api.h b/onnxruntime/wasm/api.h index b10e0576213a7..b18fcf2e337b6 100644 --- a/onnxruntime/wasm/api.h +++ b/onnxruntime/wasm/api.h @@ -31,6 +31,9 @@ using ort_run_options_handle_t = OrtRunOptions*; struct OrtValue; using ort_tensor_handle_t = OrtValue*; +struct OrtEpDevice; +using ort_ep_device_handle_t = const OrtEpDevice*; + #ifdef ENABLE_TRAINING_APIS struct OrtTrainingSession; using ort_training_session_handle_t = OrtTrainingSession*; @@ -94,6 +97,44 @@ int EMSCRIPTEN_KEEPALIVE OrtAppendExecutionProvider(ort_session_options_handle_t const char* const* provider_options_values, size_t num_keys); +/** + * get the execution provider devices registered with the environment. + * + * The array is owned by the environment and remains valid for its lifetime. The caller must not free it. + * + * @param ep_devices receives a pointer to the array of ort_ep_device_handle_t + * @param num_ep_devices receives the number of entries in the array + * @returns ORT error code. If not zero, call OrtGetLastError() to get detailed error message. + */ +int EMSCRIPTEN_KEEPALIVE OrtGetEpDevices(const ort_ep_device_handle_t** ep_devices, size_t* num_ep_devices); + +/** + * get the execution provider name that an execution provider device is registered under. + * + * More than one device may share an EP name, e.g. when an EP reports one device per GPU. + * + * @returns the name, owned by the environment. It remains valid for the lifetime of the environment. + */ +const char* EMSCRIPTEN_KEEPALIVE OrtEpDevice_EpName(ort_ep_device_handle_t ep_device); + +/** + * append an execution provider for a session, selecting it by execution provider device. + * + * Unlike OrtAppendExecutionProvider, which resolves an EP name against ORT's built-in EP table, this selects + * from the devices registered with the environment, which is how a plugin EP is selected. Use OrtGetEpDevices + * and OrtEpDevice_EpName to choose the devices to pass in. + * + * @param ep_devices the devices to select, all of which must belong to the same execution provider + * @param num_ep_devices the number of entries in ep_devices + * @returns ORT error code. If not zero, call OrtGetLastError() to get detailed error message. + */ +int EMSCRIPTEN_KEEPALIVE OrtAppendExecutionProviderV2(ort_session_options_handle_t session_options, + ort_ep_device_handle_t* ep_devices, + size_t num_ep_devices, + const char* const* provider_options_keys, + const char* const* provider_options_values, + size_t num_keys); + /** * add a free dimension override for one dimension of a session's input. */ diff --git a/onnxruntime/wasm/pre-async.js b/onnxruntime/wasm/pre-async.js index 1f8f17535e7d4..a19e3c3be1fdf 100644 --- a/onnxruntime/wasm/pre-async.js +++ b/onnxruntime/wasm/pre-async.js @@ -16,6 +16,7 @@ let initAsyncImpl = () => { // // Currently in ASYNCIFY build, we only use this for the following functions: // - OrtAppendExecutionProvider() + // - OrtAppendExecutionProviderV2() // - OrtCreateSession() // - OrtRun() // - OrtRunWithBinding() @@ -49,6 +50,7 @@ let initAsyncImpl = () => { wrapAsyncAPIs([ "_OrtAppendExecutionProvider", + "_OrtAppendExecutionProviderV2", "_OrtCreateSession", "_OrtRun", "_OrtRunWithBinding", diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index fe1bfa3f637d8..25d08695fcfde 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -575,6 +575,10 @@ def generate_build_tree( "-Donnxruntime_USE_ACL=" + ("ON" if args.use_acl else "OFF"), "-Donnxruntime_USE_JSEP=" + ("ON" if args.use_jsep else "OFF"), "-Donnxruntime_USE_WEBGPU=" + ("ON" if args.use_webgpu else "OFF"), + # The WebGPU EP library kind selects these. Emit them unconditionally, including the OFF case, so that + # reusing a build directory cannot leak a previous build's setting in via the CMake cache. + "-Donnxruntime_USE_EP_API_ADAPTERS=" + ("ON" if args.use_webgpu in ("shared_lib", "static_plugin") else "OFF"), + "-Donnxruntime_WEBGPU_STATIC_PLUGIN=" + ("ON" if args.use_webgpu == "static_plugin" else "OFF"), "-Donnxruntime_USE_EXTERNAL_DAWN=" + ("ON" if args.use_external_dawn else "OFF"), "-DDAWN_USE_AGILITY_SDK=" + ("ON" if args.use_dawn_agility_sdk else "OFF"), # Training related flags @@ -991,7 +995,6 @@ def generate_build_tree( ) elif args.use_webgpu == "shared_lib": # Shared library build (plugin EP) - cmake_args += ["-Donnxruntime_USE_EP_API_ADAPTERS=ON"] if args.build_wasm: raise BuildError("Only static library build of WebGPU EP is supported for WebAssembly build.") diff --git a/tools/ci_build/build_args.py b/tools/ci_build/build_args.py index 57077d044c086..341c1f4d0cfda 100644 --- a/tools/ci_build/build_args.py +++ b/tools/ci_build/build_args.py @@ -99,7 +99,7 @@ def invalid_hetero_build() -> None: def _webgpu_verify_library_kind(library_kind: str) -> str: """Verifies the library kind for the WebGPU Execution Provider.""" - choices = ["shared_lib", "static_lib"] + choices = ["shared_lib", "static_lib", "static_plugin"] if library_kind not in choices: print("\nYou have specified an invalid library kind for WebGPU EP.") print(f"The invalid library kind was: {library_kind}") @@ -828,7 +828,10 @@ def add_execution_provider_args(parser: argparse.ArgumentParser) -> None: nargs="?", const="static_lib", type=_webgpu_verify_library_kind, - help="Enable WebGPU EP. Optionally specify 'static_lib' (default) or 'shared_lib'.", + help="Enable WebGPU EP. Optionally specify 'static_lib' (default), 'shared_lib', or 'static_plugin'. " + "'static_lib' builds the EP as an internal ORT EP. 'shared_lib' builds it as a separate plugin EP " + "library that is loaded at runtime. 'static_plugin' builds it as a plugin EP that is linked into the " + "ORT binary and registered at environment creation.", ) webgpu_group.add_argument( "--use_external_dawn", action="store_true", help="Use external Dawn dependency for WebGPU."