diff --git a/cmake/onnxruntime.cmake b/cmake/onnxruntime.cmake index 146e37f8ecda8..75e440bfe5576 100644 --- a/cmake/onnxruntime.cmake +++ b/cmake/onnxruntime.cmake @@ -255,6 +255,10 @@ if (CMAKE_SYSTEM_NAME MATCHES "AIX") list(APPEND onnxruntime_INTERNAL_LIBRARIES iconv) endif() +if(NOT onnxruntime_MINIMAL_BUILD AND TARGET model_package) + list(APPEND onnxruntime_INTERNAL_LIBRARIES model_package) +endif() + if (onnxruntime_USE_EXTENSIONS) list(APPEND onnxruntime_INTERNAL_LIBRARIES onnxruntime_extensions diff --git a/cmake/onnxruntime_session.cmake b/cmake/onnxruntime_session.cmake index 1e3357464e991..6fb895cd1800c 100644 --- a/cmake/onnxruntime_session.cmake +++ b/cmake/onnxruntime_session.cmake @@ -11,6 +11,19 @@ file(GLOB onnxruntime_session_srcs CONFIGURE_DEPENDS "${ONNXRUNTIME_ROOT}/core/session/model_package/*.cc" ) +# Standalone model package library (parsing/inspection with no ORT dependency). +# Compiled as a static library and linked into onnxruntime_session. +# NOTE: ORT intentionally uses the library's internal C++ types directly (model_package::ParsePackage, +# model_package_internal.h) rather than going through its public C API (ModelPackage_*). This avoids +# double-wrapping (ORT C API -> standalone C API -> C++ internals). The public C API exists for +# external consumers (GenAI, FL) who link against the standalone library independently. +set(MODEL_PACKAGE_LIB_DIR "${REPO_ROOT}/model_package") +if(NOT onnxruntime_MINIMAL_BUILD) + set(MODEL_PACKAGE_BUILD_SHARED OFF CACHE BOOL "" FORCE) + set(MODEL_PACKAGE_BUILD_TESTS OFF CACHE BOOL "" FORCE) + add_subdirectory(${MODEL_PACKAGE_LIB_DIR} ${CMAKE_CURRENT_BINARY_DIR}/model_package EXCLUDE_FROM_ALL) +endif() + if (onnxruntime_ENABLE_TRAINING_APIS) file(GLOB_RECURSE training_api_srcs CONFIGURE_DEPENDS "${ORTTRAINING_SOURCE_DIR}/training_api/*.cc" @@ -44,6 +57,10 @@ source_group(TREE ${REPO_ROOT} FILES ${onnxruntime_session_srcs}) onnxruntime_add_static_library(onnxruntime_session ${onnxruntime_session_srcs}) onnxruntime_add_include_to_target(onnxruntime_session onnxruntime_common onnxruntime_framework onnxruntime_lora onnx onnx_proto ${PROTOBUF_LIB} flatbuffers::flatbuffers Boost::mp11 safeint_interface nlohmann_json::nlohmann_json Eigen3::Eigen) target_link_libraries(onnxruntime_session PRIVATE onnxruntime_lora) +if(TARGET model_package) + target_link_libraries(onnxruntime_session PRIVATE model_package) + target_include_directories(onnxruntime_session PRIVATE ${MODEL_PACKAGE_LIB_DIR}/include ${MODEL_PACKAGE_LIB_DIR}/src) +endif() if(onnxruntime_ENABLE_INSTRUMENT) target_compile_definitions(onnxruntime_session PUBLIC ONNXRUNTIME_ENABLE_INSTRUMENT) endif() @@ -74,4 +91,10 @@ if (NOT onnxruntime_BUILD_SHARED_LIB) LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} FRAMEWORK DESTINATION ${CMAKE_INSTALL_BINDIR}) + if(TARGET model_package) + install(TARGETS model_package EXPORT ${PROJECT_NAME}Targets + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) + endif() endif() diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index d90189496af42..331137292463c 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -348,6 +348,9 @@ ORT_RUNTIME_CLASS(ExternalSemaphoreHandle); // EP-imported view of shared exte ORT_RUNTIME_CLASS(DeviceEpIncompatibilityDetails); ORT_RUNTIME_CLASS(EpAssignedSubgraph); ORT_RUNTIME_CLASS(EpAssignedNode); +ORT_RUNTIME_CLASS(ModelPackageOptions); +ORT_RUNTIME_CLASS(ModelPackageContext); +ORT_RUNTIME_CLASS(ModelPackageComponentContext); #ifdef _MSC_VER typedef _Return_type_success_(return == 0) OrtStatus* OrtStatusPtr; @@ -925,6 +928,9 @@ typedef struct OrtCompileApi OrtCompileApi; struct OrtInteropApi; typedef struct OrtInteropApi OrtInteropApi; +struct OrtModelPackageApi; +typedef struct OrtModelPackageApi OrtModelPackageApi; + struct OrtEpApi; typedef struct OrtEpApi OrtEpApi; @@ -7486,6 +7492,26 @@ struct OrtApi { * \since Version 1.27. */ ORT_API2_STATUS(SessionReleaseCapturedGraph, _In_ OrtSession* session, _In_ int graph_annotation_id); + + /** \brief Get the model package API table. + * + * Returns a pointer to the ::OrtModelPackageApi function table, which provides APIs to: + * - create and release model package options and contexts, + * - inspect model package metadata (components/variants), + * - select a component/variant and query selected files/options, + * - create a session from model package selection results. + * + * The returned pointer is owned by ONNX Runtime and is valid for the process lifetime. + * Do not free it. + * + * \note May return NULL if model package support is not available in the current build + * (for example, minimal builds). + * + * \return Pointer to ::OrtModelPackageApi, or NULL if unsupported. + * + * \since Version 1.27. + */ + const OrtModelPackageApi*(ORT_API_CALL* GetModelPackageApi)(void); }; /* @@ -8573,6 +8599,250 @@ struct OrtInteropApi { /// @} }; +/** \brief API table for model package workflows. + * + * A model package is a directory containing one or more *components* (logical models). + * Each component has one or more *variants*, where each variant targets a single + * execution provider (EP). The package manifest declares the EP name, device type, + * and an optional compatibility string for every variant so that the runtime can + * automatically select the best variant for the hardware and EPs available in the + * caller's session options. + * + * Obtain this table from OrtApi::GetModelPackageApi(). The APIs support: + * - creating model package options that capture EP configuration from OrtSessionOptions, + * - loading a package context (manifest + metadata) from a package root path, + * - querying component/variant metadata including per-variant EP information, + * - selecting a component (which also resolves the best-matching variant), + * - querying the selected variant's name and folder path, + * - creating an OrtSession from the selected component context. + * + * Typical flow: + * 1) Create model package options: + * - CreateModelPackageOptionsFromSessionOptions() + * 2) Load package metadata: + * - CreateModelPackageContext() + * 3) Query metadata (optional): + * - ModelPackage_GetSchemaVersion() + * - ModelPackage_GetComponentCount() + * - ModelPackage_GetComponentNames() + * - ModelPackage_GetVariantCount() + * - ModelPackage_GetVariantNames() + * - ModelPackage_GetVariantEpName() + * 4) Select a component and resolve variant: + * - SelectComponent() + * 5) Query selected variant info (optional): + * - ModelPackageComponent_GetSelectedVariantName() + * - ModelPackageComponent_GetSelectedVariantFolderPath() + * 6) Create session: + * - CreateSession() + * + * Ownership: + * - Release objects created by this API with the corresponding release methods: + * ReleaseModelPackageOptions(), ReleaseModelPackageContext(), + * ReleaseModelPackageComponentContext(). + * + * \since Version 1.27. + */ +struct OrtModelPackageApi { + /// \name OrtModelPackageOptions + /// @{ + + /** \brief Create model package options from an existing OrtSessionOptions. + * + * Captures EP configuration (registered execution providers and their devices) from + * the session options for use during variant selection. The resulting OrtModelPackageOptions + * is passed to SelectComponent() to resolve the best variant for the available EPs. + * + * \param[in] env The ORT environment. + * \param[in] session_options Session options containing registered EPs. + * \param[out] out Receives the newly created OrtModelPackageOptions. Must be released + * with ReleaseModelPackageOptions(). + * + * \since Version 1.27. + */ + ORT_API2_STATUS(CreateModelPackageOptionsFromSessionOptions, + _In_ const OrtEnv* env, + _In_ const OrtSessionOptions* session_options, + _Outptr_ OrtModelPackageOptions** out); + + ORT_CLASS_RELEASE(ModelPackageOptions); + /// @} + /// \name OrtModelPackageContext + /// @{ + + /** \brief Create a model package context by parsing the package at the given root path. + * + * Parses the manifest.json and component metadata from the specified directory. + * The returned context provides read-only access to the package structure (components, + * variants, EP declarations). + * + * \param[in] package_root Path to the model package root directory (containing manifest.json). + * \param[out] out Receives the newly created OrtModelPackageContext. Must be released + * with ReleaseModelPackageContext(). + * + * \since Version 1.27. + */ + ORT_API2_STATUS(CreateModelPackageContext, + _In_ const ORTCHAR_T* package_root, + _Outptr_ OrtModelPackageContext** out); + + ORT_CLASS_RELEASE(ModelPackageContext); + + /** \brief Get the schema version declared in the model package manifest. + * + * \param[in] ctx The model package context. + * \param[out] out_version Receives the schema version number. + * + * \since Version 1.27. + */ + ORT_API2_STATUS(ModelPackage_GetSchemaVersion, + _In_ const OrtModelPackageContext* ctx, + _Out_ int64_t* out_version); + + /** \brief Get the number of components in the model package. + * + * \param[in] ctx The model package context. + * \param[out] out_count Receives the component count. + * + * \since Version 1.27. + */ + ORT_API2_STATUS(ModelPackage_GetComponentCount, + _In_ const OrtModelPackageContext* ctx, + _Out_ size_t* out_count); + + /** \brief Get the names of all components in the model package. + * + * Returns a pointer to an array of UTF-8 component name strings. The array and its + * strings are owned by `ctx` and remain valid until the context is released. + * + * \param[in] ctx The model package context. + * \param[out] out_names Receives a pointer to an array of component name strings. + * \param[out] out_count Receives the number of elements in the array. + * + * \since Version 1.27. + */ + ORT_API2_STATUS(ModelPackage_GetComponentNames, + _In_ const OrtModelPackageContext* ctx, + _Outptr_result_buffer_maybenull_(*out_count) const char* const** out_names, + _Out_ size_t* out_count); + + /** \brief Get the number of variants for a given component. + * + * \param[in] ctx The model package context. + * \param[in] component_name Name of the component to query. + * \param[out] out_count Receives the variant count. + * + * \since Version 1.27. + */ + ORT_API2_STATUS(ModelPackage_GetVariantCount, + _In_ const OrtModelPackageContext* ctx, + _In_ const char* component_name, + _Out_ size_t* out_count); + + /** \brief Get the names of all variants for a given component. + * + * Returns a pointer to an array of UTF-8 variant name strings. The array and its + * strings are owned by `ctx` and remain valid until the context is released. + * + * \param[in] ctx The model package context. + * \param[in] component_name Name of the component to query. + * \param[out] out_variant_names Receives a pointer to an array of variant name strings. + * \param[out] out_count Receives the number of elements in the array. + * + * \since Version 1.27. + */ + ORT_API2_STATUS(ModelPackage_GetVariantNames, + _In_ const OrtModelPackageContext* ctx, + _In_ const char* component_name, + _Outptr_result_buffer_maybenull_(*out_count) const char* const** out_variant_names, + _Out_ size_t* out_count); + + /** \brief Get the EP name declared for a (component, variant) pair. + * + * Each variant targets a single EP. `out_ep` receives the EP name string. + * When the variant does not declare an EP, the returned pointer is NULL. + * String memory is owned by `ctx` and remains valid until the context is released. + * + * \since Version 1.27. + */ + ORT_API2_STATUS(ModelPackage_GetVariantEpName, + _In_ const OrtModelPackageContext* ctx, + _In_ const char* component_name, + _In_ const char* variant_name, + _Outptr_result_maybenull_ const char** out_ep); + + /** \brief Select a component model and return an opaque component instance. + * + * The variant selection is also performed during this call based on the component metadata and the provided options. + * The returned `OrtModelPackgeComponentContext*` is independent of `context` lifetime and must be released via + * `ReleaseComponentInstance`. + * + * \since Version 1.27. + */ + ORT_API2_STATUS(SelectComponent, + _In_ const OrtModelPackageContext* context, + _In_ const char* component_name, + _In_ const OrtModelPackageOptions* options, + _Outptr_ OrtModelPackageComponentContext** out); + + ORT_CLASS_RELEASE(ModelPackageComponentContext); + + /** \brief Get the name of the selected variant after SelectComponent has been called. + * + * String memory is owned by `ctx` and remains valid until the context is released. + * + * \param[in] ctx The component context returned by SelectComponent(). + * \param[out] out_name Receives the selected variant's name string. + * + * \since Version 1.27. + */ + ORT_API2_STATUS(ModelPackageComponent_GetSelectedVariantName, + _In_ const OrtModelPackageComponentContext* ctx, + _Outptr_ const char** out_name); + + /** \brief Get the folder path of the selected variant. + * + * Returns the resolved absolute path to the variant's directory on disk. + * The string is owned by `ctx` and remains valid until the context is released. + * + * \param[in] ctx The component context returned by SelectComponent(). + * \param[out] folder_path Receives the variant folder path string. + * + * \since Version 1.27. + */ + ORT_API2_STATUS(ModelPackageComponent_GetSelectedVariantFolderPath, + _In_ const OrtModelPackageComponentContext* ctx, + _Outptr_ const ORTCHAR_T** folder_path); + + /// @} + /** \brief Create an OrtSession for a selected file within a component model variant. + * + * The chosen variant (and thus its EP selection) is determined by `context`, which + * was built from an OrtSessionOptions via CreateModelPackageOptionsFromSessionOptions. + * + * Session options precedence: + * 1. session_options == NULL (default path): + * ORT uses the OrtSessionOptions that was captured when `context` was created. + * Any variant-specific session and provider options declared in the package + * metadata are merged on top. + * + * 2. session_options != NULL (advanced path): + * ORT uses the caller-provided OrtSessionOptions as-is. Variant-specific + * session and provider options from the package metadata are NOT applied. + * Use this when custom EP setup is required (e.g., shared CUDA streams, + * shared QNN EP contexts, custom allocators). + * + * \since Version 1.27. + */ + ORT_API2_STATUS(CreateSession, + _In_ const OrtEnv* env, + _In_ OrtModelPackageComponentContext* context, + _In_opt_ const OrtSessionOptions* session_options, + _Outptr_ OrtSession** session); + + // End of Version 1.27 - DO NOT MODIFY ABOVE +}; + /* * This is the old way to add the CUDA provider to the session, please use SessionOptionsAppendExecutionProvider_CUDA above to access the latest functionality * This function always exists, but will only succeed if Onnxruntime was built with CUDA support and the CUDA provider shared library exists diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_api.h b/include/onnxruntime/core/session/onnxruntime_cxx_api.h index 42eeac19da377..59c08928b8068 100644 --- a/include/onnxruntime/core/session/onnxruntime_cxx_api.h +++ b/include/onnxruntime/core/session/onnxruntime_cxx_api.h @@ -264,6 +264,20 @@ inline const OrtEpApi& GetEpApi() { return *api; } +/// +/// This returns a reference to the ORT C Model Package API. Used for loading models from model packages. +/// +/// ORT C Model Package API reference +inline const OrtModelPackageApi& GetModelPackageApi() { + auto* api = GetApi().GetModelPackageApi(); + if (api == nullptr) { + // minimal build + ORT_CXX_API_THROW("Model Package API is not available in this build", ORT_FAIL); + } + + return *api; +} + /** \brief IEEE 754 half-precision floating point data type * * \details This struct is used for converting float to float16 and back @@ -664,6 +678,9 @@ ORT_DEFINE_RELEASE_FROM_API_STRUCT(KernelDefBuilder, GetEpApi); ORT_DEFINE_RELEASE_FROM_API_STRUCT(KernelRegistry, GetEpApi); ORT_DEFINE_RELEASE_FROM_API_STRUCT(OpSchema, GetEpApi); ORT_DEFINE_RELEASE_FROM_API_STRUCT(ProfilingEvent, GetEpApi); +ORT_DEFINE_RELEASE_FROM_API_STRUCT(ModelPackageOptions, GetModelPackageApi); +ORT_DEFINE_RELEASE_FROM_API_STRUCT(ModelPackageContext, GetModelPackageApi); +ORT_DEFINE_RELEASE_FROM_API_STRUCT(ModelPackageComponentContext, GetModelPackageApi); // This is defined explicitly since OrtTensorRTProviderOptionsV2 is not a C API type, // but the struct has V2 in its name to indicate that it is the second version of the options. @@ -790,6 +807,9 @@ struct EpDevice; struct ExternalInitializerInfo; struct Graph; struct Model; +struct ModelPackageOptions; +struct ModelPackageContext; +struct ModelPackageComponentContext; struct Node; struct ModelMetadata; struct TypeInfo; @@ -1786,6 +1806,70 @@ struct ModelCompilationOptions : detail::Base { */ Status CompileModel(const Env& env, const ModelCompilationOptions& model_compilation_options); +/** \brief Options for selecting a component from a model package. + * + * Wraps ::OrtModelPackageOptions. Created from an Env and SessionOptions, which captures the + * EP configuration used for variant selection. + */ +struct ModelPackageOptions : detail::Base { + using Base = detail::Base; + using Base::Base; + + explicit ModelPackageOptions(std::nullptr_t) {} ///< Create an empty object, must be assigned a valid one to be used. + + ModelPackageOptions(const Env& env, const SessionOptions& session_options); ///< Wraps OrtModelPackageApi::CreateModelPackageOptionsFromSessionOptions + ModelPackageOptions(const Env& env, ConstSessionOptions session_options); ///< Wraps OrtModelPackageApi::CreateModelPackageOptionsFromSessionOptions +}; + +/** \brief Context for inspecting and selecting components from a model package. + * + * Wraps ::OrtModelPackageContext. Provides traversal APIs to enumerate components, variants, + * and EP compatibility, as well as component selection. + */ +struct ModelPackageContext : detail::Base { + using Base = detail::Base; + using Base::Base; + + explicit ModelPackageContext(std::nullptr_t) {} ///< Create an empty object, must be assigned a valid one to be used. + + explicit ModelPackageContext(const ORTCHAR_T* package_root); ///< Wraps OrtModelPackageApi::CreateModelPackageContext + + size_t GetComponentCount() const; ///< Wraps OrtModelPackageApi::ModelPackage_GetComponentCount + std::vector GetComponentNames() const; ///< Wraps OrtModelPackageApi::ModelPackage_GetComponentNames + size_t GetVariantCount(const char* component_name) const; ///< Wraps OrtModelPackageApi::ModelPackage_GetVariantCount + std::vector GetVariantNames(const char* component_name) const; ///< Wraps OrtModelPackageApi::ModelPackage_GetVariantNames + + /// Get the EP name for a variant. Returns nullptr if not declared. + /// Returned string is owned by this context and valid until it is released. + const char* GetVariantEpName(const char* component_name, + const char* variant_name) const; ///< Wraps OrtModelPackageApi::ModelPackage_GetVariantEpName + + int64_t GetSchemaVersion() const; ///< Wraps OrtModelPackageApi::ModelPackage_GetSchemaVersion + + ModelPackageComponentContext SelectComponent(const char* component_name, + const ModelPackageOptions& options) const; ///< Wraps OrtModelPackageApi::SelectComponent +}; + +/** \brief Context for a selected component within a model package. + * + * Wraps ::OrtModelPackageComponentContext. Provides accessors for the selected variant's + * folder path and variant name. + */ +struct ModelPackageComponentContext : detail::Base { + using Base = detail::Base; + using Base::Base; + + explicit ModelPackageComponentContext(std::nullptr_t) {} ///< Create an empty object, must be assigned a valid one to be used. + + std::basic_string GetSelectedVariantFolderPath() const; ///< Wraps OrtModelPackageApi::ModelPackageComponent_GetSelectedVariantFolderPath + + std::string GetSelectedVariantName() const; ///< Wraps OrtModelPackageApi::ModelPackageComponent_GetSelectedVariantName + + Session CreateSession(const Env& env); ///< Wraps OrtModelPackageApi::CreateSession (default path, NULL session_options) + Session CreateSession(const Env& env, const SessionOptions& session_options); ///< Wraps OrtModelPackageApi::CreateSession (advanced path) + Session CreateSession(const Env& env, ConstSessionOptions session_options); ///< Wraps OrtModelPackageApi::CreateSession (advanced path) +}; + /** \brief Wrapper around ::OrtModelMetadata * */ diff --git a/include/onnxruntime/core/session/onnxruntime_cxx_inline.h b/include/onnxruntime/core/session/onnxruntime_cxx_inline.h index 61bc31736f5b5..78175729cbefd 100644 --- a/include/onnxruntime/core/session/onnxruntime_cxx_inline.h +++ b/include/onnxruntime/core/session/onnxruntime_cxx_inline.h @@ -1360,6 +1360,110 @@ inline ModelCompilationOptions& ModelCompilationOptions::SetInputModel(const Ort return *this; } +// ModelPackageOptions +inline ModelPackageOptions::ModelPackageOptions(const Env& env, const SessionOptions& session_options) { + ThrowOnError(GetModelPackageApi().CreateModelPackageOptionsFromSessionOptions(env, session_options, &this->p_)); +} + +inline ModelPackageOptions::ModelPackageOptions(const Env& env, ConstSessionOptions session_options) { + ThrowOnError(GetModelPackageApi().CreateModelPackageOptionsFromSessionOptions(env, session_options, &this->p_)); +} + +// ModelPackageContext +inline ModelPackageContext::ModelPackageContext(const ORTCHAR_T* package_root) { + ThrowOnError(GetModelPackageApi().CreateModelPackageContext(package_root, &this->p_)); +} + +inline size_t ModelPackageContext::GetComponentCount() const { + size_t count = 0; + ThrowOnError(GetModelPackageApi().ModelPackage_GetComponentCount(this->p_, &count)); + return count; +} + +inline std::vector ModelPackageContext::GetComponentNames() const { + const char* const* names = nullptr; + size_t count = 0; + ThrowOnError(GetModelPackageApi().ModelPackage_GetComponentNames(this->p_, &names, &count)); + std::vector result; + result.reserve(count); + for (size_t i = 0; i < count; ++i) { + result.emplace_back(names[i]); + } + return result; +} + +inline size_t ModelPackageContext::GetVariantCount(const char* component_name) const { + size_t count = 0; + ThrowOnError(GetModelPackageApi().ModelPackage_GetVariantCount(this->p_, component_name, &count)); + return count; +} + +inline std::vector ModelPackageContext::GetVariantNames(const char* component_name) const { + const char* const* names = nullptr; + size_t count = 0; + ThrowOnError(GetModelPackageApi().ModelPackage_GetVariantNames(this->p_, component_name, &names, &count)); + std::vector result; + result.reserve(count); + for (size_t i = 0; i < count; ++i) { + result.emplace_back(names[i]); + } + return result; +} + +inline const char* ModelPackageContext::GetVariantEpName(const char* component_name, + const char* variant_name) const { + const char* ep = nullptr; + ThrowOnError(GetModelPackageApi().ModelPackage_GetVariantEpName( + this->p_, component_name, variant_name, &ep)); + return ep; +} + +inline int64_t ModelPackageContext::GetSchemaVersion() const { + int64_t version = 0; + ThrowOnError(GetModelPackageApi().ModelPackage_GetSchemaVersion(this->p_, &version)); + return version; +} + +inline ModelPackageComponentContext ModelPackageContext::SelectComponent( + const char* component_name, const ModelPackageOptions& options) const { + OrtModelPackageComponentContext* out = nullptr; + ThrowOnError(GetModelPackageApi().SelectComponent(this->p_, component_name, options, &out)); + return ModelPackageComponentContext{out}; +} + +// ModelPackageComponentContext +inline std::basic_string ModelPackageComponentContext::GetSelectedVariantFolderPath() const { + const ORTCHAR_T* path = nullptr; + ThrowOnError(GetModelPackageApi().ModelPackageComponent_GetSelectedVariantFolderPath(this->p_, &path)); + return std::basic_string{path}; +} + +inline std::string ModelPackageComponentContext::GetSelectedVariantName() const { + const char* name = nullptr; + ThrowOnError(GetModelPackageApi().ModelPackageComponent_GetSelectedVariantName(this->p_, &name)); + return (name != nullptr) ? std::string{name} : std::string{}; +} + +inline Session ModelPackageComponentContext::CreateSession(const Env& env) { + OrtSession* out = nullptr; + ThrowOnError(GetModelPackageApi().CreateSession(env, this->p_, nullptr, &out)); + return Session{out}; +} + +inline Session ModelPackageComponentContext::CreateSession(const Env& env, + const SessionOptions& session_options) { + OrtSession* out = nullptr; + ThrowOnError(GetModelPackageApi().CreateSession(env, this->p_, session_options, &out)); + return Session{out}; +} + +inline Session ModelPackageComponentContext::CreateSession(const Env& env, + ConstSessionOptions session_options) { + OrtSession* out = nullptr; + ThrowOnError(GetModelPackageApi().CreateSession(env, this->p_, session_options, &out)); + return Session{out}; +} + namespace detail { template diff --git a/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h b/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h index 9942ce3d0bf8b..0b6d009e072ad 100644 --- a/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h +++ b/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h @@ -558,3 +558,72 @@ static const char* const kOrtSessionOptionsRecordEpGraphAssignmentInfo = "sessio // - "0": disable. (default) // - "1": enable. static const char* const kOrtSessionOptionEpEnableWeightlessEpContextNodes = "ep.enable_weightless_ep_context_nodes"; + +// Controls the intra-op thread pool size for a session. +// Value should be a base-10 int32 string. +// Equivalent to OrtApi::SetIntraOpNumThreads. +static const char* const kOrtSessionOptionsConfigIntraOpNumThreads = "session.intra_op_num_threads"; + +// Controls the inter-op thread pool size for a session. +// Value should be a base-10 int32 string. +// Equivalent to OrtApi::SetInterOpNumThreads. +static const char* const kOrtSessionOptionsConfigInterOpNumThreads = "session.inter_op_num_threads"; + +// Enable or disable the CPU memory arena for a session. +// "0": disable; "1": enable. +// Equivalent to OrtApi::DisableCpuMemArena / OrtApi::EnableCpuMemArena. +static const char* const kOrtSessionOptionsConfigEnableCpuMemArena = "session.enable_cpu_mem_arena"; + +// Enable or disable memory pattern optimization for a session. +// "0": disable; "1": enable. +// Equivalent to OrtApi::DisableMemPattern / OrtApi::EnableMemPattern. +static const char* const kOrtSessionOptionsConfigEnableMemPattern = "session.enable_mem_pattern"; + +// Session log identifier. +// Value should be a UTF-8 string. +// Equivalent to OrtApi::SetSessionLogId. +static const char* const kOrtSessionOptionsConfigLogId = "session.log_id"; + +// Session log severity level. +// Value should be a base-10 int32 string (refer to OrtLoggingLevel values). +// Equivalent to OrtApi::SetSessionLogSeverityLevel. +static const char* const kOrtSessionOptionsConfigLogSeverityLevel = "session.log_severity_level"; + +// Session log verbosity level. +// Value should be a base-10 int32 string. +// Equivalent to OrtApi::SetSessionLogVerbosityLevel. +static const char* const kOrtSessionOptionsConfigLogVerbosityLevel = "session.log_verbosity_level"; + +// Enable or disable profiling for a session. +// Empty string: disable profiling. +// Non-empty string: enable profiling and use value as profile file prefix. +// Equivalent to OrtApi::DisableProfiling / OrtApi::EnableProfiling. +static const char* const kOrtSessionOptionsConfigEnableProfiling = "session.enable_profiling"; + +// Graph optimization level for a session. +// Value should be one of: +// "disable_all", "enable_basic", "enable_extended", "enable_layout", "enable_all". +// Equivalent to OrtApi::SetSessionGraphOptimizationLevel. +static const char* const kOrtSessionOptionsConfigGraphOptimizationLevel = "session.graph_optimization_level"; + +// File path for saving the optimized model. +// Value should be a path string. +// Equivalent to OrtApi::SetOptimizedModelFilePath. +static const char* const kOrtSessionOptionsConfigOptimizedModelFilePath = "session.optimized_model_filepath"; + +// Session execution mode. +// Value should be one of: +// "sequential", "parallel", "ort_sequential", "ort_parallel". +// Equivalent to OrtApi::SetSessionExecutionMode. +static const char* const kOrtSessionOptionsConfigExecutionMode = "session.execution_mode"; + +// Controls whether to use per-session thread pools. +// "0": disable per-session threads (use global thread pools). +// "1": keep per-session threads enabled (default behavior before disable call). +// Equivalent to OrtApi::DisablePerSessionThreads (one-way via API). +static const char* const kOrtSessionOptionsConfigUsePerSessionThreads = "session.use_per_session_threads"; + +// Enable or disable deterministic compute for a session. +// "0": disable; "1": enable. +// Equivalent to OrtApi::SetDeterministicCompute. +static const char* const kOrtSessionOptionsConfigUseDeterministicCompute = "session.use_deterministic_compute"; diff --git a/model_package/CMakeLists.txt b/model_package/CMakeLists.txt new file mode 100644 index 0000000000000..326a1e541696a --- /dev/null +++ b/model_package/CMakeLists.txt @@ -0,0 +1,96 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +cmake_minimum_required(VERSION 3.18) + +project(model_package VERSION 1.0.0 LANGUAGES CXX) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_CXX_EXTENSIONS OFF) + +# ───────────────────────────────────────────────────────────────────────────── +# Options +# ───────────────────────────────────────────────────────────────────────────── + +option(MODEL_PACKAGE_BUILD_SHARED "Build as shared library" ON) +option(MODEL_PACKAGE_BUILD_TESTS "Build tests" OFF) + +# ───────────────────────────────────────────────────────────────────────────── +# Dependencies +# ───────────────────────────────────────────────────────────────────────────── + +# Allow the user to provide a pre-existing nlohmann_json via NLOHMANN_JSON_DIR +# (pointing to the source tree with include/nlohmann/json.hpp). +# Falls back to FetchContent if not provided. +# When built as a subdirectory (e.g., within ORT), the target may already exist. +if(TARGET nlohmann_json::nlohmann_json) + # Already available from parent project — nothing to do. +elseif(NLOHMANN_JSON_DIR) + # Use the provided directory directly. + add_library(nlohmann_json_header_only INTERFACE) + target_include_directories(nlohmann_json_header_only INTERFACE + "${NLOHMANN_JSON_DIR}/include" + "${NLOHMANN_JSON_DIR}/single_include" + ) + add_library(nlohmann_json::nlohmann_json ALIAS nlohmann_json_header_only) +else() + find_package(nlohmann_json QUIET) + if(NOT nlohmann_json_FOUND) + include(FetchContent) + FetchContent_Declare( + nlohmann_json + URL https://github.com/nlohmann/json/releases/download/v3.11.3/json.tar.xz + URL_HASH SHA256=d6c65aca6b1ed68e7a182f4757f0f1b0b77571b1c626ade15f15d6f0b7b16d31 + ) + FetchContent_MakeAvailable(nlohmann_json) + endif() +endif() + +# ───────────────────────────────────────────────────────────────────────────── +# Library target +# ───────────────────────────────────────────────────────────────────────────── + +set(MODEL_PACKAGE_SOURCES + src/api.cc + src/parser.cc +) + +if(MODEL_PACKAGE_BUILD_SHARED) + add_library(model_package SHARED ${MODEL_PACKAGE_SOURCES}) + target_compile_definitions(model_package PRIVATE MODEL_PACKAGE_DLL_EXPORT) +else() + add_library(model_package STATIC ${MODEL_PACKAGE_SOURCES}) +endif() + +target_include_directories(model_package + PUBLIC + $ + $ + PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/src +) + +target_link_libraries(model_package PRIVATE nlohmann_json::nlohmann_json) + +set_target_properties(model_package PROPERTIES + POSITION_INDEPENDENT_CODE ON + VERSION ${PROJECT_VERSION} + SOVERSION 1 +) + +# ───────────────────────────────────────────────────────────────────────────── +# Install +# ───────────────────────────────────────────────────────────────────────────── + +include(GNUInstallDirs) + +install(TARGETS model_package + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} + RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} +) + +install(FILES include/model_package_api.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR} +) diff --git a/model_package/README.md b/model_package/README.md new file mode 100644 index 0000000000000..604720916d764 --- /dev/null +++ b/model_package/README.md @@ -0,0 +1,78 @@ +# Model Package Library + +A standalone C library for parsing and inspecting ONNX Runtime Model Packages. + +**No dependency on ONNX Runtime.** This library can be consumed independently by any component (ORT, GenAI, FL, or external tools). + +## What it does + +- Parses model package directory structures (`manifest.json`, `metadata.json`, `variant.json`) +- Provides read-only access to: + - Components and their variants + - EP compatibility declarations (opaque strings) + - Model file paths within variants + - Session/provider options per file + - Consumer metadata (opaque JSON) + +## What it does NOT do + +- Variant selection (requires runtime EP factory validation → stays in ORT) +- Session creation (requires ORT `InferenceSession`) +- Any interpretation of `compatibility_string` tokens + +## Building + +```bash +cmake -B build -S . +cmake --build build +``` + +Options: +- `-DMODEL_PACKAGE_BUILD_SHARED=ON|OFF` — Build as shared (default) or static library +- `-DMODEL_PACKAGE_BUILD_TESTS=ON` — Build tests (default OFF) + +## C API Usage + +```c +#include "model_package_api.h" + +ModelPackageContext* ctx = NULL; +ModelPackageStatus* status = ModelPackage_CreateContext("/path/to/package", &ctx); +if (status != NULL) { + printf("Error: %s\n", ModelPackage_GetErrorMessage(status)); + ModelPackage_ReleaseStatus(status); + return; +} + +size_t count = 0; +ModelPackage_GetComponentCount(ctx, &count); + +for (size_t i = 0; i < count; i++) { + const char* name = NULL; + ModelPackage_GetComponentName(ctx, i, &name); + printf("Component: %s\n", name); +} + +ModelPackage_ReleaseContext(ctx); +``` + +## Integration with ORT + +ORT compiles this library as part of its build and wraps the C API through `OrtModelPackageApi`, adding: +- Variant selection via EP factory compatibility validation +- Session creation with merged options + +## Package Format + +``` +package_root/ +├── manifest.json # schema_version, components list +└── models/ + └── / + ├── metadata.json # variants + EP compatibility declarations + └── / + ├── variant.json # files list, consumer_metadata + └── model.onnx # (or other model files) +``` + +Single-component shorthand (metadata.json at root, no manifest.json) is also supported. diff --git a/model_package/include/model_package_api.h b/model_package/include/model_package_api.h new file mode 100644 index 0000000000000..ca840c8a33e0e --- /dev/null +++ b/model_package/include/model_package_api.h @@ -0,0 +1,155 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +/// \file model_package_api.h +/// \brief Standalone C API for parsing and inspecting ONNX Runtime Model Packages. +/// +/// This library has no dependency on ONNX Runtime. It provides read-only access to +/// model package structure: components, variants, EP compatibility declarations, +/// model files, session/provider options, and consumer metadata. +/// +/// Error handling: Functions that can fail return `ModelPackageStatus*`. +/// A nullptr return indicates success. On failure, use `ModelPackage_GetErrorMessage()` +/// to retrieve the error string, and `ModelPackage_ReleaseStatus()` to free it. +/// +/// Lifetime: All `const char*` pointers returned by this API are owned by the +/// `ModelPackageContext` and remain valid until it is released. + +#pragma once + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +// ───────────────────────────────────────────────────────────────────────────── +// Export macros +// ───────────────────────────────────────────────────────────────────────────── + +#ifdef _WIN32 +#ifdef MODEL_PACKAGE_DLL_EXPORT +#define MODEL_PACKAGE_API __declspec(dllexport) +#elif defined(MODEL_PACKAGE_DLL_IMPORT) +#define MODEL_PACKAGE_API __declspec(dllimport) +#else +#define MODEL_PACKAGE_API +#endif +#else +#ifdef MODEL_PACKAGE_DLL_EXPORT +#define MODEL_PACKAGE_API __attribute__((visibility("default"))) +#else +#define MODEL_PACKAGE_API +#endif +#endif + +// ───────────────────────────────────────────────────────────────────────────── +// Opaque types +// ───────────────────────────────────────────────────────────────────────────── + +/// Opaque status type. nullptr indicates success. +typedef struct ModelPackageStatus ModelPackageStatus; + +/// Opaque context holding a parsed model package. +typedef struct ModelPackageContext ModelPackageContext; + +// ───────────────────────────────────────────────────────────────────────────── +// Status API +// ───────────────────────────────────────────────────────────────────────────── + +/// Release a status object. Safe to call with nullptr. +MODEL_PACKAGE_API void ModelPackage_ReleaseStatus(ModelPackageStatus* status); + +/// Get the error message from a status object. Returns nullptr if status is nullptr. +/// The returned string is owned by the status object. +MODEL_PACKAGE_API const char* ModelPackage_GetErrorMessage(const ModelPackageStatus* status); + +// ───────────────────────────────────────────────────────────────────────────── +// Context lifecycle +// ───────────────────────────────────────────────────────────────────────────── + +/// Parse a model package from a directory path and create a context. +/// +/// \param[in] package_root_path Null-terminated UTF-8 path to the package root directory. +/// \param[out] out_context On success, receives the created context. Caller must release +/// via ModelPackage_ReleaseContext(). +/// \return nullptr on success, or a status object describing the error. +MODEL_PACKAGE_API ModelPackageStatus* ModelPackage_CreateContext( + const char* package_root_path, + ModelPackageContext** out_context); + +/// Release a model package context and all associated resources. +/// Safe to call with nullptr. +MODEL_PACKAGE_API void ModelPackage_ReleaseContext(ModelPackageContext* context); + +// ───────────────────────────────────────────────────────────────────────────── +// Package-level queries +// ───────────────────────────────────────────────────────────────────────────── + +/// Get the schema version declared in manifest.json. +MODEL_PACKAGE_API ModelPackageStatus* ModelPackage_GetSchemaVersion( + const ModelPackageContext* context, + int64_t* out_version); + +// ───────────────────────────────────────────────────────────────────────────── +// Component queries +// ───────────────────────────────────────────────────────────────────────────── + +/// Get the number of components in the package. +MODEL_PACKAGE_API ModelPackageStatus* ModelPackage_GetComponentCount( + const ModelPackageContext* context, + size_t* out_count); + +/// Get the name of a component by index. +/// +/// \param[in] context The package context. +/// \param[in] component_idx Zero-based index (must be < component count). +/// \param[out] out_name Receives a pointer to the component name string. +/// Lifetime is tied to the context. +MODEL_PACKAGE_API ModelPackageStatus* ModelPackage_GetComponentName( + const ModelPackageContext* context, + size_t component_idx, + const char** out_name); + +// ───────────────────────────────────────────────────────────────────────────── +// Variant queries +// ───────────────────────────────────────────────────────────────────────────── + +/// Get the number of variants for a component. +MODEL_PACKAGE_API ModelPackageStatus* ModelPackage_GetVariantCount( + const ModelPackageContext* context, + const char* component_name, + size_t* out_count); + +/// Get the name of a variant by index. +MODEL_PACKAGE_API ModelPackageStatus* ModelPackage_GetVariantName( + const ModelPackageContext* context, + const char* component_name, + size_t variant_idx, + const char** out_name); + +/// Get the folder path for a variant (resolved absolute path). +MODEL_PACKAGE_API ModelPackageStatus* ModelPackage_GetVariantFolderPath( + const ModelPackageContext* context, + const char* component_name, + const char* variant_name, + const char** out_path); + +// ───────────────────────────────────────────────────────────────────────────── +// EP compatibility queries +// ───────────────────────────────────────────────────────────────────────────── + +/// Get the EP name declared for a variant. +/// +/// Each variant targets a single EP. When the variant does not declare an EP, +/// the returned pointer is set to nullptr. +MODEL_PACKAGE_API ModelPackageStatus* ModelPackage_GetVariantEpName( + const ModelPackageContext* context, + const char* component_name, + const char* variant_name, + const char** out_ep); + +#ifdef __cplusplus +} // extern "C" +#endif diff --git a/model_package/src/api.cc b/model_package/src/api.cc new file mode 100644 index 0000000000000..103bff8e1a4a3 --- /dev/null +++ b/model_package/src/api.cc @@ -0,0 +1,243 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "model_package_api.h" +#include "model_package_internal.h" +#include "parser.h" + +#include +#include + +// ───────────────────────────────────────────────────────────────────────────── +// Status implementation +// ───────────────────────────────────────────────────────────────────────────── + +struct ModelPackageStatus { + std::string message; +}; + +static ModelPackageStatus* MakeError(std::string msg) { + return new (std::nothrow) ModelPackageStatus{std::move(msg)}; +} + +// ───────────────────────────────────────────────────────────────────────────── +// Context is the public opaque type wrapping ContextImpl +// ───────────────────────────────────────────────────────────────────────────── + +struct ModelPackageContext { + model_package::ContextImpl impl; +}; + +// ───────────────────────────────────────────────────────────────────────────── +// ContextImpl lookup helpers +// ───────────────────────────────────────────────────────────────────────────── + +namespace model_package { + +const Component* ContextImpl::FindComponent(const char* name) const { + for (const auto& c : package_info.components) { + if (c.name == name) return &c; + } + return nullptr; +} + +const Variant* ContextImpl::FindVariant(const char* component_name, const char* variant_name) const { + const auto* comp = FindComponent(component_name); + if (!comp) return nullptr; + for (const auto& v : comp->variants) { + if (v.name == variant_name) return &v; + } + return nullptr; +} + +} // namespace model_package + +// ───────────────────────────────────────────────────────────────────────────── +// Validation macro +// ───────────────────────────────────────────────────────────────────────────── + +#define RETURN_IF_NULL(ptr, param_name) \ + do { \ + if ((ptr) == nullptr) \ + return MakeError(std::string(param_name) + " must not be null."); \ + } while (0) + +// ───────────────────────────────────────────────────────────────────────────── +// C API implementation +// ───────────────────────────────────────────────────────────────────────────── + +extern "C" { + +void ModelPackage_ReleaseStatus(ModelPackageStatus* status) { + delete status; +} + +const char* ModelPackage_GetErrorMessage(const ModelPackageStatus* status) { + if (status == nullptr) return nullptr; + return status->message.c_str(); +} + +ModelPackageStatus* ModelPackage_CreateContext( + const char* package_root_path, + ModelPackageContext** out_context) { + RETURN_IF_NULL(package_root_path, "package_root_path"); + RETURN_IF_NULL(out_context, "out_context"); + + *out_context = nullptr; + + auto ctx = std::make_unique(); + std::string error; + + if (!model_package::ParsePackage( + std::filesystem::path(std::string(package_root_path)), + ctx->impl.package_info, error)) { + return MakeError(std::move(error)); + } + + // Build component names cache. + ctx->impl.component_names_cache.clear(); + for (const auto& c : ctx->impl.package_info.components) { + ctx->impl.component_names_cache.push_back(c.name); + } + + // Build variant names cache. + for (const auto& c : ctx->impl.package_info.components) { + auto& names = ctx->impl.variant_names_cache[c.name]; + names.clear(); + for (const auto& v : c.variants) { + names.push_back(v.name); + } + } + + *out_context = ctx.release(); + return nullptr; +} + +void ModelPackage_ReleaseContext(ModelPackageContext* context) { + delete context; +} + +ModelPackageStatus* ModelPackage_GetSchemaVersion( + const ModelPackageContext* context, + int64_t* out_version) { + RETURN_IF_NULL(context, "context"); + RETURN_IF_NULL(out_version, "out_version"); + *out_version = context->impl.package_info.schema_version; + return nullptr; +} + +ModelPackageStatus* ModelPackage_GetComponentCount( + const ModelPackageContext* context, + size_t* out_count) { + RETURN_IF_NULL(context, "context"); + RETURN_IF_NULL(out_count, "out_count"); + *out_count = context->impl.package_info.components.size(); + return nullptr; +} + +ModelPackageStatus* ModelPackage_GetComponentName( + const ModelPackageContext* context, + size_t component_idx, + const char** out_name) { + RETURN_IF_NULL(context, "context"); + RETURN_IF_NULL(out_name, "out_name"); + + if (component_idx >= context->impl.component_names_cache.size()) { + return MakeError("component_idx out of range: " + std::to_string(component_idx)); + } + + *out_name = context->impl.component_names_cache[component_idx].c_str(); + return nullptr; +} + +ModelPackageStatus* ModelPackage_GetVariantCount( + const ModelPackageContext* context, + const char* component_name, + size_t* out_count) { + RETURN_IF_NULL(context, "context"); + RETURN_IF_NULL(component_name, "component_name"); + RETURN_IF_NULL(out_count, "out_count"); + + const auto* comp = context->impl.FindComponent(component_name); + if (!comp) { + return MakeError(std::string("Component not found: '") + component_name + "'."); + } + + *out_count = comp->variants.size(); + return nullptr; +} + +ModelPackageStatus* ModelPackage_GetVariantName( + const ModelPackageContext* context, + const char* component_name, + size_t variant_idx, + const char** out_name) { + RETURN_IF_NULL(context, "context"); + RETURN_IF_NULL(component_name, "component_name"); + RETURN_IF_NULL(out_name, "out_name"); + + auto it = context->impl.variant_names_cache.find(component_name); + if (it == context->impl.variant_names_cache.end()) { + return MakeError(std::string("Component not found: '") + component_name + "'."); + } + + if (variant_idx >= it->second.size()) { + return MakeError("variant_idx out of range: " + std::to_string(variant_idx)); + } + + *out_name = it->second[variant_idx].c_str(); + return nullptr; +} + +ModelPackageStatus* ModelPackage_GetVariantFolderPath( + const ModelPackageContext* context, + const char* component_name, + const char* variant_name, + const char** out_path) { + RETURN_IF_NULL(context, "context"); + RETURN_IF_NULL(component_name, "component_name"); + RETURN_IF_NULL(variant_name, "variant_name"); + RETURN_IF_NULL(out_path, "out_path"); + + const auto* variant = context->impl.FindVariant(component_name, variant_name); + if (!variant) { + return MakeError(std::string("Variant '") + variant_name + "' not found in component '" + + component_name + "'."); + } + + // Cache the path string for stable pointer. + std::string cache_key = std::string(component_name) + "/" + variant_name; + auto& cached = const_cast(context)->impl.folder_path_strings_cache[cache_key]; + if (cached.empty()) { + cached = variant->folder_path.string(); + } + *out_path = cached.c_str(); + return nullptr; +} + +ModelPackageStatus* ModelPackage_GetVariantEpName( + const ModelPackageContext* context, + const char* component_name, + const char* variant_name, + const char** out_ep) { + RETURN_IF_NULL(context, "context"); + RETURN_IF_NULL(component_name, "component_name"); + RETURN_IF_NULL(variant_name, "variant_name"); + + const auto* variant = context->impl.FindVariant(component_name, variant_name); + if (!variant) { + return MakeError(std::string("Variant '") + variant_name + "' not found in component '" + + component_name + "'."); + } + + if (out_ep) { + if (variant->ep_compatibility.ep.has_value()) { + *out_ep = variant->ep_compatibility.ep->c_str(); + } else { + *out_ep = nullptr; + } + } + return nullptr; +} + +} // extern "C" diff --git a/model_package/src/model_package_internal.h b/model_package/src/model_package_internal.h new file mode 100644 index 0000000000000..8d116b78f5880 --- /dev/null +++ b/model_package/src/model_package_internal.h @@ -0,0 +1,80 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +/// \file model_package_internal.h +/// \brief Internal C++ types for the model package library. + +#pragma once + +#include +#include +#include +#include +#include + +namespace model_package { + +// ───────────────────────────────────────────────────────────────────────────── +// Data types +// ───────────────────────────────────────────────────────────────────────────── + +/// EP compatibility declaration for a variant (opaque to this library). +struct EpCompatibility { + std::optional ep; + std::optional device; + std::optional compatibility_string; +}; + +/// A single model file within a variant. +struct VariantFile { + std::string filename; + std::filesystem::path resolved_path; + + std::optional> session_options; + std::optional> provider_options; + std::optional> shared_files; +}; + +/// A variant of a component. +struct Variant { + std::string name; + std::filesystem::path folder_path; + // Single EP compatibility entry per variant (from metadata.json). + EpCompatibility ep_compatibility; + // Single model file entry (from variant.json). Empty when variant.json is absent. + std::optional file; + std::optional consumer_metadata_json; +}; + +/// A component in the model package. +struct Component { + std::string name; + std::vector variants; +}; + +/// Top-level model package descriptor. +struct PackageInfo { + int64_t schema_version{}; + std::filesystem::path root_path; + std::vector components; +}; + +// ───────────────────────────────────────────────────────────────────────────── +// Context implementation +// ───────────────────────────────────────────────────────────────────────────── + +/// Internal context holding parsed package data and C API caches. +struct ContextImpl { + PackageInfo package_info; + + // Caches for C API string access (stable pointers). + std::vector component_names_cache; + std::unordered_map> variant_names_cache; + std::unordered_map folder_path_strings_cache; + + // Lookup helpers. + const Component* FindComponent(const char* name) const; + const Variant* FindVariant(const char* component_name, const char* variant_name) const; +}; + +} // namespace model_package diff --git a/model_package/src/parser.cc b/model_package/src/parser.cc new file mode 100644 index 0000000000000..70d95b0297e38 --- /dev/null +++ b/model_package/src/parser.cc @@ -0,0 +1,595 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "parser.h" + +#include +#include +#include +#include +#include +#include + +#include "nlohmann/json.hpp" + +using json = nlohmann::json; + +namespace model_package { +namespace { + +// ───────────────────────────────────────────────────────────────────────────── +// JSON key constants +// ───────────────────────────────────────────────────────────────────────────── + +constexpr const char* kManifestFileName = "manifest.json"; +constexpr const char* kMetadataFileName = "metadata.json"; +constexpr const char* kVariantDescriptorFileName = "variant.json"; + +constexpr const char* kSchemaVersionKey = "schema_version"; +constexpr const char* kComponentsKey = "components"; +constexpr const char* kComponentNameKey = "component_name"; +constexpr const char* kVariantsKey = "variants"; + +constexpr const char* kEpKey = "ep"; +constexpr const char* kDeviceKey = "device"; +constexpr const char* kCompatibilityStringKey = "compatibility_string"; + +constexpr const char* kFilenameKey = "filename"; +constexpr const char* kSessionOptionsKey = "session_options"; +constexpr const char* kProviderOptionsKey = "provider_options"; +constexpr const char* kSharedFilesKey = "shared_files"; +constexpr const char* kConsumerMetadataKey = "consumer_metadata"; + +// ───────────────────────────────────────────────────────────────────────────── +// Internal schema types for deserialization +// ───────────────────────────────────────────────────────────────────────────── + +struct VariantMetadataSchema { + std::string filename; + std::optional> session_options; + std::optional> provider_options; + std::optional> shared_files; +}; + +struct EpCompatibilitySchema { + std::optional ep; + std::optional device; + std::optional compatibility_string; +}; + +struct VariantSchema { + EpCompatibilitySchema ep_info; +}; + +struct ComponentSchema { + std::optional component_name; + std::unordered_map variants; +}; + +struct ManifestSchema { + int64_t schema_version; + std::optional> components; +}; + +// ───────────────────────────────────────────────────────────────────────────── +// JSON helpers +// ───────────────────────────────────────────────────────────────────────────── + +std::string JsonScalarToString(const json& v, const char* key_name, const std::string& parent_key) { + if (v.is_string()) return v.get(); + if (v.is_number_integer()) return std::to_string(v.get()); + if (v.is_number_unsigned()) return std::to_string(v.get()); + if (v.is_number_float()) return v.dump(); + if (v.is_boolean()) return v.get() ? "true" : "false"; + + throw std::invalid_argument( + std::string("\"") + key_name + "\" under '" + parent_key + + "' must contain scalar (string/number/bool) values."); +} + +std::optional> ParseFlatOptionsObject( + const json& j, const char* key_name) { + if (!j.contains(key_name) || j[key_name].is_null()) { + return std::nullopt; + } + + const auto& obj = j[key_name]; + if (!obj.is_object()) { + throw std::invalid_argument(std::string("\"") + key_name + "\" must be an object."); + } + + std::unordered_map result; + result.reserve(obj.size()); + + for (auto it = obj.begin(); it != obj.end(); ++it) { + result.emplace(it.key(), JsonScalarToString(it.value(), key_name, it.key())); + } + + return result; +} + +std::optional ParseOptionalString(const json& j, const char* key_name) { + if (!j.contains(key_name) || j[key_name].is_null()) { + return std::nullopt; + } + + const auto& value = j[key_name]; + if (!value.is_string()) { + throw std::invalid_argument(std::string("\"") + key_name + "\" must be a string."); + } + return value.get(); +} + +// ───────────────────────────────────────────────────────────────────────────── +// nlohmann from_json overloads +// ───────────────────────────────────────────────────────────────────────────── + +void from_json(const json& j, EpCompatibilitySchema& c) { + if (!j.contains(kEpKey) || j[kEpKey].is_null()) { + throw std::invalid_argument(std::string("\"") + kEpKey + "\" is required in each ep_compatibility entry."); + } + if (!j[kEpKey].is_string()) { + throw std::invalid_argument(std::string("\"") + kEpKey + "\" must be a string."); + } + c.ep = j[kEpKey].get(); + if (c.ep->empty()) { + throw std::invalid_argument(std::string("\"") + kEpKey + "\" must be a non-empty string."); + } + + if (j.contains(kDeviceKey) && !j[kDeviceKey].is_null()) { + if (!j[kDeviceKey].is_string()) { + throw std::invalid_argument(std::string("\"") + kDeviceKey + "\" must be a string when present."); + } + c.device = j[kDeviceKey].get(); + } + c.compatibility_string = ParseOptionalString(j, kCompatibilityStringKey); +} + +void from_json(const json& j, VariantSchema& v) { + // EP fields (ep, device, compatibility_string) are now directly on the variant object. + // "ep" is required. + v.ep_info = j.get(); +} + +void from_json(const json& j, VariantMetadataSchema& v) { + v.filename = j.at(kFilenameKey).get(); + v.session_options = ParseFlatOptionsObject(j, kSessionOptionsKey); + v.provider_options = ParseFlatOptionsObject(j, kProviderOptionsKey); + v.shared_files = ParseFlatOptionsObject(j, kSharedFilesKey); +} + +void from_json(const json& j, ManifestSchema& m) { + m.schema_version = j.at(kSchemaVersionKey).get(); + + if (j.contains(kComponentsKey)) { + if (!j[kComponentsKey].is_array()) { + throw std::invalid_argument(std::string("\"") + kComponentsKey + "\" must be an array of strings"); + } + m.components = j[kComponentsKey].get>(); + } +} + +void from_json(const json& j, ComponentSchema& m) { + if (j.contains(kComponentNameKey) && j[kComponentNameKey].is_string()) { + m.component_name = j[kComponentNameKey].get(); + } + + m.variants = j.at(kVariantsKey).get>(); +} + +// ───────────────────────────────────────────────────────────────────────────── +// Parsing variants in declaration order (from the JSON object) +// ───────────────────────────────────────────────────────────────────────────── + +std::vector> ParseVariantsInOrder(const json& variants_obj) { + std::vector> result; + result.reserve(variants_obj.size()); + for (auto it = variants_obj.begin(); it != variants_obj.end(); ++it) { + result.emplace_back(it.key(), it.value().get()); + } + return result; +} + +// ───────────────────────────────────────────────────────────────────────────── +// Path validation +// ───────────────────────────────────────────────────────────────────────────── + +bool ValidatePathSegment(const std::string& segment, const char* segment_type, std::string& error) { + if (segment.empty()) { + error = std::string(segment_type) + " must not be empty."; + return false; + } + + if (std::filesystem::path(segment).is_absolute()) { + error = std::string(segment_type) + " must not be an absolute path: '" + segment + "'."; + return false; + } + + for (const auto& part : std::filesystem::path(segment)) { + if (part == "..") { + error = std::string(segment_type) + " must not contain '..' path components: '" + segment + "'."; + return false; + } + } + + return true; +} + +bool ValidatePathConfinement(const std::filesystem::path& resolved_path, + const std::filesystem::path& root, + const char* description, + std::string& error) { + auto normal_root = root.lexically_normal(); + auto normal_path = resolved_path.lexically_normal(); + + auto root_str = normal_root.string(); + auto path_str = normal_path.string(); + + if (path_str.size() < root_str.size() || + path_str.compare(0, root_str.size(), root_str) != 0 || + (path_str.size() > root_str.size() && path_str[root_str.size()] != std::filesystem::path::preferred_separator +#ifndef _WIN32 + && path_str[root_str.size()] != '/' +#endif + )) { + error = std::string(description) + " resolves outside the package root. Path: '" + + resolved_path.string() + "', Root: '" + root.string() + "'."; + return false; + } + + return true; +} + +// ───────────────────────────────────────────────────────────────────────────── +// Find single ONNX file in directory +// ───────────────────────────────────────────────────────────────────────────── + +bool FindSingleOnnxFile(const std::filesystem::path& search_dir, + std::filesystem::path& resolved_path, + std::string& error) { + std::vector onnx_files; + for (const auto& entry : std::filesystem::directory_iterator(search_dir)) { + if (!entry.is_regular_file()) continue; + + std::string ext = entry.path().extension().string(); + std::transform(ext.begin(), ext.end(), ext.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + if (ext == ".onnx") { + onnx_files.push_back(entry.path()); + } + } + + if (onnx_files.empty()) { + error = "No ONNX model file found under " + search_dir.string(); + return false; + } + + if (onnx_files.size() > 1) { + error = "Multiple ONNX model files found under " + search_dir.string() + + ". Multiple ONNX files per variant are not supported yet."; + return false; + } + + resolved_path = onnx_files.front(); + return true; +} + +// ───────────────────────────────────────────────────────────────────────────── +// Parse variants from a single component +// ───────────────────────────────────────────────────────────────────────────── + +bool ParseVariantsFromComponent(const std::string& component_name, + const std::filesystem::path& component_root, + const json* variants_obj, + std::vector& out_variants, + std::string& error) { + if (variants_obj == nullptr) { + error = "Missing metadata variants for component: " + component_name; + return false; + } + + std::vector> variants; + try { + variants = ParseVariantsInOrder(*variants_obj); + } catch (const std::exception& ex) { + error = "Invalid metadata variant schema for component '" + component_name + "': " + ex.what(); + return false; + } + + for (const auto& [variant_name, variant_schema] : variants) { + if (!ValidatePathSegment(variant_name, "Variant name", error)) return false; + + const std::filesystem::path variant_root = component_root / variant_name; + if (!ValidatePathConfinement(variant_root, component_root, "Variant directory", error)) return false; + + const std::filesystem::path variant_descriptor_path = variant_root / kVariantDescriptorFileName; + + Variant variant_info{}; + variant_info.name = variant_name; + variant_info.folder_path = variant_root; + + // variant.json is optional. If present, it declares the file list, + // per-file session/provider options, and consumer metadata. + if (std::filesystem::exists(variant_descriptor_path)) { + std::ifstream vf(variant_descriptor_path, std::ios::binary); + if (!vf) { + error = "Failed to open variant.json at " + variant_descriptor_path.string(); + return false; + } + + json variant_doc; + try { + variant_doc = json::parse(vf); + } catch (const std::exception& ex) { + error = "variant.json at " + variant_descriptor_path.string() + " is not valid JSON: " + ex.what(); + return false; + } + + VariantMetadataSchema variant_metadata; + try { + variant_metadata = variant_doc.get(); + } catch (const std::exception& ex) { + error = "variant.json at " + variant_descriptor_path.string() + " has invalid schema: " + ex.what(); + return false; + } + + // consumer_metadata is a top-level optional field parsed separately from the schema struct. + if (variant_doc.contains(kConsumerMetadataKey) && variant_doc[kConsumerMetadataKey].is_object()) { + variant_info.consumer_metadata_json = variant_doc[kConsumerMetadataKey].dump(); + } + + if (!ValidatePathSegment(variant_metadata.filename, "File name", error)) return false; + + const std::filesystem::path candidate_path = variant_root / variant_metadata.filename; + if (!ValidatePathConfinement(candidate_path, variant_root, "Variant file path", error)) return false; + + if (!std::filesystem::exists(candidate_path)) { + error = "Variant '" + variant_name + "', file '" + variant_metadata.filename + + "' path does not exist: " + candidate_path.string(); + return false; + } + + std::filesystem::path resolved_model_path; + if (std::filesystem::is_regular_file(candidate_path)) { + resolved_model_path = candidate_path; + } else if (std::filesystem::is_directory(candidate_path)) { + if (!FindSingleOnnxFile(candidate_path, resolved_model_path, error)) return false; + } else { + error = "Variant '" + variant_name + "', file '" + variant_metadata.filename + + "' path is neither a file nor directory: " + candidate_path.string(); + return false; + } + + VariantFile file_info{}; + file_info.filename = variant_metadata.filename; + file_info.resolved_path = std::move(resolved_model_path); + file_info.session_options = variant_metadata.session_options; + file_info.provider_options = variant_metadata.provider_options; + file_info.shared_files = variant_metadata.shared_files; + + variant_info.file = std::move(file_info); + } + + // EP compatibility from metadata.json (single entry per variant) + variant_info.ep_compatibility.ep = variant_schema.ep_info.ep; + variant_info.ep_compatibility.device = variant_schema.ep_info.device; + variant_info.ep_compatibility.compatibility_string = variant_schema.ep_info.compatibility_string; + + out_variants.push_back(std::move(variant_info)); + } + + return true; +} + +} // namespace + +// ───────────────────────────────────────────────────────────────────────────── +// Public parser entry point +// ───────────────────────────────────────────────────────────────────────────── + +bool ParsePackage(const std::filesystem::path& package_root, + PackageInfo& out_package, + std::string& out_error) { + out_package = {}; + out_package.root_path = package_root; + + // Check for single-component mode: metadata.json at root + const auto root_metadata_path = package_root / kMetadataFileName; + if (std::filesystem::exists(root_metadata_path) && + std::filesystem::is_regular_file(root_metadata_path)) { + std::ifstream mf(root_metadata_path, std::ios::binary); + if (!mf) { + out_error = "Failed to open metadata.json at " + root_metadata_path.string(); + return false; + } + + json metadata_doc; + try { + metadata_doc = json::parse(mf); + } catch (const std::exception& ex) { + out_error = "metadata.json at " + root_metadata_path.string() + " is not valid JSON: " + ex.what(); + return false; + } + + ComponentSchema metadata_schema; + try { + metadata_schema = metadata_doc.get(); + } catch (const std::exception& ex) { + out_error = "metadata.json at " + root_metadata_path.string() + " has invalid schema: " + ex.what(); + return false; + } + + const std::string component_name = + metadata_schema.component_name.has_value() + ? *metadata_schema.component_name + : package_root.filename().string(); + + const json* variants_obj = &metadata_doc.at(kVariantsKey); + + Component component{}; + component.name = component_name; + + if (!ParseVariantsFromComponent(component_name, package_root, variants_obj, + component.variants, out_error)) { + return false; + } + + out_package.schema_version = 0; // Single-component mode doesn't have a manifest + out_package.components.push_back(std::move(component)); + return true; + } + + // Multi-component mode: manifest.json at root + const auto manifest_path = package_root / kManifestFileName; + if (!std::filesystem::exists(manifest_path)) { + out_error = "No manifest.json found at " + manifest_path.string(); + return false; + } + + std::ifstream f(manifest_path, std::ios::binary); + if (!f) { + out_error = "Failed to open manifest.json at " + manifest_path.string(); + return false; + } + + json doc; + try { + doc = json::parse(f); + } catch (const std::exception& ex) { + out_error = std::string("manifest.json is not valid JSON: ") + ex.what(); + return false; + } + + ManifestSchema manifest_schema; + try { + manifest_schema = doc.get(); + } catch (const std::exception& ex) { + out_error = std::string("manifest.json has invalid schema: ") + ex.what(); + return false; + } + + if (manifest_schema.schema_version != 1) { + out_error = "Unsupported schema_version in manifest.json: " + + std::to_string(manifest_schema.schema_version) + ". Expected 1."; + return false; + } + + out_package.schema_version = manifest_schema.schema_version; + + const bool has_components = manifest_schema.components.has_value(); + std::vector component_names; + std::unordered_map discovered_metadata_docs; + + if (has_components) { + component_names = *manifest_schema.components; + } else { + const auto models_dir = package_root / "models"; + if (!std::filesystem::exists(models_dir) || !std::filesystem::is_directory(models_dir)) { + out_error = "manifest.json missing \"components\" and no discoverable models directory at " + + models_dir.string(); + return false; + } + + for (const auto& entry : std::filesystem::directory_iterator(models_dir)) { + if (!entry.is_directory()) continue; + + const auto name = entry.path().filename().string(); + const auto metadata_path = entry.path() / kMetadataFileName; + if (!std::filesystem::exists(metadata_path)) continue; + + std::ifstream mf(metadata_path, std::ios::binary); + if (!mf) { + out_error = "Failed to open metadata.json at " + metadata_path.string(); + return false; + } + + json metadata_doc; + try { + metadata_doc = json::parse(mf); + (void)metadata_doc.get(); + } catch (const std::exception& ex) { + out_error = "metadata.json at " + metadata_path.string() + + " has invalid schema: " + std::string(ex.what()); + return false; + } + + discovered_metadata_docs.emplace(name, std::move(metadata_doc)); + component_names.push_back(name); + } + + if (component_names.empty()) { + out_error = + "manifest.json missing \"components\" and no component model folders with " + "metadata.json were found under " + + models_dir.string(); + return false; + } + } + + for (const auto& component_name : component_names) { + if (!ValidatePathSegment(component_name, "Component name", out_error)) return false; + + const auto component_root = package_root / "models" / component_name; + if (!ValidatePathConfinement(component_root, package_root, "Component directory", out_error)) return false; + + if (has_components && + (!std::filesystem::exists(component_root) || !std::filesystem::is_directory(component_root))) { + // Skip missing component directories (just warn — standalone library doesn't have logging, + // so we skip silently for now). + continue; + } + + json metadata_doc; + const json* variants_obj = nullptr; + const auto metadata_path = component_root / kMetadataFileName; + + if (!has_components) { + auto it_meta = discovered_metadata_docs.find(component_name); + if (it_meta != discovered_metadata_docs.end()) { + metadata_doc = it_meta->second; + variants_obj = &metadata_doc.at(kVariantsKey); + } + } else if (std::filesystem::exists(metadata_path)) { + std::ifstream mf(metadata_path, std::ios::binary); + if (mf) { + try { + metadata_doc = json::parse(mf); + (void)metadata_doc.get(); + variants_obj = &metadata_doc.at(kVariantsKey); + } catch (const std::exception&) { + // Ignore parse errors, fall through. + } + } + } + + if (!metadata_doc.is_null() && + metadata_doc.contains(kComponentNameKey) && + metadata_doc[kComponentNameKey].is_string()) { + const auto metadata_component_name = metadata_doc[kComponentNameKey].get(); + if (metadata_component_name != component_name) { + out_error = "metadata.json component_name '" + metadata_component_name + + "' does not match directory/manifest component name '" + component_name + "'."; + return false; + } + } + + Component component{}; + component.name = component_name; + + if (!ParseVariantsFromComponent(component_name, component_root, variants_obj, + component.variants, out_error)) { + return false; + } + + out_package.components.push_back(std::move(component)); + } + + if (out_package.components.empty()) { + out_error = "No valid component models were found under " + (package_root / "models").string(); + return false; + } + + return true; +} + +} // namespace model_package diff --git a/model_package/src/parser.h b/model_package/src/parser.h new file mode 100644 index 0000000000000..ed3d22cb29d36 --- /dev/null +++ b/model_package/src/parser.h @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +/// \file parser.h +/// \brief Model package JSON parser (internal). + +#pragma once + +#include +#include + +#include "model_package_internal.h" + +namespace model_package { + +/// Parse a model package from a directory. +/// Reads manifest.json, metadata.json per component, variant.json per variant. +/// +/// \param[in] package_root Path to the model package root directory. +/// \param[out] out_package On success, filled with the parsed package info. +/// \param[out] out_error On failure, filled with an error message. +/// \return true on success, false on error. +bool ParsePackage(const std::filesystem::path& package_root, + PackageInfo& out_package, + std::string& out_error); + +} // namespace model_package diff --git a/onnxruntime/__init__.py b/onnxruntime/__init__.py index 16b33baaf1b7f..9041efe63f886 100644 --- a/onnxruntime/__init__.py +++ b/onnxruntime/__init__.py @@ -29,6 +29,9 @@ GraphOptimizationLevel, # noqa: F401 LoraAdapter, # noqa: F401 ModelMetadata, # noqa: F401 + ModelPackageComponentContext, # noqa: F401 + ModelPackageContext, # noqa: F401 + ModelPackageOptions, # noqa: F401 NodeArg, # noqa: F401 OrtAllocatorType, # noqa: F401 OrtArenaCfg, # noqa: F401 @@ -147,7 +150,7 @@ def _extract_cuda_major_version(version_str: str) -> str: Returns: Major version as string, or "12" if parsing fails """ - return version_str.split(".")[0] if version_str else "12" + return version_str.split(".", maxsplit=1)[0] if version_str else "12" def _get_cufft_version(cuda_major: str) -> str: diff --git a/onnxruntime/core/session/abi_session_options.cc b/onnxruntime/core/session/abi_session_options.cc index 06bd5c4d84089..3d2d61d409afa 100644 --- a/onnxruntime/core/session/abi_session_options.cc +++ b/onnxruntime/core/session/abi_session_options.cc @@ -2,10 +2,16 @@ // Licensed under the MIT License. #include +#include +#include #include +#include +#include #include +#include #include "core/common/inlined_containers.h" +#include "core/common/path_string.h" #include "core/framework/error_code_helper.h" #include "core/graph/onnx_protobuf.h" #include "core/session/abi_session_options_impl.h" @@ -13,6 +19,7 @@ #include "core/session/onnxruntime_c_api.h" #include "core/session/ort_apis.h" #include "core/session/utils.h" +#include "core/session/onnxruntime_session_options_config_keys.h" OrtSessionOptions::~OrtSessionOptions() = default; @@ -264,9 +271,229 @@ ORT_API_STATUS_IMPL(OrtApis::DisablePerSessionThreads, _In_ OrtSessionOptions* o return nullptr; } +namespace { + +// Dispatcher for OrtApis::AddSessionConfigEntry. Maps a fixed set of well-known string keys to the +// corresponding dedicated OrtSessionOptions setter, parsing the value into the appropriate type. +// Any key not in the table falls through to config_options.AddConfigEntry. +// +// See OrtApi::AddSessionConfigEntry documentation for key/value rules. + +enum class SessionOptionSetterKind { + kIntraOpNumThreads, + kInterOpNumThreads, + kEnableCpuMemArena, + kEnableMemPattern, + kSessionLogId, + kLogSeverityLevel, + kLogVerbosityLevel, + kEnableProfiling, + kGraphOptimizationLevel, + kOptimizedModelFilePath, + kExecutionMode, + kUsePerSessionThreads, + kUseDeterministicCompute, +}; + +struct KnownSessionOption { + const char* key; + SessionOptionSetterKind kind; +}; + +static const KnownSessionOption kKnownSessionOptions[] = { + {kOrtSessionOptionsConfigIntraOpNumThreads, SessionOptionSetterKind::kIntraOpNumThreads}, + {kOrtSessionOptionsConfigInterOpNumThreads, SessionOptionSetterKind::kInterOpNumThreads}, + {kOrtSessionOptionsConfigEnableCpuMemArena, SessionOptionSetterKind::kEnableCpuMemArena}, + {kOrtSessionOptionsConfigEnableMemPattern, SessionOptionSetterKind::kEnableMemPattern}, + {kOrtSessionOptionsConfigLogId, SessionOptionSetterKind::kSessionLogId}, + {kOrtSessionOptionsConfigLogSeverityLevel, SessionOptionSetterKind::kLogSeverityLevel}, + {kOrtSessionOptionsConfigLogVerbosityLevel, SessionOptionSetterKind::kLogVerbosityLevel}, + {kOrtSessionOptionsConfigEnableProfiling, SessionOptionSetterKind::kEnableProfiling}, + {kOrtSessionOptionsConfigGraphOptimizationLevel, SessionOptionSetterKind::kGraphOptimizationLevel}, + {kOrtSessionOptionsConfigOptimizedModelFilePath, SessionOptionSetterKind::kOptimizedModelFilePath}, + {kOrtSessionOptionsConfigExecutionMode, SessionOptionSetterKind::kExecutionMode}, + {kOrtSessionOptionsConfigUsePerSessionThreads, SessionOptionSetterKind::kUsePerSessionThreads}, + {kOrtSessionOptionsConfigUseDeterministicCompute, SessionOptionSetterKind::kUseDeterministicCompute}, +}; + +bool LookupKnownSessionOption(const char* key, SessionOptionSetterKind& out) { + for (const auto& entry : kKnownSessionOptions) { + if (std::strcmp(entry.key, key) == 0) { + out = entry.kind; + return true; + } + } + return false; +} + +std::string ToLowerAscii(std::string_view s) { + std::string out(s); + std::transform(out.begin(), out.end(), out.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + return out; +} + +OrtStatus* ParseBool(const char* key, const char* value, bool& out) { + const std::string v = ToLowerAscii(value); + if (v == "1" || v == "true") { + out = true; + return nullptr; + } + if (v == "0" || v == "false") { + out = false; + return nullptr; + } + std::ostringstream oss; + oss << "AddSessionConfigEntry: value for '" << key + << "' must be a boolean ('0'/'1' or 'true'/'false'); got '" << value << "'"; + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, oss.str().c_str()); +} + +OrtStatus* ParseInt(const char* key, const char* value, int& out) { + if (value == nullptr || *value == '\0') { + std::ostringstream oss; + oss << "AddSessionConfigEntry: value for '" << key << "' must be an integer; got empty string"; + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, oss.str().c_str()); + } + char* end = nullptr; + errno = 0; + long parsed = std::strtol(value, &end, 10); + if (errno != 0 || end == value || *end != '\0' || + parsed < std::numeric_limits::min() || parsed > std::numeric_limits::max()) { + std::ostringstream oss; + oss << "AddSessionConfigEntry: value for '" << key << "' must be a base-10 int32; got '" << value << "'"; + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, oss.str().c_str()); + } + out = static_cast(parsed); + return nullptr; +} + +OrtStatus* ParseGraphOptLevel(const char* value, GraphOptimizationLevel& out) { + const std::string v = ToLowerAscii(value); + if (v == "disable_all") { + out = ORT_DISABLE_ALL; + } else if (v == "enable_basic") { + out = ORT_ENABLE_BASIC; + } else if (v == "enable_extended") { + out = ORT_ENABLE_EXTENDED; + } else if (v == "enable_layout") { + out = ORT_ENABLE_LAYOUT; + } else if (v == "enable_all") { + out = ORT_ENABLE_ALL; + } else { + std::ostringstream oss; + oss << "AddSessionConfigEntry: value for 'graph_optimization_level' must be one of " + << "disable_all/enable_basic/enable_extended/enable_layout/enable_all; got '" << value << "'"; + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, oss.str().c_str()); + } + return nullptr; +} + +OrtStatus* ParseExecutionMode(const char* value, ExecutionMode& out) { + const std::string v = ToLowerAscii(value); + if (v == "sequential" || v == "ort_sequential") { + out = ORT_SEQUENTIAL; + } else if (v == "parallel" || v == "ort_parallel") { + out = ORT_PARALLEL; + } else { + std::ostringstream oss; + oss << "AddSessionConfigEntry: value for 'execution_mode' must be 'sequential' or 'parallel'; got '" << value << "'"; + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, oss.str().c_str()); + } + return nullptr; +} + +} // namespace + ORT_API_STATUS_IMPL(OrtApis::AddSessionConfigEntry, _Inout_ OrtSessionOptions* options, _In_z_ const char* config_key, _In_z_ const char* config_value) { - return onnxruntime::ToOrtStatus(options->value.config_options.AddConfigEntry(config_key, config_value)); + API_IMPL_BEGIN + if (options == nullptr || config_key == nullptr || config_value == nullptr) { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, + "AddSessionConfigEntry: options, config_key, and config_value must be non-null"); + } + + SessionOptionSetterKind kind; + if (!LookupKnownSessionOption(config_key, kind)) { + return onnxruntime::ToOrtStatus(options->value.config_options.AddConfigEntry(config_key, config_value)); + } + + switch (kind) { + case SessionOptionSetterKind::kIntraOpNumThreads: { + int v = 0; + if (auto* st = ParseInt(config_key, config_value, v); st != nullptr) return st; + return OrtApis::SetIntraOpNumThreads(options, v); + } + case SessionOptionSetterKind::kInterOpNumThreads: { + int v = 0; + if (auto* st = ParseInt(config_key, config_value, v); st != nullptr) return st; + return OrtApis::SetInterOpNumThreads(options, v); + } + case SessionOptionSetterKind::kEnableCpuMemArena: { + bool v = false; + if (auto* st = ParseBool(config_key, config_value, v); st != nullptr) return st; + return v ? OrtApis::EnableCpuMemArena(options) : OrtApis::DisableCpuMemArena(options); + } + case SessionOptionSetterKind::kEnableMemPattern: { + bool v = false; + if (auto* st = ParseBool(config_key, config_value, v); st != nullptr) return st; + return v ? OrtApis::EnableMemPattern(options) : OrtApis::DisableMemPattern(options); + } + case SessionOptionSetterKind::kSessionLogId: + return OrtApis::SetSessionLogId(options, config_value); + case SessionOptionSetterKind::kLogSeverityLevel: { + int v = 0; + if (auto* st = ParseInt(config_key, config_value, v); st != nullptr) return st; + return OrtApis::SetSessionLogSeverityLevel(options, v); + } + case SessionOptionSetterKind::kLogVerbosityLevel: { + int v = 0; + if (auto* st = ParseInt(config_key, config_value, v); st != nullptr) return st; + return OrtApis::SetSessionLogVerbosityLevel(options, v); + } + case SessionOptionSetterKind::kEnableProfiling: { + if (config_value[0] == '\0') { + return OrtApis::DisableProfiling(options); + } + const auto prefix = onnxruntime::ToPathString(std::string_view(config_value)); + return OrtApis::EnableProfiling(options, prefix.c_str()); + } + case SessionOptionSetterKind::kGraphOptimizationLevel: { + GraphOptimizationLevel v = ORT_ENABLE_ALL; + if (auto* st = ParseGraphOptLevel(config_value, v); st != nullptr) return st; + return OrtApis::SetSessionGraphOptimizationLevel(options, v); + } + case SessionOptionSetterKind::kOptimizedModelFilePath: { + const auto path = onnxruntime::ToPathString(std::string_view(config_value)); + return OrtApis::SetOptimizedModelFilePath(options, path.c_str()); + } + case SessionOptionSetterKind::kExecutionMode: { + ExecutionMode v = ORT_SEQUENTIAL; + if (auto* st = ParseExecutionMode(config_value, v); st != nullptr) return st; + return OrtApis::SetSessionExecutionMode(options, v); + } + case SessionOptionSetterKind::kUsePerSessionThreads: { + bool v = false; + if (auto* st = ParseBool(config_key, config_value, v); st != nullptr) return st; + if (v) { + if (!options->value.use_per_session_threads) { + return OrtApis::CreateStatus( + ORT_INVALID_ARGUMENT, + "AddSessionConfigEntry: cannot set 'use_per_session_threads' back to true after it was disabled"); + } + return nullptr; + } + return OrtApis::DisablePerSessionThreads(options); + } + case SessionOptionSetterKind::kUseDeterministicCompute: { + bool v = false; + if (auto* st = ParseBool(config_key, config_value, v); st != nullptr) return st; + return OrtApis::SetDeterministicCompute(options, v); + } + } + + return OrtApis::CreateStatus(ORT_FAIL, "AddSessionConfigEntry: internal dispatch error"); + API_IMPL_END } ORT_API_STATUS_IMPL(OrtApis::HasSessionConfigEntry, _In_ const OrtSessionOptions* options, diff --git a/onnxruntime/core/session/model_package/README.md b/onnxruntime/core/session/model_package/README.md deleted file mode 100644 index 5b44f1d05d827..0000000000000 --- a/onnxruntime/core/session/model_package/README.md +++ /dev/null @@ -1,134 +0,0 @@ -# Model Package Format - -This document describes the model package directory layout and the JSON files used by ONNX Runtime to discover and load model packages. All JSON files must be UTF-8 encoded. - -## Definitions - -- Model Package - - - A model package defines the overall logical ‘model’ - - A model package contains one or more ‘component models’ - - The component models are executed when running the model package to provide the overall functionality of the logical model - - A model package may contain configuration information to support running multiple component models - -- Component Model - - A component model comprises one or more ‘model variants’ - - All variants have the same model inputs and outputs with the same shapes. - - The data types may vary. - -- Model Variant - - A ‘model variant’ is a single ONNX or ORT format model. - -## Directory layout - -```` -.ortpackage/  -├── manifest.json -├── pipeline.json -├── configs/ -| ├── genai_config.json -| └── chat_template.jinja -└── models/  -    └── model_name/  -        ├── metadata.json - | └── Contains general information on the component model, - | and specific information about each model variant - | such as data types, quantization algo, EP, etc. that - | is updated on add/remove of model variant - └── shared_weights/ (shared weights from all variants) - └── / - └── model.data - └── / - └── model.data - └── ... -        └── base model /    -            ├── model.onnx   -        └── variant A /  -            ├── optimized model.onnx (contains EPContext nodes)  -            └── [Compilation artifacts]  -        └── variant B /  -            ├── optimized model.onnx (contains EPContext nodes)  -            └── [Compilation artifacts]  -```` - - -## Notes: -- Shared weights is not yet supported, but the format allows for it in the future. - -## `manifest.json` (required) - -Location: `/manifest.json` - -Purpose: Provides the overall package identity and (optionally) lists component models available in the package. - -Schema: -- `model_name` (string, required): Logical package name. -- `model_version` (string, optional): Version of the model package. -- `component_models` (array of strings, optional): List of component model names. If this field is omitted, ONNX Runtime will discover component models by enumerating subdirectories under `models/`. If present, the names listed here must match the subdirectory names under `models/`. - -### `manifest.json` example - -```json -{ - "model_name": , - "model_version": "1.0", - "component_models": [ - , - - ] -} -``` - -## `metadata.json` (required per component model) - -Location: `/models//metadata.json` - -Purpose: Describes the variants available for a specific component model. - -Schema: -- `component_model_name` (string, required): Name of the component model. -- `model_variants` (object, required): Map of variant names to variant descriptors. - - `` (object, required): - - `model_type` (string, optional): Type of the model (e.g., `"onnx"`, `"ORT"`). If omitted, ORT will treat it as an ONNX model by default. - - `model_file` (string, optional): Path relative to the model variant directory. Can point to an ONNX model file or a directory. If it is a directory, or if `model_file` is omitted, ORT will discover the ONNX model file within that directory. - - `model_id` (string, optional): Unique identifier for the model variant. It should match a catalog value if the model comes from a catalog. If `model_id` is present, the model will be in the /`model_id`/ directory. - - `constraints` (object, required): - - `ep` (string, required (except base model)): Execution provider name (e.g., `"TensorrtExecutionProvider"`, `"QNNExecutionProvider"`, `"OpenVINOExecutionProvider"`). - - `device` (string, optional): Target device type (e.g., `"cpu"`, `"gpu"`, `"npu"`). Must match a supported `OrtHardwareDevice`. If the EPContext model can support multiple device types, this field can be omitted and EP should record supported device types in `ep_compatibility_info` instead. - - `architecture` (string, optional): Hardware architecture hint; interpreted by the EP if needed. - - `ep_compatibility_info` (string, optional): EP-specific compatibility string (as produced by `OrtEp::GetCompiledModelCompatibilityInfo()`); validated by the EP when selecting a variant. **The compatibility value returned by the EP is critical—ORT uses it to rank and choose the model variant.** - -### `metadata.json` example -```json -{ - "component_model_name": , - "model_variants": { - : { - "model_type": "onnx", - "model_file": "model_ctx.onnx", - "constraints": { - "ep": "TensorrtExecutionProvider", - "ep_compatibility_info": "..." - } - }, - : { - "model_type": "onnx", - "model_file": "model_ctx.onnx", - "constraints": { - "ep": "OpenVINOExecutionProvider", - "device": "cpu", - "ep_compatibility_info": "..." - } - } - } -} -``` - - -## Processing rules (runtime expectations) - -- ONNX Runtime reads `manifest.json` if the path passed in is the package root directory; if `component_models` is present, it uses that to determine which component models to load. If `component_models` is not present, ONNX Runtime discovers component models by enumerating subdirectories under `models/`. (In this case, ONNX Runtime expects only one component model exist in the model package.) -- ONNX Runtime reads component model's `metadata.json` and ignores `manifest.json` if the path passed in points directly to a component model directory. -- For each component model, `metadata.json` supplies the definitive list of variants and constraints. -- Variant selection is performed by matching constraints (EP, device, `ep_compatibility_info`, and optionally architecture). **The EP’s returned compatibility value (e.g., `EP_SUPPORTED_OPTIMAL`, `EP_SUPPORTED_PREFER_RECOMPILATION`) is used to score and pick the winning model variant.** -- All file paths must be relative paths; avoid absolute paths to keep packages portable diff --git a/onnxruntime/core/session/model_package/model_package_context.cc b/onnxruntime/core/session/model_package/model_package_context.cc index 5e75c1de90138..ca4adb9c877a5 100644 --- a/onnxruntime/core/session/model_package/model_package_context.cc +++ b/onnxruntime/core/session/model_package/model_package_context.cc @@ -9,291 +9,513 @@ #include #include #include +#include #include "core/common/logging/logging.h" #include "core/framework/error_code_helper.h" +#include "core/graph/constants.h" +#include "core/providers/providers.h" #include "core/session/model_package/model_package_context.h" -#include "core/session/model_package/model_package_descriptor_parser.h" +#include "core/session/model_package/model_package_options.h" +#include "core/session/model_package/model_package_variant_selector.h" +#include "core/session/ort_env.h" +#include "core/session/provider_policy_context.h" +#include "core/session/utils.h" + +// We intentionally use the standalone model_package library's internal C++ types directly +// (model_package::ParsePackage, model_package_internal.h) rather than its public C API +// (ModelPackage_* functions). This avoids double-wrapping since ORT compiles the library in-tree. +// The public C API exists for external consumers (GenAI, FL) who link independently. +#include "model_package_internal.h" +#include "parser.h" namespace onnxruntime { + namespace { -std::string ToLower(std::string_view s) { - std::string result(s); - std::transform(result.begin(), result.end(), result.begin(), - [](unsigned char c) { return std::tolower(c); }); - return result; -} -bool MatchesDevice(const OrtHardwareDevice* hd, std::string_view value) { - if (value.empty() || hd == nullptr) { - return value.empty(); +Status FillOptionCachesFromMap( + const std::optional>& options_map, + std::vector& key_cache, + std::vector& value_cache, + gsl::span& out_keys, + gsl::span& out_values) { + key_cache.clear(); + value_cache.clear(); + + if (!options_map.has_value()) { + out_keys = gsl::span{}; + out_values = gsl::span{}; + return Status::OK(); } - const std::string device_type = ToLower(value); - switch (hd->type) { - case OrtHardwareDeviceType::OrtHardwareDeviceType_CPU: - return device_type == "cpu"; - case OrtHardwareDeviceType::OrtHardwareDeviceType_GPU: - return device_type == "gpu"; - case OrtHardwareDeviceType::OrtHardwareDeviceType_NPU: - return device_type == "npu"; - default: - return false; + key_cache.reserve(options_map->size()); + value_cache.reserve(options_map->size()); + + for (const auto& kv : *options_map) { + key_cache.push_back(kv.first); + value_cache.push_back(kv.second); } + + out_keys = gsl::span(key_cache.data(), key_cache.size()); + out_values = gsl::span(value_cache.data(), value_cache.size()); + return Status::OK(); } -const OrtHardwareDevice* FindMatchingHardwareDevice(std::string_view device_constraint, - gsl::span hardware_devices) { - if (device_constraint.empty()) { - return nullptr; - } +} // namespace + +ModelPackageComponentContext::ModelPackageComponentContext(const std::string& component_name, + const ComponentInfo& component_model_info, + const ModelPackageOptions& options) + : component_model_name_(component_name), + component_model_info_(component_model_info), + owned_ep_infos_(options.EpInfos()), + execution_devices_(options.ExecutionDevices()), + devices_selected_(options.DevicesSelected()), + from_policy_(options.FromPolicy()) { + // Point the span at our owned copy. + ep_infos_ = gsl::span(owned_ep_infos_); +} - for (const auto* hd : hardware_devices) { - if (MatchesDevice(hd, device_constraint)) { - return hd; +ModelPackageComponentContext::ModelPackageComponentContext(const std::string& component_name, + const ComponentInfo& component_model_info, + gsl::span ep_infos) + : component_model_name_(component_name), + component_model_info_(component_model_info), + ep_infos_(ep_infos) { +} + +Status ModelPackageComponentContext::ResolveVariant() { + ORT_RETURN_IF(ep_infos_.empty(), + "ModelPackageComponentContext::ResolveVariant requires non-empty ep_infos " + "(from ModelPackageOptions or explicit span)."); + + return ResolveVariantImpl(ep_infos_); +} + +Status ModelPackageComponentContext::ResolveVariantImpl(gsl::span ep_infos) { + std::optional selected_variant; + VariantSelector selector; + ORT_RETURN_IF_ERROR(selector.SelectVariant(*this, ep_infos, selected_variant)); + + ORT_RETURN_IF(!selected_variant.has_value(), + "No suitable model variant found for the configured execution providers."); + + ORT_RETURN_IF(selected_variant->component_name != component_model_name_, + "Selected variant's component model name does not match context's component model name."); + + component_model_info_.selected_variant_index.reset(); + + size_t matched_variants = 0; + for (size_t i = 0; i < component_model_info_.variants.size(); ++i) { + if (component_model_info_.variants[i].variant_name == selected_variant->variant_name) { + component_model_info_.selected_variant_index = i; + component_model_info_.variants[i] = *selected_variant; + ++matched_variants; } } - return nullptr; + ORT_RETURN_IF(matched_variants == 0, + "Selected variant was not found in model package context."); + ORT_RETURN_IF(matched_variants > 1, + "Selected variant matched multiple variants; selection is ambiguous."); + + return Status::OK(); } -Status ValidateCompiledModelCompatibilityInfo(const VariantSelectionEpInfo& ep_info, - const std::string& compatibility_info, - std::vector& constraint_devices, - OrtCompiledModelCompatibility* compiled_model_compatibility) { - if (compatibility_info.empty()) { - LOGS_DEFAULT(INFO) << "No compatibility info constraint for this variant. Skip compatibility validation."; - return Status::OK(); - } +Status ModelPackageComponentContext::GetSelectedVariantFolderPath(const std::filesystem::path*& out_folder_path) const { + out_folder_path = nullptr; - auto* ep_factory = ep_info.ep_factory; - - if (ep_factory && - ep_factory->ort_version_supported >= 23 && - ep_factory->ValidateCompiledModelCompatibilityInfo != nullptr) { - auto status = ep_factory->ValidateCompiledModelCompatibilityInfo(ep_factory, - constraint_devices.data(), - constraint_devices.size(), - compatibility_info.c_str(), - compiled_model_compatibility); - ORT_RETURN_IF_ERROR(ToStatusAndRelease(status)); - } + ORT_RETURN_IF(!component_model_info_.selected_variant_index.has_value(), + "No variant selected for component: ", component_model_name_); + const size_t selected_idx = *component_model_info_.selected_variant_index; + ORT_RETURN_IF(selected_idx >= component_model_info_.variants.size(), + "Selected variant index out of range for component: ", component_model_name_); + + const auto& selected_variant = component_model_info_.variants[selected_idx]; + + folder_path_cache_ = selected_variant.folder_path; + out_folder_path = &folder_path_cache_; return Status::OK(); } -bool MatchesVariant(ModelVariantInfo& variant, const VariantSelectionEpInfo& ep_info) { - LOGS_DEFAULT(INFO) << "Checking model variant with EP constraint '" << variant.ep - << "', device constraint '" << variant.device - << "' and compatibility info constraint '" << variant.compatibility_info - << "' against EP '" << ep_info.ep_name << "' with supported devices: " - << [&ep_info]() { - std::string devices_str; - for (const auto* hd : ep_info.hardware_devices) { - if (!devices_str.empty()) { - devices_str += ", "; - } - devices_str += hd->vendor + " " + std::to_string(hd->device_id); - } - return devices_str.empty() ? "none" : devices_str; - }(); - - // 1) Check EP constraint - if (!variant.ep.empty() && variant.ep != ep_info.ep_name) { - LOGS_DEFAULT(INFO) << "Variant EP constraint '" << variant.ep << "' does not match EP name '" - << ep_info.ep_name << "'. Skip this variant."; - return false; - } +Status ModelPackageComponentContext::GetSelectedVariantFilePath(std::filesystem::path& out_path) const { + out_path.clear(); - // 2) Check device constraint - bool device_ok = variant.device.empty(); + ORT_RETURN_IF(!component_model_info_.selected_variant_index.has_value(), + "No variant selected for component: ", component_model_name_); - // For EPs that don't implement the OrtEpFactory interface, ep_info.hardware_devices will be empty and ORT won't - // have the supported device information for those EPs. - // In that case, we will skip the device constraint validation for those EPs. - if (ep_info.hardware_devices.empty()) { - device_ok = true; - } + const size_t selected_idx = *component_model_info_.selected_variant_index; + ORT_RETURN_IF(selected_idx >= component_model_info_.variants.size(), + "Selected variant index out of range for component: ", component_model_name_); - // The constraint_devices is the target device(s) and will be passed to ValidateCompiledModelCompatibilityInfo - // for compatibility validation. - std::vector constraint_devices = ep_info.hardware_devices; + const auto& selected_variant = component_model_info_.variants[selected_idx]; + ORT_RETURN_IF(!selected_variant.file.has_value(), + "Selected variant '", selected_variant.variant_name, + "' does not have a variant.json descriptor (or it lacks a 'filename' entry). " + "Component: ", + component_model_name_); - if (!device_ok) { - if (const auto* matched = FindMatchingHardwareDevice(variant.device, ep_info.hardware_devices)) { - device_ok = true; - constraint_devices = {matched}; - } - } + out_path = selected_variant.file->model_file_path; + return Status::OK(); +} - if (!device_ok) { - LOGS_DEFAULT(INFO) << "Variant device constraint '" << variant.device - << "' does not match any device supported by EP '" - << ep_info.ep_name << "'. Skip this variant."; - return false; - } +Status ModelPackageComponentContext::GetSelectedVariantInfo(const VariantInfo*& out_variant) const { + out_variant = nullptr; - // 3) Check ep_compatibility_info constraint - // - // ORT does not directly evaluate the architecture constraint. Instead, it relies on - // the ep_compatibility_info constraint, which may encode architecture information - // if needed. - // - // The ep_compatibility_info value is expected to match the EP compatibility string - // stored in the EPContext model metadata. - // (See OrtEp::GetCompiledModelCompatibilityInfo() for how this string is generated.) - // - // The EP implementation of EpFactory::ValidateCompiledModelCompatibilityInfo() - // is responsible for validating the compatibility string against the target device - // (i.e. OrtHardwareDevice), and returning the compatibility result. - auto status = ValidateCompiledModelCompatibilityInfo(ep_info, variant.compatibility_info, - constraint_devices, &variant.compiled_model_compatibility); - if (!status.IsOK()) { - LOGS_DEFAULT(WARNING) << "Failed to validate compatibility info for variant with EP constraint '" - << variant.ep << "': " << status.ToString() - << ". Simply skip this compatibility validation."; - - variant.compiled_model_compatibility = OrtCompiledModelCompatibility_EP_NOT_APPLICABLE; + if (!component_model_info_.selected_variant_index.has_value()) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, + "No variant is selected for component: ", component_model_name_); } - if (variant.compiled_model_compatibility == OrtCompiledModelCompatibility_EP_UNSUPPORTED) { - LOGS_DEFAULT(INFO) << "Variant compatibility info indicates unsupported model for EP '" << ep_info.ep_name - << "'. Compatibility info: '" << variant.compatibility_info - << "'. Skip this variant."; - return false; + const size_t selected_idx = *component_model_info_.selected_variant_index; + if (selected_idx >= component_model_info_.variants.size()) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, + "Selected variant index out of range for component: ", component_model_name_); } - LOGS_DEFAULT(INFO) << "This model variant is selected and could be used for EP '" << ep_info.ep_name << "'."; + out_variant = &component_model_info_.variants[selected_idx]; + return Status::OK(); +} + +Status ModelPackageComponentContext::GetSelectedVariantFileSessionOptions(gsl::span& out_keys, + gsl::span& out_values) const { + out_keys = {}; + out_values = {}; + + const VariantInfo* selected_variant = nullptr; + ORT_RETURN_IF_ERROR(GetSelectedVariantInfo(selected_variant)); + ORT_RETURN_IF(selected_variant == nullptr, "Selected variant is null for component: ", component_model_name_); + ORT_RETURN_IF(!selected_variant->file.has_value(), "Selected variant has no file entry."); + + // Fast path: return cached key/value vectors if both exist. + if (!session_option_keys_cache_.empty() || !session_option_values_cache_.empty()) { + ORT_RETURN_IF(session_option_keys_cache_.size() != session_option_values_cache_.size(), + "Session options key/value cache size mismatch"); + + out_keys = gsl::span(session_option_keys_cache_.data(), session_option_keys_cache_.size()); + out_values = gsl::span(session_option_values_cache_.data(), session_option_values_cache_.size()); + return Status::OK(); + } - return true; + const auto& selected_file = *selected_variant->file; + return FillOptionCachesFromMap(selected_file.session_options, + session_option_keys_cache_, + session_option_values_cache_, + out_keys, + out_values); } -} // namespace -// Calculate a score for the model variant based on its constraints and metadata. -// -// It's only used to choose the best model variant among multiple candidates that match constraints. -// Higher score means more preferred. -// -// For example: -// If one model variant/EPContext is compatible with the EP and has compatiliby value indicating optimal compatibility -// (i.e. compiled_model_compatibility == OrtCompiledModelCompatibility_EP_SUPPORTED_OPTIMAL) while another model variant/EPContext -// is also compatible with the EP but has compatibility value indicating prefer recompilation -// (i.e. compiled_model_compatibility == OrtCompiledModelCompatibility_EP_SUPPORTED_PREFER_RECOMPILATION), -// the former will have a higher score and thus be selected. -// -int ModelVariantSelector::CalculateVariantScore(const ModelVariantInfo& variant) const { - int score = 0; - - if (variant.compiled_model_compatibility == OrtCompiledModelCompatibility_EP_SUPPORTED_OPTIMAL) { - score += 100; - } else if (variant.compiled_model_compatibility == OrtCompiledModelCompatibility_EP_SUPPORTED_PREFER_RECOMPILATION) { - score += 50; +Status ModelPackageComponentContext::GetSelectedVariantFileProviderOptions(gsl::span& out_keys, + gsl::span& out_values) const { + out_keys = {}; + out_values = {}; + + const VariantInfo* selected_variant = nullptr; + ORT_RETURN_IF_ERROR(GetSelectedVariantInfo(selected_variant)); + ORT_RETURN_IF(selected_variant == nullptr, "Selected variant is null for component: ", component_model_name_); + ORT_RETURN_IF(!selected_variant->file.has_value(), "Selected variant has no file entry."); + + // Fast path: return cached key/value vectors if both exist. + if (!provider_option_keys_cache_.empty() || !provider_option_values_cache_.empty()) { + ORT_RETURN_IF(provider_option_keys_cache_.size() != provider_option_values_cache_.size(), + "Provider options key/value cache size mismatch"); + + out_keys = gsl::span(provider_option_keys_cache_.data(), provider_option_keys_cache_.size()); + out_values = gsl::span(provider_option_values_cache_.data(), provider_option_values_cache_.size()); + return Status::OK(); } - // The base model with no constraints (meaning any EP can run it) gets a base score of 0. - // Other model variants with EP constraint get a higher score, so that they will be preferred over the base model. - if (!variant.ep.empty()) { - score += 10; + const auto& selected_file = *selected_variant->file; + return FillOptionCachesFromMap(selected_file.provider_options, + provider_option_keys_cache_, + provider_option_values_cache_, + out_keys, + out_values); +} + +namespace { +void BuildPtrCache(gsl::span strings, + std::vector& ptrs_cache, + const char* const*& out_ptrs, size_t& out_count) { + ptrs_cache.clear(); + ptrs_cache.reserve(strings.size()); + for (const auto& s : strings) { + ptrs_cache.push_back(s.c_str()); } + out_count = ptrs_cache.size(); + out_ptrs = ptrs_cache.empty() ? nullptr : ptrs_cache.data(); +} +} // namespace - return score; +Status ModelPackageComponentContext::GetSelectedVariantFileSessionOptionPtrs( + const char* const*& out_keys, + const char* const*& out_values, + size_t& out_count) const { + out_keys = nullptr; + out_values = nullptr; + out_count = 0; + + gsl::span keys; + gsl::span values; + ORT_RETURN_IF_ERROR(GetSelectedVariantFileSessionOptions(keys, values)); + ORT_RETURN_IF(keys.size() != values.size(), "Session options keys/values size mismatch."); + + BuildPtrCache(keys, session_option_key_ptrs_cache_, out_keys, out_count); + size_t dummy; + BuildPtrCache(values, session_option_value_ptrs_cache_, out_values, dummy); + return Status::OK(); } -Status ModelVariantSelector::SelectVariant(const ModelPackageContext& context, - gsl::span ep_infos, - std::optional& selected_variant_path) const { - selected_variant_path.reset(); +Status ModelPackageComponentContext::GetSelectedVariantFileProviderOptionPtrs( + const char* const*& out_keys, + const char* const*& out_values, + size_t& out_count) const { + out_keys = nullptr; + out_values = nullptr; + out_count = 0; + + gsl::span keys; + gsl::span values; + ORT_RETURN_IF_ERROR(GetSelectedVariantFileProviderOptions(keys, values)); + ORT_RETURN_IF(keys.size() != values.size(), "Provider options keys/values size mismatch."); + + BuildPtrCache(keys, provider_option_key_ptrs_cache_, out_keys, out_count); + size_t dummy; + BuildPtrCache(values, provider_option_value_ptrs_cache_, out_values, dummy); + return Status::OK(); +} - // explicit local copy as this function will be modifying the variant info - // (e.g. setting compatibility validation result) during the selection process. - std::vector variants = context.GetModelVariantInfos(); +Status ModelPackageComponentContext::RebuildProviderListForSession( + const Environment& env, const OrtSessionOptions& effective_options) { + provider_list_.clear(); - if (variants.empty()) { + if (owned_ep_infos_.empty()) { return Status::OK(); } - // Only one SelectionEpInfo in `ep_infos` is supported for now. - if (ep_infos.empty()) { + const auto& ep_info = owned_ep_infos_[0]; + if (ep_info.ep_name == kCpuExecutionProvider || ep_info.ep_devices.empty()) { + // CPU is built-in; no provider to register. return Status::OK(); } - if (ep_infos.size() > 1) { - LOGS_DEFAULT(WARNING) << "Multiple EP info provided for model variant selection, but only the first one with ep name '" - << ep_infos[0].ep_name << "' will be used."; - } - const auto& ep_info = ep_infos[0]; - std::unordered_set candidate_indices_set; + std::unique_ptr provider_factory; + ORT_RETURN_IF_ERROR(CreateIExecutionProviderFactoryForEpDevices( + env, + gsl::span(ep_info.ep_devices.data(), ep_info.ep_devices.size()), + provider_factory)); + + const auto& logger = *logging::LoggingManager::DefaultLogger().ToExternal(); + provider_list_.push_back(provider_factory->CreateProvider(effective_options, logger)); + + return Status::OK(); +} + +Status ModelPackageComponentContext::GetSelectedVariantConsumerMetadata(const std::string*& out_json_str) const { + out_json_str = nullptr; + + const VariantInfo* selected_variant = nullptr; + ORT_RETURN_IF_ERROR(GetSelectedVariantInfo(selected_variant)); + ORT_RETURN_IF(selected_variant == nullptr, + "Selected variant is null for component: ", component_model_name_); - // 1) Check unconstrained variants. - for (size_t i = 0, end = variants.size(); i < end; ++i) { - const auto& c = variants[i]; - if (c.ep.empty() && c.device.empty() && c.architecture.empty() && c.compatibility_info.empty()) { - candidate_indices_set.insert(i); + if (!consumer_metadata_cache_valid_) { + if (selected_variant->consumer_metadata.has_value()) { + consumer_metadata_cache_ = selected_variant->consumer_metadata->dump(); + } else { + consumer_metadata_cache_.clear(); } + consumer_metadata_cache_valid_ = true; } - // 2) Check all variants that match the EP info. - for (size_t i = 0, end = variants.size(); i < end; ++i) { - if (candidate_indices_set.count(i) > 0) { - continue; - } - auto& c = variants[i]; - if (MatchesVariant(c, ep_info)) { - candidate_indices_set.insert(i); - } + out_json_str = &consumer_metadata_cache_; + return Status::OK(); +} + +Status ModelPackageComponentContext::GetSelectedVariantName(const std::string*& out_name) const { + out_name = nullptr; + + const VariantInfo* selected_variant = nullptr; + ORT_RETURN_IF_ERROR(GetSelectedVariantInfo(selected_variant)); + ORT_RETURN_IF(selected_variant == nullptr, + "Selected variant is null for component: ", component_model_name_); + + out_name = &selected_variant->variant_name; + return Status::OK(); +} + +ModelPackageContext::ModelPackageContext(const std::filesystem::path& package_root) { + // Use the standalone model_package library for parsing. + model_package::PackageInfo pkg_info; + std::string error; + if (!model_package::ParsePackage(package_root, pkg_info, error)) { + ORT_THROW("Failed to parse model package: ", error); } - // Log all matched candidates. - { - std::ostringstream oss; - oss << candidate_indices_set.size() << " Model variant(s) matched constraints: "; - size_t i = 0; - for (size_t idx : candidate_indices_set) { - const auto& path = variants[idx].model_path; - oss << path.string(); - if (i + 1 < candidate_indices_set.size()) { - oss << "; "; + // Convert standalone library types to ORT internal types. + model_package_info_.schema_version = pkg_info.schema_version; + model_package_info_.components.clear(); + component_name_to_index_.clear(); + + for (const auto& component : pkg_info.components) { + const auto& name = component.name; + size_t component_idx = model_package_info_.components.size(); + component_name_to_index_[name] = component_idx; + + ComponentInfo ort_component{}; + ort_component.component_name = name; + ort_component.selected_variant_index.reset(); + + for (const auto& variant : component.variants) { + VariantInfo ort_variant{}; + ort_variant.component_name = name; + ort_variant.variant_name = variant.name; + ort_variant.folder_path = variant.folder_path; + + // Convert EP compatibility (single entry per variant). + ort_variant.ep_compatibility.ep = variant.ep_compatibility.ep; + ort_variant.ep_compatibility.device = variant.ep_compatibility.device; + ort_variant.ep_compatibility.compatibility_string = variant.ep_compatibility.compatibility_string; + ort_variant.ep_compatibility.compiled_model_compatibility = OrtCompiledModelCompatibility_EP_NOT_APPLICABLE; + + // Convert file entry (single file per variant). + if (variant.file.has_value()) { + VariantModelInfo ort_file{}; + ort_file.identifier = variant.file->filename; + ort_file.model_file_path = variant.file->resolved_path; + ort_file.session_options = variant.file->session_options; + ort_file.provider_options = variant.file->provider_options; + ort_file.shared_files = variant.file->shared_files; + ort_variant.file = std::move(ort_file); } - ++i; + + // Consumer metadata. + if (variant.consumer_metadata_json.has_value()) { + ort_variant.consumer_metadata = nlohmann::json::parse(*variant.consumer_metadata_json); + } + + model_variant_infos_.push_back(ort_variant); + ort_component.variants.push_back(std::move(ort_variant)); } - LOGS_DEFAULT(INFO) << oss.str(); - } - if (candidate_indices_set.empty()) { - return Status::OK(); + model_package_info_.components.push_back(std::move(ort_component)); } - // 3) If only one candidate, select it. - if (candidate_indices_set.size() == 1) { - selected_variant_path = variants[*candidate_indices_set.begin()].model_path; - return Status::OK(); + // Create component names cache for quick lookup. + component_names_cache_.clear(); + component_names_cache_.reserve(model_package_info_.components.size()); + + for (const auto& component : model_package_info_.components) { + component_names_cache_.push_back(component.component_name); } +} - // 4) If there are multiple candidates, choose the highest-score model variant among them. - int best_score = std::numeric_limits::min(); - size_t best_index = *candidate_indices_set.begin(); +size_t ModelPackageContext::GetComponentCount() const noexcept { + return model_package_info_.components.size(); +} - for (size_t idx : candidate_indices_set) { - const auto& v = variants[idx]; - int variant_best_score = std::numeric_limits::min(); - variant_best_score = std::max(variant_best_score, CalculateVariantScore(v)); +Status ModelPackageContext::GetComponentNames(gsl::span& out_names) const { + out_names = gsl::span(component_names_cache_.data(), + component_names_cache_.size()); + return Status::OK(); +} - if (variant_best_score > best_score) { - best_score = variant_best_score; - best_index = idx; +void ModelPackageContext::GetComponentNamePtrs(const char* const*& out_ptrs, size_t& out_count) const { + if (component_name_ptrs_cache_.empty() && !component_names_cache_.empty()) { + component_name_ptrs_cache_.reserve(component_names_cache_.size()); + for (const auto& s : component_names_cache_) { + component_name_ptrs_cache_.push_back(s.c_str()); } } + out_count = component_name_ptrs_cache_.size(); + out_ptrs = component_name_ptrs_cache_.empty() ? nullptr : component_name_ptrs_cache_.data(); +} + +void ModelPackageContext::GetVariantNamePtrs(const std::string& component_name, + const char* const*& out_ptrs, size_t& out_count) const { + // Ensure variant name strings are cached first. + gsl::span variant_names; + (void)GetVariantNames(component_name, variant_names); + + auto& ptrs = variant_name_ptrs_cache_[component_name]; + ptrs.clear(); + ptrs.reserve(variant_names.size()); + for (const auto& s : variant_names) { + ptrs.push_back(s.c_str()); + } + out_count = ptrs.size(); + out_ptrs = ptrs.empty() ? nullptr : ptrs.data(); +} + +Status ModelPackageContext::GetVariantCount(const std::string& component_name, size_t& out_count) const { + out_count = 0; - selected_variant_path = variants[best_index].model_path; + auto it = component_name_to_index_.find(component_name); + if (it == component_name_to_index_.end()) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Component model not found: ", component_name); + } + out_count = model_package_info_.components[it->second].variants.size(); return Status::OK(); } -ModelPackageContext::ModelPackageContext(const std::filesystem::path& package_root) { - ModelPackageDescriptorParser parser(logging::LoggingManager::DefaultLogger()); - ORT_THROW_IF_ERROR(parser.ParseVariantsFromRoot(package_root, model_variant_infos_)); +Status ModelPackageContext::GetVariantNames(const std::string& component_name, + gsl::span& out_variant_names) const { + out_variant_names = gsl::span{}; + + auto it = component_name_to_index_.find(component_name); + if (it == component_name_to_index_.end()) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Component model not found: ", component_name); + } + + const auto& variants = model_package_info_.components[it->second].variants; + component_to_variant_names_cache_[component_name].clear(); + component_to_variant_names_cache_[component_name].reserve(variants.size()); + + for (const auto& variant : variants) { + component_to_variant_names_cache_[component_name].push_back(variant.variant_name); + } + + out_variant_names = gsl::span(component_to_variant_names_cache_[component_name].data(), + component_to_variant_names_cache_[component_name].size()); + return Status::OK(); } -} // namespace onnxruntime +namespace { +Status FindVariant(const ModelPackageInfo& model_package_info, + const std::unordered_map& component_name_to_index, + const std::string& component_name, + const std::string& variant_name, + const VariantInfo*& out_variant) { + out_variant = nullptr; + auto it = component_name_to_index.find(component_name); + if (it == component_name_to_index.end()) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Component model not found: ", component_name); + } + + const auto& variants = model_package_info.components[it->second].variants; + for (const auto& v : variants) { + if (v.variant_name == variant_name) { + out_variant = &v; + return Status::OK(); + } + } + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, + "Variant '", variant_name, "' not found in component '", component_name, "'."); +} +} // namespace + +Status ModelPackageContext::GetVariantEpCompatibility(const std::string& component_name, + const std::string& variant_name, + const VariantEpCompatibilityInfo*& out_info) const { + out_info = nullptr; + const VariantInfo* variant = nullptr; + ORT_RETURN_IF_ERROR(FindVariant(model_package_info_, component_name_to_index_, + component_name, variant_name, variant)); + out_info = &variant->ep_compatibility; + return Status::OK(); +} #endif // !defined(ORT_MINIMAL_BUILD) +} diff --git a/onnxruntime/core/session/model_package/model_package_context.h b/onnxruntime/core/session/model_package/model_package_context.h index ceadf3d03dba3..1ecfffd7f74e0 100644 --- a/onnxruntime/core/session/model_package/model_package_context.h +++ b/onnxruntime/core/session/model_package/model_package_context.h @@ -8,21 +8,67 @@ #include "core/session/abi_session_options_impl.h" #include "core/session/environment.h" #include "core/session/onnxruntime_c_api.h" +#include "core/framework/execution_provider.h" #include "core/common/common.h" +#include "core/session/model_package/model_package_variant_selector.h" +#include +#include +#include +#include #include "nlohmann/json.hpp" using json = nlohmann::json; namespace onnxruntime { -struct ModelVariantInfo { - std::string ep{}; - std::string device{}; - std::string architecture{}; - std::string compatibility_info{}; - std::unordered_map metadata{}; - OrtCompiledModelCompatibility compiled_model_compatibility{}; - std::filesystem::path model_path{}; +struct VariantEpCompatibilityInfo { + std::optional ep; + std::optional device; + // Single opaque, EP-owned compatibility token for this (ep, device) entry. + // If a variant needs to advertise compatibility against multiple sub-targets + // (e.g. several SoC models or arch variants), the EP encodes that internally + // in this single string (e.g. comma-joined). ORT does not parse it. + std::optional compatibility_string; + OrtCompiledModelCompatibility compiled_model_compatibility{OrtCompiledModelCompatibility_EP_NOT_APPLICABLE}; +}; + +struct VariantModelInfo { + std::string identifier; // deterministic id (e.g., filename) + std::filesystem::path model_file_path; // resolved path under // + + // from variant.json file entry + std::optional> session_options; + std::optional> provider_options; + std::optional> shared_files; // logical_name -> checksum/path +}; + +// variant-level info (metadata.json + variant.json) +struct VariantInfo { + std::string component_name; + std::string variant_name; + + // The variant's root folder (always available regardless of variant.json presence). + std::filesystem::path folder_path; + + // from metadata.json: single EP target per variant (ep, device, compatibility_string) + VariantEpCompatibilityInfo ep_compatibility; + + // from variant.json: single model file entry. Empty when variant.json is absent. + std::optional file; + + // from variant.json + std::optional consumer_metadata; +}; + +struct ComponentInfo { + std::string component_name{}; + std::vector variants{}; + std::optional selected_variant_index{}; // index into variants +}; + +struct ModelPackageInfo { + int64_t schema_version{0}; + std::vector components{}; }; struct VariantSelectionEpInfo { @@ -31,34 +77,139 @@ struct VariantSelectionEpInfo { std::vector ep_devices{}; std::vector hardware_devices{}; std::vector ep_metadata{}; - ProviderOptions ep_options{}; }; -class ModelPackageContext { +class ModelPackageOptions; // forward declaration + +class ModelPackageComponentContext { public: - ModelPackageContext(const std::filesystem::path& package_root); + explicit ModelPackageComponentContext(const std::string& component_name, + const ComponentInfo& component_model_info, + const ModelPackageOptions& options); - const std::vector& GetModelVariantInfos() const noexcept { - return model_variant_infos_; + explicit ModelPackageComponentContext(const std::string& component_name, + const ComponentInfo& component_model_info, + gsl::span ep_infos); + + Status ResolveVariant(); + + const std::vector& GetVariantInfos() const noexcept { + return component_model_info_.variants; } + Status GetSelectedVariantFolderPath(const std::filesystem::path*& out_folder_path) const; + + // Get the single model file path for the selected variant. + Status GetSelectedVariantFilePath(std::filesystem::path& out_path) const; + + Status GetSelectedVariantFileSessionOptions(gsl::span& out_keys, + gsl::span& out_values) const; + + Status GetSelectedVariantFileProviderOptions(gsl::span& out_keys, + gsl::span& out_values) const; + + // Returns the consumer_metadata JSON object (from variant.json) serialized to a string for the + // selected variant. Returns an empty string if the variant did not declare consumer_metadata. + // Pointer lifetime is owned by this context. + + // C API helpers: return const char* pointer arrays with context-owned lifetime. + Status GetSelectedVariantFileSessionOptionPtrs(const char* const*& out_keys, + const char* const*& out_values, + size_t& out_count) const; + Status GetSelectedVariantFileProviderOptionPtrs(const char* const*& out_keys, + const char* const*& out_values, + size_t& out_count) const; + + Status GetSelectedVariantConsumerMetadata(const std::string*& out_json_str) const; + + Status GetSelectedVariantName(const std::string*& out_name) const; + + std::vector>& MutableProviderList() { return provider_list_; } + const std::vector& ExecutionDevices() const { return execution_devices_; } + const std::vector& DevicesSelected() const { return devices_selected_; } + gsl::span EpInfos() const { return ep_infos_; } + bool IsFromPolicy() const { return from_policy_; } + + // Rebuild the provider list for a new session creation call (providers are consumed/moved + // when registered, so they must be rebuilt for each session). + // Uses the provided session options for provider creation (should include merged provider options). + Status RebuildProviderListForSession(const Environment& env, const OrtSessionOptions& effective_options); + private: - std::vector model_variant_infos_; + std::string component_model_name_; + ComponentInfo component_model_info_{}; + + gsl::span ep_infos_{}; // non-owning EP intent when options are not used + std::vector owned_ep_infos_{}; // owned copy when constructed from ModelPackageOptions + std::vector> provider_list_{}; + + // optional runtime state mirrors (if needed by callers) + std::vector execution_devices_{}; + std::vector devices_selected_{}; + bool from_policy_{false}; + + // Caches for selected variant info. + mutable std::string consumer_metadata_cache_{}; + mutable bool consumer_metadata_cache_valid_{false}; + mutable std::filesystem::path folder_path_cache_{}; + mutable std::vector session_option_keys_cache_{}; + mutable std::vector session_option_values_cache_{}; + mutable std::vector provider_option_keys_cache_{}; + mutable std::vector provider_option_values_cache_{}; + + // C API pointer caches for session/provider options, owned by context for stable lifetime. + mutable std::vector session_option_key_ptrs_cache_{}; + mutable std::vector session_option_value_ptrs_cache_{}; + mutable std::vector provider_option_key_ptrs_cache_{}; + mutable std::vector provider_option_value_ptrs_cache_{}; + + Status ResolveVariantImpl(gsl::span ep_infos); + Status GetSelectedVariantInfo(const VariantInfo*& out_variant) const; }; -class ModelVariantSelector { +class ModelPackageContext { public: - ModelVariantSelector() = default; + explicit ModelPackageContext(const std::filesystem::path& package_root); + + size_t GetComponentCount() const noexcept; + Status GetComponentNames(gsl::span& out_names) const; + + // C API helpers: return const char* pointer arrays with context-owned lifetime. + void GetComponentNamePtrs(const char* const*& out_ptrs, size_t& out_count) const; + void GetVariantNamePtrs(const std::string& component_name, + const char* const*& out_ptrs, size_t& out_count) const; - // Select model variants that match the provided EP/device info. If multiple - // variants match, the one with the highest variant score is chosen. - Status SelectVariant(const ModelPackageContext& context, - gsl::span ep_infos, - std::optional& selected_variant_path) const; + Status GetVariantCount(const std::string& component_name, size_t& out_count) const; + Status GetVariantNames(const std::string& component_name, + gsl::span& out_variant_names) const; + + // Get the EP compatibility info declared on a variant. + // Lets callers (e.g. GenAI defaulting logic) inspect what EP a variant targets + // before any EP has been resolved / before SelectComponent has been called. + Status GetVariantEpCompatibility(const std::string& component_name, + const std::string& variant_name, + const VariantEpCompatibilityInfo*& out_info) const; + + const ModelPackageInfo& GetModelPackageInfo() const noexcept { + return model_package_info_; + } + + const std::vector& GetVariantInfos() const noexcept { + return model_variant_infos_; + } private: - // Compute a score for a variant - int CalculateVariantScore(const ModelVariantInfo& variant) const; + ModelPackageInfo model_package_info_{}; + std::vector model_variant_infos_; + + std::unordered_map component_name_to_index_{}; + std::vector component_names_cache_{}; + mutable std::unordered_map> component_to_variant_names_cache_{}; + mutable std::unordered_map> variant_to_file_identifiers_cache_{}; + + // C API pointer caches: owned by the context so their lifetime matches the documented contract. + mutable std::vector component_name_ptrs_cache_{}; + mutable std::unordered_map> variant_name_ptrs_cache_{}; }; } // namespace onnxruntime diff --git a/onnxruntime/core/session/model_package/model_package_descriptor_parser.cc b/onnxruntime/core/session/model_package/model_package_descriptor_parser.cc deleted file mode 100644 index bd297a2a32407..0000000000000 --- a/onnxruntime/core/session/model_package/model_package_descriptor_parser.cc +++ /dev/null @@ -1,482 +0,0 @@ -// # Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -#if !defined(ORT_MINIMAL_BUILD) - -#include -#include -#include -#include -#include -#include -#include - -#include "core/common/logging/logging.h" -#include "core/framework/error_code_helper.h" -#include "core/session/model_package/model_package_descriptor_parser.h" - -namespace onnxruntime { -namespace { - -struct VariantConstraintsSchema { - std::optional ep; - std::optional device; - std::optional architecture; - std::optional ep_compatibility_info; -}; - -struct VariantSchema { - std::optional model_type; - std::optional model_file; - std::optional model_id; - std::optional constraints; -}; - -struct ManifestSchema { - std::string model_name; - std::optional model_version; - std::optional> component_models; -}; - -struct MetadataSchema { - std::optional component_model_name; - std::unordered_map model_variants; -}; - -void from_json(const json& j, VariantConstraintsSchema& c) { - if (j.contains(kEpKey) && j[kEpKey].is_string()) c.ep = j[kEpKey].get(); - if (j.contains(kDeviceKey) && j[kDeviceKey].is_string()) c.device = j[kDeviceKey].get(); - if (j.contains(kArchitectureKey) && j[kArchitectureKey].is_string()) c.architecture = j[kArchitectureKey].get(); - if (j.contains(kEpCompatibilityInfoKey) && j[kEpCompatibilityInfoKey].is_string()) { - c.ep_compatibility_info = j[kEpCompatibilityInfoKey].get(); - } -} - -void from_json(const json& j, VariantSchema& v) { - if (j.contains(kModelTypeKey) && j[kModelTypeKey].is_string()) v.model_type = j[kModelTypeKey].get(); - if (j.contains(kModelFileKey) && j[kModelFileKey].is_string()) v.model_file = j[kModelFileKey].get(); - if (j.contains(kModelIdKey) && j[kModelIdKey].is_string()) v.model_id = j[kModelIdKey].get(); - if (j.contains(kConstraintsKey) && j[kConstraintsKey].is_object()) { - v.constraints = j[kConstraintsKey].get(); - } -} - -void from_json(const json& j, ManifestSchema& m) { - m.model_name = j.at(kModelNameKey).get(); // required - - if (j.contains(kModelVersionKey) && j[kModelVersionKey].is_string()) { - m.model_version = j[kModelVersionKey].get(); - } - - if (j.contains(kComponentModelsKey)) { - if (!j[kComponentModelsKey].is_array()) { - throw std::invalid_argument("\"component_models\" must be an array of strings"); - } - m.component_models = j[kComponentModelsKey].get>(); - } -} - -void from_json(const json& j, MetadataSchema& m) { - if (j.contains("component_model_name") && j["component_model_name"].is_string()) { - m.component_model_name = j["component_model_name"].get(); - } - m.model_variants = j.at(kModelVariantsKey).get>(); // required -} - -Status FindSingleOnnxFile(const std::filesystem::path& search_dir, - std::filesystem::path& resolved_path) { - std::vector onnx_files; - for (const auto& entry : std::filesystem::directory_iterator(search_dir)) { - if (!entry.is_regular_file()) { - continue; - } - - std::string ext = entry.path().extension().string(); - std::transform(ext.begin(), ext.end(), ext.begin(), - [](unsigned char c) { return static_cast(std::tolower(c)); }); - if (ext == ".onnx") { - onnx_files.push_back(entry.path()); - } - } - - if (onnx_files.empty()) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, - "No ONNX model file found under ", search_dir.string()); - } - - if (onnx_files.size() > 1) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, - "Multiple ONNX model files found under ", search_dir.string(), - ". Multiple ONNX files per variant are not supported yet."); - } - - resolved_path = onnx_files.front(); - return Status::OK(); -}; - -void ApplyVariantConstraints(const VariantConstraintsSchema& c, ModelVariantInfo& variant) { - if (c.ep.has_value()) variant.ep = *c.ep; - if (c.device.has_value()) variant.device = *c.device; - if (c.architecture.has_value()) variant.architecture = *c.architecture; - if (c.ep_compatibility_info.has_value()) variant.compatibility_info = *c.ep_compatibility_info; -} - -std::string SanitizeModelIdForPath(std::string model_id) { - for (char& ch : model_id) { - switch (ch) { - case '<': - case '>': - case ':': - case '"': - case '/': - case '\\': - case '|': - case '?': - case '*': - ch = '_'; - break; - default: - break; - } - } - return model_id; -} - -} // namespace - -// The package_root could be either a component model root (contains metadata.json) or a model package root (contains -// manifest.json). The parsing logic will first check for metadata.json to see if it's a component model root, and if -// not found, it will look for manifest.json to treat it as a model package root. -// -// This function parses information from manifest.json and/or metadata.json for the model variants from the same -// component model, producing a unified list of ModelVariantInfo. -// -// Note: If the package_root is a model package root, currently it only supports one component model. -// #TODO: In the future we may want ORT to support running multiple component models in a single "virtual" session. -// -// A manifest.json may look like this: -// -// { -// "model_name" : , -// "model_version": "1.0", -// "component_models" : [, -// ] -// } -// -// A metadata.json for the component model may look like this: -// -// { -// "component_model_name" : , -// "model_variants" : { -// : { -// "model_type": "onnx", -// "model_file" : , -// "constraints" : { -// "ep" : , -// "device" : , -// "ep_compatibility_info" : -// } -// }, -// : { -// "model_type": "onnx", -// "model_file" : , -// "constraints" : { -// "ep" : , -// "device" : , -// "ep_compatibility_info" : -// } -// } -// } -// } -Status ModelPackageDescriptorParser::ParseVariantsFromRoot(const std::filesystem::path& package_root, - /*out*/ std::vector& variants) const { - variants.clear(); - - // package_root could be either a component model root (contains metadata.json) or a model package root (contains manifest.json). - - // Mode 1: package_root is a component model root (contains metadata.json). - // In this mode metadata.json is the source of truth and manifest.json is ignored. - const auto component_metadata_path = package_root / kComponentModelMetadataFileName; - if (std::filesystem::exists(component_metadata_path) && - std::filesystem::is_regular_file(component_metadata_path)) { - std::ifstream mf(component_metadata_path, std::ios::binary); - if (!mf) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, - "Failed to open metadata.json at ", component_metadata_path.string()); - } - - json metadata_doc; - ORT_TRY { - metadata_doc = json::parse(mf); - } - ORT_CATCH(const std::exception& ex) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, - "metadata.json at ", component_metadata_path.string(), - " is not valid JSON: ", ex.what()); - } - - MetadataSchema metadata_schema; - ORT_TRY { - metadata_schema = metadata_doc.get(); - } - ORT_CATCH(const std::exception& ex) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, - "metadata.json at ", component_metadata_path.string(), - " has invalid schema: ", ex.what()); - } - - const std::string component_model_name = - metadata_schema.component_model_name.has_value() - ? *metadata_schema.component_model_name - : package_root.filename().string(); - - // component-root mode - ORT_RETURN_IF_ERROR(ParseVariantsFromComponent(component_model_name, - package_root, - &metadata_doc[kModelVariantsKey], - variants)); - - for (const auto& v : variants) { - LOGS(logger_, INFO) << "model variant: file='" << v.model_path.string() - << "' ep='" << v.ep << "' device='" << v.device - << "' arch='" << v.architecture << "'"; - } - - return Status::OK(); - } - - // Mode 2: package_root is a model package root, resolve via manifest.json. - const auto manifest_path = package_root / kModelPackageManifestFileName; - if (!std::filesystem::exists(manifest_path)) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, - "No manifest.json found at ", manifest_path.string()); - } - - std::ifstream f(manifest_path, std::ios::binary); - if (!f) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, - "Failed to open manifest.json at ", manifest_path.string()); - } - - json doc; - ORT_TRY { - doc = json::parse(f); - } - ORT_CATCH(const std::exception& ex) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, - "manifest.json is not valid JSON: ", ex.what()); - } - - ManifestSchema manifest_schema; - ORT_TRY { - manifest_schema = doc.get(); - } - ORT_CATCH(const std::exception& ex) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, - "manifest.json has invalid schema: ", ex.what()); - } - - if (manifest_schema.model_name.empty()) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, - "The \"model_name\" field in the manifest.json is missing or empty"); - } - - // Locate component models. - const bool has_component_models = manifest_schema.component_models.has_value(); - std::vector component_model_names; - std::unordered_map discovered_metadata_docs; - - if (has_component_models) { - component_model_names = *manifest_schema.component_models; - } else { - // Fallback: discover component models under package_root/models where a metadata.json exists. - const auto models_dir = package_root / "models"; - if (!std::filesystem::exists(models_dir) || !std::filesystem::is_directory(models_dir)) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, - "manifest.json missing \"component_models\" and no discoverable models directory at ", - models_dir.string()); - } - - for (const auto& entry : std::filesystem::directory_iterator(models_dir)) { - if (!entry.is_directory()) { - continue; - } - - const auto component_model_name = entry.path().filename().string(); - const auto metadata_path = entry.path() / kComponentModelMetadataFileName; - if (!std::filesystem::exists(metadata_path)) { - continue; // only folders with metadata.json count as component models - } - - std::ifstream mf(metadata_path, std::ios::binary); - if (!mf) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, - "Failed to open metadata.json at ", metadata_path.string()); - } - - json metadata_doc; - ORT_TRY { - metadata_doc = json::parse(mf); - } - ORT_CATCH(const std::exception& ex) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, - "metadata.json at ", metadata_path.string(), - " is not valid JSON: ", ex.what()); - } - - ORT_TRY { - (void)metadata_doc.get(); - } - ORT_CATCH(const std::exception& ex) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, - "metadata.json at ", metadata_path.string(), - " has invalid schema: ", ex.what()); - } - - discovered_metadata_docs.emplace(component_model_name, std::move(metadata_doc)); - component_model_names.push_back(component_model_name); - } - - if (component_model_names.empty()) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, - "manifest.json missing \"component_models\" and no component model folders with metadata.json were found under ", - models_dir.string()); - } - } - - if (component_model_names.size() != 1) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, - "manifest.json should contain exactly one component model name in \"component_models\" array " - "or the models directory should contain exactly one component model."); - } - - for (const auto& component_model_name : component_model_names) { - const auto component_model_root = package_root / "models" / component_model_name; - - // If component model is listed in manifest.json but corresponding directory is missing, warn and skip. - if (has_component_models && - (!std::filesystem::exists(component_model_root) || !std::filesystem::is_directory(component_model_root))) { - LOGS(logger_, WARNING) << "Component model '" << component_model_name - << "' is listed in manifest.json but directory does not exist: " - << component_model_root.string() - << ". Skipping this component model."; - continue; - } - - // Load metadata.json (if present) for this component model. - json metadata_doc; - const json* metadata_variants_obj = nullptr; - const auto metadata_path = component_model_root / kComponentModelMetadataFileName; - - if (!has_component_models) { - auto it_meta = discovered_metadata_docs.find(component_model_name); - if (it_meta != discovered_metadata_docs.end()) { - metadata_doc = it_meta->second; - metadata_variants_obj = &metadata_doc[kModelVariantsKey]; - } - } else if (std::filesystem::exists(metadata_path)) { - std::ifstream mf(metadata_path, std::ios::binary); - if (mf) { - ORT_TRY { - metadata_doc = json::parse(mf); - (void)metadata_doc.get(); // typed schema validation - metadata_variants_obj = &metadata_doc[kModelVariantsKey]; - } - ORT_CATCH(const std::exception&) { - // Ignore metadata parse/schema errors; fall back to metadata-absent handling below. - } - } - } - - ORT_RETURN_IF_ERROR(ParseVariantsFromComponent(component_model_name, - component_model_root, - metadata_variants_obj, - variants)); - } - - if (variants.empty()) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, - "No valid component models were found under ", (package_root / "models").string()); - } - - for (const auto& v : variants) { - LOGS(logger_, INFO) << "model variant: file='" << v.model_path.string() - << "' ep='" << v.ep << "' device='" << v.device - << "' arch='" << v.architecture << "'"; - } - - return Status::OK(); -} - -Status ModelPackageDescriptorParser::ParseVariantsFromComponent( - const std::string& component_model_name, - const std::filesystem::path& component_model_root, - const json* metadata_variants_obj, - /*in,out*/ std::vector& variants) const { - if (metadata_variants_obj == nullptr) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, - "Missing metadata model variants for component model: ", - component_model_name); - } - - std::unordered_map metadata_variants; - ORT_TRY { - metadata_variants = metadata_variants_obj->get>(); - } - ORT_CATCH(const std::exception& ex) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, - "Invalid metadata variant schema for component model '", - component_model_name, "': ", ex.what()); - } - - for (const auto& kv : metadata_variants) { - const std::string& variant_name = kv.first; - const VariantSchema& variant_schema = kv.second; - - ModelVariantInfo variant{}; - std::filesystem::path model_dir = component_model_root / variant_name; - - if (variant_schema.model_id.has_value() && !variant_schema.model_id->empty()) { - model_dir = component_model_root / SanitizeModelIdForPath(*variant_schema.model_id); - } - - std::filesystem::path resolved_model_path; - - if (variant_schema.model_file.has_value()) { - const std::filesystem::path candidate_path = model_dir / *variant_schema.model_file; - - if (!std::filesystem::exists(candidate_path)) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, - "Variant '", variant_name, "' file path does not exist: ", - candidate_path.string()); - } - - if (std::filesystem::is_regular_file(candidate_path)) { - resolved_model_path = candidate_path; - } else if (std::filesystem::is_directory(candidate_path)) { - ORT_RETURN_IF_ERROR(FindSingleOnnxFile(candidate_path, resolved_model_path)); - } else { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, - "Variant '", variant_name, - "' file path is neither a file nor a directory: ", - candidate_path.string()); - } - } else { - ORT_RETURN_IF_ERROR(FindSingleOnnxFile(model_dir, resolved_model_path)); - } - - variant.model_path = std::move(resolved_model_path); - - if (variant_schema.constraints.has_value()) { - ApplyVariantConstraints(*variant_schema.constraints, variant); - } - - variants.push_back(std::move(variant)); - } - - return Status::OK(); -} - -} // namespace onnxruntime - -#endif // !defined(ORT_MINIMAL_BUILD) \ No newline at end of file diff --git a/onnxruntime/core/session/model_package/model_package_descriptor_parser.h b/onnxruntime/core/session/model_package/model_package_descriptor_parser.h deleted file mode 100644 index 9f2cad736c215..0000000000000 --- a/onnxruntime/core/session/model_package/model_package_descriptor_parser.h +++ /dev/null @@ -1,49 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -#pragma once - -#if !defined(ORT_MINIMAL_BUILD) - -#include "core/session/model_package/model_package_context.h" - -namespace onnxruntime { - -// -// Keys for parsing the model package manifest json and component model metadata json. -// -static constexpr const char* kModelPackageManifestFileName = "manifest.json"; -static constexpr const char* kModelNameKey = "model_name"; -static constexpr const char* kModelVersionKey = "model_version"; -static constexpr const char* kComponentModelsKey = "component_models"; -static constexpr const char* kComponentModelMetadataFileName = "metadata.json"; -static constexpr const char* kModelVariantsKey = "model_variants"; -static constexpr const char* kVariantNameKey = "variant_name"; -static constexpr const char* kModelTypeKey = "model_type"; -static constexpr const char* kModelFileKey = "model_file"; -static constexpr const char* kModelIdKey = "model_id"; -static constexpr const char* kConstraintsKey = "constraints"; -static constexpr const char* kEpKey = "ep"; -static constexpr const char* kDeviceKey = "device"; -static constexpr const char* kArchitectureKey = "architecture"; -static constexpr const char* kEpCompatibilityInfoKey = "ep_compatibility_info"; - -class ModelPackageDescriptorParser { - public: - explicit ModelPackageDescriptorParser(const logging::Logger& logger) : logger_(logger) {} - - Status ParseVariantsFromRoot(const std::filesystem::path& package_root, - /*out*/ std::vector& components) const; - - private: - Status ParseVariantsFromComponent(const std::string& component_model_name, - const std::filesystem::path& component_model_root, - const json* metadata_variants_obj, - /*in,out*/ std::vector& variants) const; - - const logging::Logger& logger_; -}; - -} // namespace onnxruntime - -#endif // !defined(ORT_MINIMAL_BUILD) \ No newline at end of file diff --git a/onnxruntime/core/session/model_package/model_package_options.cc b/onnxruntime/core/session/model_package/model_package_options.cc new file mode 100644 index 0000000000000..52765d9e0fe63 --- /dev/null +++ b/onnxruntime/core/session/model_package/model_package_options.cc @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#if !defined(ORT_MINIMAL_BUILD) + +#include "core/session/model_package/model_package_options.h" + +#include "core/common/common.h" +#include "core/common/logging/logging.h" +#include "core/graph/constants.h" +#include "core/providers/providers.h" +#include "core/session/provider_policy_context.h" +#include "core/session/utils.h" + +namespace onnxruntime { + +ModelPackageOptions::ModelPackageOptions(const Environment& env, + const OrtSessionOptions& session_options) { + ResolveEpSelection(env, session_options); +} + +void ModelPackageOptions::ResolveEpSelection(const Environment& env, + const OrtSessionOptions& session_options) { + const bool has_provider_factories = !session_options.provider_factories.empty(); + from_policy_ = !has_provider_factories && session_options.value.ep_selection_policy.enable; + + provider_list_.clear(); + execution_devices_.clear(); + devices_selected_.clear(); + ep_infos_.clear(); + + if (has_provider_factories) { + const auto& logger = *logging::LoggingManager::DefaultLogger().ToExternal(); + for (auto& factory : session_options.provider_factories) { + provider_list_.push_back(factory->CreateProvider(session_options, logger)); + } + ORT_THROW_IF_ERROR(GetVariantSelectionEpInfo(provider_list_, ep_infos_)); + } else if (from_policy_) { + OrtKeyValuePairs model_metadata; + ProviderPolicyContext provider_policy_context; + OrtSessionOptions mutable_session_options = session_options; + ORT_THROW_IF_ERROR(provider_policy_context.SelectEpsForModelPackage( + env, mutable_session_options, model_metadata, + execution_devices_, devices_selected_, provider_list_)); + ORT_THROW_IF_ERROR(GetVariantSelectionEpInfo(provider_list_, ep_infos_)); + } else { + // No explicit providers and no policy: default to CPU for variant selection. + ep_infos_.push_back(VariantSelectionEpInfo{}); + ep_infos_.back().ep_name = kCpuExecutionProvider; + } + + ORT_THROW_IF_ERROR(PrintAvailableAndSelectedEpInfos(env, ep_infos_)); +} + +Status ModelPackageOptions::RebuildProviderListForSession(const Environment& env, + const OrtSessionOptions& effective_options) const { + provider_list_.clear(); + + if (ep_infos_.empty()) { + return Status::OK(); + } + + const auto& ep_info = ep_infos_[0]; + if (ep_info.ep_name == kCpuExecutionProvider || ep_info.ep_devices.empty()) { + // CPU is built-in; no provider to register. + return Status::OK(); + } + + std::unique_ptr provider_factory; + ORT_RETURN_IF_ERROR(CreateIExecutionProviderFactoryForEpDevices( + env, + gsl::span(ep_info.ep_devices.data(), ep_info.ep_devices.size()), + provider_factory)); + + const auto& logger = *logging::LoggingManager::DefaultLogger().ToExternal(); + provider_list_.push_back(provider_factory->CreateProvider(effective_options, logger)); + + return Status::OK(); +} + +} // namespace onnxruntime + +#endif // !defined(ORT_MINIMAL_BUILD) diff --git a/onnxruntime/core/session/model_package/model_package_options.h b/onnxruntime/core/session/model_package/model_package_options.h new file mode 100644 index 0000000000000..ffaf6d2627ca9 --- /dev/null +++ b/onnxruntime/core/session/model_package/model_package_options.h @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#if !defined(ORT_MINIMAL_BUILD) + +#include +#include + +#include "core/common/status.h" +#include "core/framework/execution_provider.h" +#include "core/session/abi_session_options_impl.h" +#include "core/session/model_package/model_package_context.h" // VariantSelectionEpInfo +#include "core/session/environment.h" + +namespace onnxruntime { + +class ModelPackageOptions { + public: + ModelPackageOptions(const Environment& env, const OrtSessionOptions& session_options); + + Status RebuildProviderListForSession(const Environment& env, + const OrtSessionOptions& effective_options) const; + + // Resolved state accessors + std::vector>& MutableProviderList() const noexcept { return provider_list_; } + const std::vector>& ProviderList() const noexcept { return provider_list_; } + const std::vector& EpInfos() const noexcept { return ep_infos_; } + const std::vector& ExecutionDevices() const noexcept { return execution_devices_; } + const std::vector& DevicesSelected() const noexcept { return devices_selected_; } + bool FromPolicy() const noexcept { return from_policy_; } + + private: + void ResolveEpSelection(const Environment& env, const OrtSessionOptions& session_options); + + // might needs to be rebuilt per session creation, as it becomes empty + // after consumed by RegisterExecutionProvider(std::move(...)). + mutable std::vector> provider_list_{}; + + std::vector ep_infos_{}; + std::vector execution_devices_{}; + std::vector devices_selected_{}; + bool from_policy_{false}; +}; + +} // namespace onnxruntime + +#endif // !defined(ORT_MINIMAL_BUILD) diff --git a/onnxruntime/core/session/model_package/model_package_variant_selector.cc b/onnxruntime/core/session/model_package/model_package_variant_selector.cc new file mode 100644 index 0000000000000..64999c7763d1c --- /dev/null +++ b/onnxruntime/core/session/model_package/model_package_variant_selector.cc @@ -0,0 +1,233 @@ +// Licensed under the MIT License. + +#if !defined(ORT_MINIMAL_BUILD) + +#include "core/session/model_package/model_package_variant_selector.h" +#include "core/session/model_package/model_package_context.h" + +#include +#include +#include + +#include "core/common/logging/logging.h" +#include "core/framework/error_code_helper.h" +#include "core/session/abi_devices.h" +#include "core/session/utils.h" + +namespace onnxruntime { +namespace { + +struct VariantMatchResult { + bool matched{false}; + int score{std::numeric_limits::min()}; +}; + +std::string ToLower(std::string_view s) { + std::string result(s); + std::transform(result.begin(), result.end(), result.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + return result; +} + +bool MatchesDevice(const OrtHardwareDevice* hd, std::string_view value) { + if (value.empty() || hd == nullptr) { + return value.empty(); + } + + const std::string device = ToLower(value); + switch (hd->type) { + case OrtHardwareDeviceType::OrtHardwareDeviceType_CPU: + return device == "cpu"; + case OrtHardwareDeviceType::OrtHardwareDeviceType_GPU: + return device == "gpu"; + case OrtHardwareDeviceType::OrtHardwareDeviceType_NPU: + return device == "npu"; + default: + return false; + } +} + +const OrtHardwareDevice* FindMatchingHardwareDevice(std::string_view device_constraint, + gsl::span hardware_devices) { + if (device_constraint.empty()) { + return nullptr; + } + + for (const auto* hd : hardware_devices) { + if (MatchesDevice(hd, device_constraint)) { + return hd; + } + } + + return nullptr; +} + +int CompatibilityToScore(OrtCompiledModelCompatibility compatibility) { + switch (compatibility) { + case OrtCompiledModelCompatibility_EP_SUPPORTED_OPTIMAL: + return 100; + case OrtCompiledModelCompatibility_EP_SUPPORTED_PREFER_RECOMPILATION: + return 50; + case OrtCompiledModelCompatibility_EP_NOT_APPLICABLE: + return 0; + case OrtCompiledModelCompatibility_EP_UNSUPPORTED: + default: + return -100; + } +} + +// Calls the EP's ValidateCompiledModelCompatibilityInfo for a single compatibility string. +// Writes OrtCompiledModelCompatibility_EP_NOT_APPLICABLE when the EP does not implement the ABI, +// the EP is too old, or the compatibility string is empty. +Status ValidateCompatibilityString(const VariantSelectionEpInfo& ep_info, + const std::string& compatibility_string, + std::vector& constraint_devices, + OrtCompiledModelCompatibility& out_compat) { + out_compat = OrtCompiledModelCompatibility_EP_NOT_APPLICABLE; + + if (compatibility_string.empty()) { + return Status::OK(); + } + + auto* ep_factory = ep_info.ep_factory; + if (ep_factory == nullptr || + ep_factory->ort_version_supported < 23 || + ep_factory->ValidateCompiledModelCompatibilityInfo == nullptr) { + return Status::OK(); + } + + auto status = ep_factory->ValidateCompiledModelCompatibilityInfo(ep_factory, + constraint_devices.data(), + constraint_devices.size(), + compatibility_string.c_str(), + &out_compat); + ORT_RETURN_IF_ERROR(ToStatusAndRelease(status)); + return Status::OK(); +} + +// Checks whether a variant's single EP compatibility entry matches the given EP. +// Returns a match result with a score derived from ValidateCompiledModelCompatibilityInfo(). +VariantMatchResult MatchVariantForEp(VariantInfo& variant, const VariantSelectionEpInfo& ep_info) { + VariantMatchResult result{}; + const auto& ec = variant.ep_compatibility; + + // 1. Match EP name. Required and non-empty per schema. + if (!ec.ep.has_value() || *ec.ep != ep_info.ep_name) { + return result; + } + + // 2. Match device constraint. + bool device_ok = !ec.device.has_value() || ec.device->empty(); + std::vector constraint_devices = ep_info.hardware_devices; + + if (!device_ok) { + if (ep_info.hardware_devices.empty()) { + return result; + } + if (const auto* matched_hd = FindMatchingHardwareDevice(*ec.device, ep_info.hardware_devices)) { + device_ok = true; + constraint_devices = {matched_hd}; + } + } + + if (!device_ok) { + return result; + } + + // 3. Validate compatibility string via EP callback. + const std::string& compat_str = + ec.compatibility_string.has_value() ? *ec.compatibility_string : std::string{}; + + OrtCompiledModelCompatibility compat = OrtCompiledModelCompatibility_EP_NOT_APPLICABLE; + auto st = ValidateCompatibilityString(ep_info, compat_str, constraint_devices, compat); + if (!st.IsOK()) { + LOGS_DEFAULT(WARNING) << "Failed to validate compiled model compatibility for variant '" + << variant.variant_name << "'. Error: " << st.ErrorMessage(); + return result; + } + + variant.ep_compatibility.compiled_model_compatibility = compat; + + if (compat == OrtCompiledModelCompatibility_EP_UNSUPPORTED) { + return result; + } + + result.matched = true; + result.score = CompatibilityToScore(compat); + return result; +} + +} // namespace + +Status VariantSelector::SelectVariant(const ModelPackageComponentContext& context, + gsl::span ep_infos, + std::optional& selected_variant) const { + selected_variant.reset(); + + std::vector variants = context.GetVariantInfos(); + if (variants.empty()) { + return Status::OK(); + } + + const VariantSelectionEpInfo* selected_ep_info = nullptr; + if (!ep_infos.empty()) { + // For now we only consider the first captured EP for variant matching. + // + // ModelPackageOptions may capture multiple EPs (e.g. when EP selection comes from a policy that + // returns a ranked list such as [CUDA, WebGPU, CPU]). We use only the first EP. Callers that + // need a specific EP should put it first in the captured order. + // TODO: extend to rank variants across the full EP list, honoring captured priority order. + selected_ep_info = &ep_infos[0]; + if (ep_infos.size() > 1) { + LOGS_DEFAULT(INFO) << "Multiple EPs captured for model package; using only the first ('" + << ep_infos[0].ep_name << "') for variant matching."; + } + } + + // EP/device compatibility pass. + // + // Each variant declares a single target EP (ep, device, compatibility_string). ORT does not + // parse or decompose the compatibility string -- EP validates the compatibility string and + // returns an OrtCompiledModelCompatibility enum indicating the level of compatibility. + // + // Selection algorithm: + // For each variant whose single EP/device declaration matches the selected EP, call + // OrtEpFactory::ValidateCompiledModelCompatibilityInfo() and map the returned + // OrtCompiledModelCompatibility enum to a numeric score. Pick the highest-scoring variant. + // On ties, the first variant in manifest declaration order wins. + // + // Planned fallback ladder (not yet wired -- TODO): + // a) If the EP implements OrtEpFactory::SelectBestCompiledModelCandidate() (PR #28387), gather + // every matching variant into a candidate list and let the EP pick the best index. SIZE_MAX + // means "none acceptable", fall through to (b). + // b) Otherwise, use ValidateCompiledModelCompatibilityInfo() per variant as we do today. + // c) If neither ABI is implemented, return the first variant whose ep/device matches. + if (selected_ep_info != nullptr) { + int best_score = std::numeric_limits::min(); + std::optional best_index; + + for (size_t i = 0, end = variants.size(); i < end; ++i) { + VariantMatchResult m = MatchVariantForEp(variants[i], *selected_ep_info); + if (!m.matched) { + continue; + } + // Strict '>' so on ties we keep the first variant seen, giving deterministic + // tie-break by manifest declaration order. + if (!best_index.has_value() || m.score > best_score) { + best_score = m.score; + best_index = i; + } + } + + if (best_index.has_value()) { + selected_variant = std::move(variants[*best_index]); + return Status::OK(); + } + } + + return Status::OK(); +} + +} // namespace onnxruntime + +#endif // !defined(ORT_MINIMAL_BUILD) diff --git a/onnxruntime/core/session/model_package/model_package_variant_selector.h b/onnxruntime/core/session/model_package/model_package_variant_selector.h new file mode 100644 index 0000000000000..95011cae990f1 --- /dev/null +++ b/onnxruntime/core/session/model_package/model_package_variant_selector.h @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#if !defined(ORT_MINIMAL_BUILD) + +#include +#include +#include "core/common/common.h" +#include "core/session/onnxruntime_c_api.h" + +namespace onnxruntime { + +// forward declaration +class ModelPackageComponentContext; +struct VariantInfo; +struct VariantModelInfo; +struct VariantEpCompatibilityInfo; +struct VariantSelectionEpInfo; + +class VariantSelector { + public: + VariantSelector() = default; + + // Select model variant (finest granularity). + Status SelectVariant(const ModelPackageComponentContext& context, + gsl::span ep_infos, + std::optional& selected_variant) const; +}; + +} // namespace onnxruntime + +#endif // !defined(ORT_MINIMAL_BUILD) diff --git a/onnxruntime/core/session/model_package_api.cc b/onnxruntime/core/session/model_package_api.cc new file mode 100644 index 0000000000000..27abb0f5f7a37 --- /dev/null +++ b/onnxruntime/core/session/model_package_api.cc @@ -0,0 +1,500 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/session/model_package_api.h" + +#include "core/common/common.h" +#include "core/framework/error_code_helper.h" +#include "core/session/inference_session.h" +#include "core/session/abi_session_options_impl.h" +#include "core/session/ort_apis.h" +#include "core/session/ort_env.h" + +#if !defined(ORT_MINIMAL_BUILD) +#include "core/session/model_package/model_package_context.h" +#include "core/session/model_package/model_package_options.h" +#include "core/session/utils.h" + +#endif + +using namespace onnxruntime; + +#define RETURN_NOT_IMPL_IN_MINIMAL_BUILD() \ + return OrtApis::CreateStatus(ORT_NOT_IMPLEMENTED, \ + "Model package API is not supported in this build") + +ORT_API(void, OrtModelPackageAPI::ReleaseModelPackageOptions, + _Frees_ptr_opt_ OrtModelPackageOptions* options) { +#if !defined(ORT_MINIMAL_BUILD) + delete reinterpret_cast(options); +#else + ORT_UNUSED_PARAMETER(options); +#endif +} + +ORT_API_STATUS_IMPL(OrtModelPackageAPI::CreateModelPackageOptionsFromSessionOptions, + _In_ const OrtEnv* env, + _In_ const OrtSessionOptions* session_options, + _Outptr_ OrtModelPackageOptions** out) { + API_IMPL_BEGIN +#if !defined(ORT_MINIMAL_BUILD) + if (env == nullptr || session_options == nullptr || out == nullptr) { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "env, session_options and out must be non-null"); + } + + auto options = std::make_unique(env->GetEnvironment(), *session_options); + *out = reinterpret_cast(options.release()); + return nullptr; +#else + ORT_UNUSED_PARAMETER(env); + ORT_UNUSED_PARAMETER(session_options); + ORT_UNUSED_PARAMETER(out); + RETURN_NOT_IMPL_IN_MINIMAL_BUILD(); +#endif + API_IMPL_END +} + +ORT_API(void, OrtModelPackageAPI::ReleaseModelPackageContext, + _Frees_ptr_opt_ OrtModelPackageContext* ctx) { +#if !defined(ORT_MINIMAL_BUILD) + delete reinterpret_cast(ctx); +#else + ORT_UNUSED_PARAMETER(ctx); +#endif +} + +ORT_API_STATUS_IMPL(OrtModelPackageAPI::CreateModelPackageContext, + _In_ const ORTCHAR_T* package_root, + _Outptr_ OrtModelPackageContext** out) { + API_IMPL_BEGIN +#if !defined(ORT_MINIMAL_BUILD) + + if (package_root == nullptr) { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "package_root must be non-null"); + } + + if (out == nullptr) { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "out must be non-null"); + } + + auto ctx = std::make_unique(std::filesystem::path{package_root}); + + *out = reinterpret_cast(ctx.release()); + return nullptr; +#else + ORT_UNUSED_PARAMETER(package_root); + ORT_UNUSED_PARAMETER(out); + RETURN_NOT_IMPL_IN_MINIMAL_BUILD(); +#endif + API_IMPL_END +} + +ORT_API_STATUS_IMPL(OrtModelPackageAPI::ModelPackage_GetComponentCount, + _In_ const OrtModelPackageContext* ctx, + _Out_ size_t* out_count) { + API_IMPL_BEGIN +#if !defined(ORT_MINIMAL_BUILD) + if (ctx == nullptr || out_count == nullptr) { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "ctx and out_count must be non-null"); + } + *out_count = reinterpret_cast(ctx)->GetComponentCount(); + return nullptr; +#else + ORT_UNUSED_PARAMETER(ctx); + ORT_UNUSED_PARAMETER(out_count); + RETURN_NOT_IMPL_IN_MINIMAL_BUILD(); +#endif + API_IMPL_END +} + +ORT_API_STATUS_IMPL(OrtModelPackageAPI::ModelPackage_GetComponentNames, + _In_ const OrtModelPackageContext* ctx, + _Outptr_result_buffer_maybenull_(*out_count) const char* const** out_names, + _Out_ size_t* out_count) { + API_IMPL_BEGIN +#if !defined(ORT_MINIMAL_BUILD) + if (ctx == nullptr || out_names == nullptr || out_count == nullptr) { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "ctx, out_names, and out_count must be non-null"); + } + + gsl::span names; + ORT_API_RETURN_IF_STATUS_NOT_OK( + reinterpret_cast(ctx)->GetComponentNames(names)); + + const char* const* ptrs = nullptr; + size_t count = 0; + reinterpret_cast(ctx)->GetComponentNamePtrs(ptrs, count); + *out_names = ptrs; + *out_count = count; + return nullptr; +#else + ORT_UNUSED_PARAMETER(ctx); + ORT_UNUSED_PARAMETER(out_names); + ORT_UNUSED_PARAMETER(out_count); + RETURN_NOT_IMPL_IN_MINIMAL_BUILD(); +#endif + API_IMPL_END +} + +ORT_API_STATUS_IMPL(OrtModelPackageAPI::ModelPackage_GetVariantCount, + _In_ const OrtModelPackageContext* ctx, + _In_ const char* component_name, + _Out_ size_t* out_count) { + API_IMPL_BEGIN +#if !defined(ORT_MINIMAL_BUILD) + if (ctx == nullptr || component_name == nullptr || out_count == nullptr) { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "ctx, component_name, and out_count must be non-null"); + } + + ORT_API_RETURN_IF_STATUS_NOT_OK( + reinterpret_cast(ctx)->GetVariantCount(component_name, *out_count)); + return nullptr; +#else + ORT_UNUSED_PARAMETER(ctx); + ORT_UNUSED_PARAMETER(component_name); + ORT_UNUSED_PARAMETER(out_count); + RETURN_NOT_IMPL_IN_MINIMAL_BUILD(); +#endif + API_IMPL_END +} + +ORT_API_STATUS_IMPL(OrtModelPackageAPI::ModelPackage_GetVariantNames, + _In_ const OrtModelPackageContext* ctx, + _In_ const char* component_name, + _Outptr_result_buffer_maybenull_(*out_count) const char* const** out_variant_names, + _Out_ size_t* out_count) { + API_IMPL_BEGIN +#if !defined(ORT_MINIMAL_BUILD) + if (ctx == nullptr || component_name == nullptr || out_variant_names == nullptr || out_count == nullptr) { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, + "ctx, component_name, out_variant_names, and out_count must be non-null"); + } + + gsl::span variant_names; + ORT_API_RETURN_IF_STATUS_NOT_OK( + reinterpret_cast(ctx)->GetVariantNames(component_name, variant_names)); + + const char* const* ptrs = nullptr; + size_t count = 0; + reinterpret_cast(ctx)->GetVariantNamePtrs(component_name, ptrs, count); + *out_variant_names = ptrs; + *out_count = count; + return nullptr; +#else + ORT_UNUSED_PARAMETER(ctx); + ORT_UNUSED_PARAMETER(component_name); + ORT_UNUSED_PARAMETER(out_variant_names); + ORT_UNUSED_PARAMETER(out_count); + RETURN_NOT_IMPL_IN_MINIMAL_BUILD(); +#endif + API_IMPL_END +} + +ORT_API_STATUS_IMPL(OrtModelPackageAPI::SelectComponent, + _In_ const OrtModelPackageContext* context, + _In_ const char* component_name, + _In_ const OrtModelPackageOptions* options, + _Outptr_ OrtModelPackageComponentContext** out) { + API_IMPL_BEGIN +#if !defined(ORT_MINIMAL_BUILD) + if (context == nullptr || component_name == nullptr || options == nullptr || out == nullptr) { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, + "context, component_name, options, and out must be non-null"); + } + + const auto* cxx_ctx = reinterpret_cast(context); + const auto* cxx_options = reinterpret_cast(options); + + const auto& package_info = cxx_ctx->GetModelPackageInfo(); + const ComponentInfo* component_info = nullptr; + for (const auto& component : package_info.components) { + if (component.component_name == component_name) { + component_info = &component; + break; + } + } + + if (component_info == nullptr) { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "Component model not found."); + } + + auto cix = std::make_unique( + component_name, *component_info, *cxx_options); + + ORT_API_RETURN_IF_STATUS_NOT_OK(cix->ResolveVariant()); + + *out = reinterpret_cast(cix.release()); + return nullptr; +#else + ORT_UNUSED_PARAMETER(context); + ORT_UNUSED_PARAMETER(component_name); + ORT_UNUSED_PARAMETER(options); + ORT_UNUSED_PARAMETER(out); + RETURN_NOT_IMPL_IN_MINIMAL_BUILD(); +#endif + API_IMPL_END +} + +ORT_API(void, OrtModelPackageAPI::ReleaseModelPackageComponentContext, + _Frees_ptr_opt_ OrtModelPackageComponentContext* cix) { +#if !defined(ORT_MINIMAL_BUILD) + delete reinterpret_cast(cix); +#else + ORT_UNUSED_PARAMETER(cix); +#endif +} + +ORT_API_STATUS_IMPL(OrtModelPackageAPI::ModelPackageComponent_GetSelectedVariantFolderPath, + _In_ const OrtModelPackageComponentContext* ctx, + _Outptr_ const ORTCHAR_T** folder_path) { + API_IMPL_BEGIN +#if !defined(ORT_MINIMAL_BUILD) + if (ctx == nullptr || folder_path == nullptr) { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "ctx and folder_path must be non-null"); + } + + const auto* cxx_ctx = reinterpret_cast(ctx); + + const std::filesystem::path* folder = nullptr; + ORT_API_RETURN_IF_STATUS_NOT_OK(cxx_ctx->GetSelectedVariantFolderPath(folder)); + ORT_API_RETURN_IF(folder == nullptr, ORT_FAIL, "Selected variant folder path is null."); + + *folder_path = folder->c_str(); + return nullptr; +#else + ORT_UNUSED_PARAMETER(ctx); + ORT_UNUSED_PARAMETER(folder_path); + RETURN_NOT_IMPL_IN_MINIMAL_BUILD(); +#endif + API_IMPL_END +} + +ORT_API_STATUS_IMPL(OrtModelPackageAPI::CreateSession, + _In_ const OrtEnv* env, + _In_ OrtModelPackageComponentContext* ctx, + _In_opt_ const OrtSessionOptions* session_options, + _Outptr_ OrtSession** session) { + API_IMPL_BEGIN +#if !defined(ORT_MINIMAL_BUILD) + if (env == nullptr || ctx == nullptr || session == nullptr) { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, + "env, ctx, and session must be non-null"); + } + + auto& mp_ctx = *reinterpret_cast(ctx); + + // 1) Get the selected variant model file path. + // ORT only supports a single ONNX file per variant folder. + std::filesystem::path selected_file_path; + ORT_API_RETURN_IF_STATUS_NOT_OK(mp_ctx.GetSelectedVariantFilePath(selected_file_path)); + + // 2) Pick the OrtSessionOptions per precedence rules: + // - session_options == nullptr (default path): start from a clean OrtSessionOptions, + // and merge variant-specific session + provider options from the package metadata. + // - session_options != nullptr (advanced path): use caller-supplied as-is, no metadata merge. + const OrtSessionOptions* effective_options = nullptr; + std::optional effective_options_storage; + + if (session_options == nullptr) { + // Start from a clean session options. The only EP-related state comes from + // the variant metadata (session options and provider options), not from the + // original session options used to create the package options. + effective_options_storage.emplace(); + + // Merge variant file session options into config options. + gsl::span session_option_keys; + gsl::span session_option_values; + ORT_API_RETURN_IF_STATUS_NOT_OK( + mp_ctx.GetSelectedVariantFileSessionOptions(session_option_keys, session_option_values)); + + ORT_API_RETURN_IF(session_option_keys.size() != session_option_values.size(), + ORT_FAIL, "Session option keys/values size mismatch."); + + for (size_t i = 0; i < session_option_keys.size(); ++i) { + OrtStatus* st = OrtApis::AddSessionConfigEntry(&*effective_options_storage, + session_option_keys[i].c_str(), + session_option_values[i].c_str()); + if (st != nullptr) { + return st; + } + } + + // Merge variant file provider options as flat key/value entries for the selected EP devices. + gsl::span provider_option_keys; + gsl::span provider_option_values; + ORT_API_RETURN_IF_STATUS_NOT_OK( + mp_ctx.GetSelectedVariantFileProviderOptions(provider_option_keys, provider_option_values)); + + ORT_API_RETURN_IF(provider_option_keys.size() != provider_option_values.size(), + ORT_FAIL, "Provider option keys/values size mismatch."); + + if (!provider_option_keys.empty()) { + // Use ep_devices from the captured EP info for applying provider options. + // DevicesSelected() is only populated for the policy path, but ep_infos[0].ep_devices + // is populated for both factory and policy paths. + const auto& ep_infos = mp_ctx.EpInfos(); + if (!ep_infos.empty() && !ep_infos[0].ep_devices.empty()) { + std::vector provider_option_key_ptrs; + std::vector provider_option_value_ptrs; + provider_option_key_ptrs.reserve(provider_option_keys.size()); + provider_option_value_ptrs.reserve(provider_option_values.size()); + + for (size_t i = 0; i < provider_option_keys.size(); ++i) { + provider_option_key_ptrs.push_back(provider_option_keys[i].c_str()); + provider_option_value_ptrs.push_back(provider_option_values[i].c_str()); + } + + ORT_API_RETURN_IF_STATUS_NOT_OK(onnxruntime::AddEpOptionsToSessionOptions( + gsl::span(ep_infos[0].ep_devices.data(), ep_infos[0].ep_devices.size()), + gsl::span(provider_option_key_ptrs.data(), provider_option_key_ptrs.size()), + gsl::span(provider_option_value_ptrs.data(), provider_option_value_ptrs.size()), + effective_options_storage->value)); + } + } + + effective_options = &*effective_options_storage; + } else { + effective_options = session_options; + } + + // 3) Create session with the resolved file and effective session options. + std::unique_ptr sess; + ORT_API_RETURN_IF_ERROR(onnxruntime::CreateSessionForModelPackage( + effective_options, + env->GetEnvironment(), + selected_file_path, + mp_ctx, + sess)); + + // 4) Initialize. + ORT_API_RETURN_IF_ERROR(InitializeSession(effective_options, *sess)); + + *session = reinterpret_cast(sess.release()); + return nullptr; +#else + ORT_UNUSED_PARAMETER(env); + ORT_UNUSED_PARAMETER(ctx); + ORT_UNUSED_PARAMETER(session_options); + ORT_UNUSED_PARAMETER(session); + RETURN_NOT_IMPL_IN_MINIMAL_BUILD(); +#endif + API_IMPL_END +} + +// ---------- API table ------------------------------------------------------ + +ORT_API_STATUS_IMPL(OrtModelPackageAPI::ModelPackage_GetVariantEpName, + _In_ const OrtModelPackageContext* ctx, + _In_ const char* component_name, + _In_ const char* variant_name, + _Outptr_result_maybenull_ const char** out_ep) { + API_IMPL_BEGIN +#if !defined(ORT_MINIMAL_BUILD) + if (ctx == nullptr || component_name == nullptr || variant_name == nullptr) { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, + "ctx, component_name, and variant_name must be non-null"); + } + + const onnxruntime::VariantEpCompatibilityInfo* info = nullptr; + auto status = reinterpret_cast(ctx)->GetVariantEpCompatibility( + component_name, variant_name, info); + + if (out_ep != nullptr) { + if (status.IsOK() && info != nullptr && info->ep.has_value()) { + *out_ep = info->ep->c_str(); + } else { + *out_ep = nullptr; + } + } + return nullptr; +#else + ORT_UNUSED_PARAMETER(ctx); + ORT_UNUSED_PARAMETER(component_name); + ORT_UNUSED_PARAMETER(variant_name); + ORT_UNUSED_PARAMETER(out_ep); + RETURN_NOT_IMPL_IN_MINIMAL_BUILD(); +#endif + API_IMPL_END +} + +ORT_API_STATUS_IMPL(OrtModelPackageAPI::ModelPackage_GetSchemaVersion, + _In_ const OrtModelPackageContext* ctx, + _Out_ int64_t* out_version) { + API_IMPL_BEGIN +#if !defined(ORT_MINIMAL_BUILD) + if (ctx == nullptr || out_version == nullptr) { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "ctx and out_version must be non-null"); + } + + const auto& package_info = reinterpret_cast(ctx)->GetModelPackageInfo(); + *out_version = package_info.schema_version; + return nullptr; +#else + ORT_UNUSED_PARAMETER(ctx); + ORT_UNUSED_PARAMETER(out_version); + RETURN_NOT_IMPL_IN_MINIMAL_BUILD(); +#endif + API_IMPL_END +} + +ORT_API_STATUS_IMPL(OrtModelPackageAPI::ModelPackageComponent_GetSelectedVariantName, + _In_ const OrtModelPackageComponentContext* ctx, + _Outptr_ const char** out_name) { + API_IMPL_BEGIN +#if !defined(ORT_MINIMAL_BUILD) + if (ctx == nullptr || out_name == nullptr) { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "ctx and out_name must be non-null"); + } + + const auto* mp_ctx = reinterpret_cast(ctx); + const std::string* name = nullptr; + ORT_API_RETURN_IF_STATUS_NOT_OK(mp_ctx->GetSelectedVariantName(name)); + ORT_API_RETURN_IF(name == nullptr, ORT_FAIL, "Selected variant name is null."); + + *out_name = name->c_str(); + return nullptr; +#else + ORT_UNUSED_PARAMETER(ctx); + ORT_UNUSED_PARAMETER(out_name); + RETURN_NOT_IMPL_IN_MINIMAL_BUILD(); +#endif + API_IMPL_END +} + +// ---------- API table dispatch --------------------------------------------- + +static constexpr OrtModelPackageApi ort_model_package_api = { + // Options + &OrtModelPackageAPI::CreateModelPackageOptionsFromSessionOptions, + &OrtModelPackageAPI::ReleaseModelPackageOptions, + + // Context + &OrtModelPackageAPI::CreateModelPackageContext, + &OrtModelPackageAPI::ReleaseModelPackageContext, + + // Package-level queries + &OrtModelPackageAPI::ModelPackage_GetSchemaVersion, + &OrtModelPackageAPI::ModelPackage_GetComponentCount, + &OrtModelPackageAPI::ModelPackage_GetComponentNames, + &OrtModelPackageAPI::ModelPackage_GetVariantCount, + &OrtModelPackageAPI::ModelPackage_GetVariantNames, + &OrtModelPackageAPI::ModelPackage_GetVariantEpName, + + // Variant selection and component queries + &OrtModelPackageAPI::SelectComponent, + &OrtModelPackageAPI::ReleaseModelPackageComponentContext, + &OrtModelPackageAPI::ModelPackageComponent_GetSelectedVariantName, + &OrtModelPackageAPI::ModelPackageComponent_GetSelectedVariantFolderPath, + + // Session + &OrtModelPackageAPI::CreateSession, + + // End of Version 1.27 - DO NOT MODIFY ABOVE +}; + +static_assert(offsetof(OrtModelPackageApi, CreateSession) / sizeof(void*) == 14, + "Size of initial OrtModelPackageApi cannot change"); + +ORT_API(const OrtModelPackageApi*, OrtModelPackageAPI::GetModelPackageApi) { + return &ort_model_package_api; +} diff --git a/onnxruntime/core/session/model_package_api.h b/onnxruntime/core/session/model_package_api.h new file mode 100644 index 0000000000000..435e3a521c24c --- /dev/null +++ b/onnxruntime/core/session/model_package_api.h @@ -0,0 +1,76 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/session/onnxruntime_c_api.h" + +namespace OrtModelPackageAPI { + +ORT_API(const OrtModelPackageApi*, GetModelPackageApi); + +ORT_API(void, ReleaseModelPackageOptions, _Frees_ptr_opt_ OrtModelPackageOptions*); +ORT_API_STATUS_IMPL(CreateModelPackageOptionsFromSessionOptions, + _In_ const OrtEnv* env, + _In_ const OrtSessionOptions* session_options, + _Outptr_ OrtModelPackageOptions** out); + +ORT_API(void, ReleaseModelPackageContext, _Frees_ptr_opt_ OrtModelPackageContext*); +ORT_API_STATUS_IMPL(CreateModelPackageContext, + _In_ const ORTCHAR_T* package_root, + _Outptr_ OrtModelPackageContext** out); + +ORT_API_STATUS_IMPL(ModelPackage_GetComponentCount, + _In_ const OrtModelPackageContext* ctx, + _Out_ size_t* out_count); + +ORT_API_STATUS_IMPL(ModelPackage_GetComponentNames, + _In_ const OrtModelPackageContext* ctx, + _Outptr_result_buffer_maybenull_(*out_count) const char* const** out_names, + _Out_ size_t* out_count); + +ORT_API_STATUS_IMPL(ModelPackage_GetVariantCount, + _In_ const OrtModelPackageContext* ctx, + _In_ const char* component_name, + _Out_ size_t* out_count); + +ORT_API_STATUS_IMPL(ModelPackage_GetVariantNames, + _In_ const OrtModelPackageContext* ctx, + _In_ const char* component_name, + _Outptr_result_buffer_maybenull_(*out_count) const char* const** out_variant_names, + _Out_ size_t* out_count); + +ORT_API_STATUS_IMPL(SelectComponent, + _In_ const OrtModelPackageContext* context, + _In_ const char* component_name, + _In_ const OrtModelPackageOptions* options, + _Outptr_ OrtModelPackageComponentContext** out); + +ORT_API(void, ReleaseModelPackageComponentContext, + _Frees_ptr_opt_ OrtModelPackageComponentContext* ctx); + +ORT_API_STATUS_IMPL(ModelPackageComponent_GetSelectedVariantFolderPath, + _In_ const OrtModelPackageComponentContext* ctx, + _Outptr_ const ORTCHAR_T** folder_path); + +ORT_API_STATUS_IMPL(CreateSession, + _In_ const OrtEnv* env, + _In_ OrtModelPackageComponentContext* ctx, + _In_opt_ const OrtSessionOptions* session_options, + _Outptr_ OrtSession** session); + +ORT_API_STATUS_IMPL(ModelPackage_GetVariantEpName, + _In_ const OrtModelPackageContext* ctx, + _In_ const char* component_name, + _In_ const char* variant_name, + _Outptr_result_maybenull_ const char** out_ep); + +ORT_API_STATUS_IMPL(ModelPackage_GetSchemaVersion, + _In_ const OrtModelPackageContext* ctx, + _Out_ int64_t* out_version); + +ORT_API_STATUS_IMPL(ModelPackageComponent_GetSelectedVariantName, + _In_ const OrtModelPackageComponentContext* ctx, + _Outptr_ const char** out_name); + +} // namespace OrtModelPackageAPI diff --git a/onnxruntime/core/session/onnxruntime_c_api.cc b/onnxruntime/core/session/onnxruntime_c_api.cc index 549334564a1cf..474fa3e52bf0a 100644 --- a/onnxruntime/core/session/onnxruntime_c_api.cc +++ b/onnxruntime/core/session/onnxruntime_c_api.cc @@ -57,6 +57,7 @@ #include "core/session/ort_env.h" #include "core/session/ort_version_check.h" #include "core/session/utils.h" +#include "core/session/model_package_api.h" #if defined(USE_CUDA) || defined(USE_CUDA_PROVIDER_INTERFACE) #include "core/providers/cuda/cuda_provider_factory.h" @@ -3709,6 +3710,10 @@ ORT_API(const OrtCompileApi*, OrtApis::GetCompileApi) { return OrtCompileAPI::GetCompileApi(); } +ORT_API(const OrtModelPackageApi*, OrtApis::GetModelPackageApi) { + return OrtModelPackageAPI::GetModelPackageApi(); +} + ORT_API(void, OrtApis::CreateKeyValuePairs, _Outptr_ OrtKeyValuePairs** out) { auto kvps = std::make_unique(); *out = reinterpret_cast(kvps.release()); @@ -4923,9 +4928,12 @@ static constexpr OrtApi ort_api_1_to_27 = { &OrtApis::SetPerSessionThreadPoolCallbacks, // End of Version 25 - DO NOT MODIFY ABOVE (see above text for more information) // End of Version 26 - DO NOT MODIFY ABOVE (see above text for more information) + &OrtApis::GetMemPatternEnabled, &OrtApis::GetSessionExecutionMode, &OrtApis::SessionReleaseCapturedGraph, + &OrtApis::GetModelPackageApi, + // End of Version 27 - DO NOT MODIFY ABOVE (see above text for more information) }; // OrtApiBase can never change as there is no way to know what version of OrtApiBase is returned by OrtGetApiBase. diff --git a/onnxruntime/core/session/ort_apis.h b/onnxruntime/core/session/ort_apis.h index adccfe09bc3f7..3f039107ac17e 100644 --- a/onnxruntime/core/session/ort_apis.h +++ b/onnxruntime/core/session/ort_apis.h @@ -825,4 +825,6 @@ ORT_API_STATUS_IMPL(GetTensorElementTypeAndShapeDataReference, _In_ const OrtVal _Outptr_result_maybenull_ const int64_t** shape_data, _Out_ size_t* shape_data_count); +ORT_API(const OrtModelPackageApi*, GetModelPackageApi); + } // namespace OrtApis diff --git a/onnxruntime/core/session/utils.cc b/onnxruntime/core/session/utils.cc index cd2a01b932674..330974aeed8d8 100644 --- a/onnxruntime/core/session/utils.cc +++ b/onnxruntime/core/session/utils.cc @@ -27,6 +27,7 @@ #include "core/session/ort_env.h" #include "core/session/onnxruntime_ep_device_ep_metadata_keys.h" #include "core/session/model_package/model_package_context.h" +#include "core/session/model_package/model_package_options.h" #if !defined(ORT_MINIMAL_BUILD) #include "core/graph/model_editor_api_types.h" @@ -107,115 +108,6 @@ Status TestAutoSelectEPsImpl(const Environment& env, InferenceSession& sess, con return Status::OK(); } -Status PrintAvailableAndSelectedEpInfos(const Environment& env, std::vector& ep_infos) { - const auto& execution_devices = env.GetOrtEpDevices(); - - std::string available_eps_info = "Available EPs and devices:\n"; - if (execution_devices.empty()) { - available_eps_info += " (none)\n"; - } else { - for (const auto* ep_device : execution_devices) { - if (ep_device == nullptr) { - continue; - } - - available_eps_info += " " + ep_device->ToString() + "\n"; - } - } - - std::string selected_eps_info = "Selected EPs:\n"; - if (ep_infos.empty()) { - selected_eps_info += " (none)\n"; - } else { - for (const auto& ep_info : ep_infos) { - selected_eps_info += " EP: " + ep_info.ep_name + "\n"; - const auto& selected_ep_devices = ep_info.ep_devices; - for (const auto* selected_ep_device : selected_ep_devices) { - if (selected_ep_device == nullptr) { - continue; - } - selected_eps_info += " " + selected_ep_device->ToString() + "\n"; - } - } - } - - LOGS_DEFAULT(INFO) << available_eps_info; - LOGS_DEFAULT(INFO) << selected_eps_info; - return Status::OK(); -} - -// Gets EP info needed for model package workflow to select suitable model. -// -// For simplicity, there are some constraints in this initial implementation: -// - Only one EP is supported, skip ORT CPU EP. -// - All devices should be supported by the same EP -// -Status GetVariantSelectionEpInfo(const OrtSessionOptions* session_options, - std::vector>& provider_list, - std::vector& ep_infos) { - if (provider_list.empty()) { - return Status::OK(); - } - - // Pick the first non-CPU provider if available; otherwise fall back to the first provider. - size_t selected_idx = 0; - for (size_t i = 0; i < provider_list.size(); ++i) { - const auto& provider = provider_list[i]; - if (provider && provider->Type() != onnxruntime::kCpuExecutionProvider) { - selected_idx = i; - break; - } - } - - auto& provider = provider_list[selected_idx]; - - if (provider && provider->Type() == onnxruntime::kCpuExecutionProvider) { - return Status::OK(); - } - - ep_infos.push_back(VariantSelectionEpInfo{}); - auto& ep_info = ep_infos.back(); - - // Add ep name to ep_info - ep_info.ep_name = provider->Type(); - ORT_ENFORCE(!ep_info.ep_name.empty(), "EP name should have been set at this point."); - - // Add ep devices to ep_info - auto& ep_devices = provider->GetEpDevices(); - ep_info.ep_devices = ep_devices; - - // Add ep factory to ep_info - ep_info.ep_factory = ep_devices.empty() ? nullptr : ep_devices.front()->ep_factory; - - // Add hardware devices to ep_info - ep_info.hardware_devices.reserve(ep_devices.size()); - for (const auto& ep_device : ep_devices) { - if (ep_device->device != nullptr) { - ep_info.hardware_devices.push_back(ep_device->device); - } - } - - // Add ep metadata to ep_info - ep_info.ep_metadata.reserve(ep_devices.size()); - for (const auto& ep_device : ep_devices) { - ep_info.ep_metadata.push_back(&ep_device->ep_metadata); - } - - // Add ep provider options to ep_info - ProviderOptions provider_options; - const std::string ep_options_prefix = OrtSessionOptions::GetProviderOptionPrefix(ep_info.ep_name.c_str()); - const auto& configs = session_options->value.config_options.configurations; - - for (const auto& kv : configs) { - if (kv.first.rfind(ep_options_prefix, 0) == 0) { // starts with prefix - provider_options.emplace(kv.first.substr(ep_options_prefix.size()), kv.second); // strip prefix - } - } - ep_info.ep_options = std::move(provider_options); - - return Status::OK(); -} - Status GetCustomOpDomainsFromEpDevice(const OrtEpDevice& ep_device, InlinedVector& domains_out) { InlinedVector domains{}; @@ -466,14 +358,13 @@ static OrtStatus* CreateSessionAndLoadModelImpl(_In_ const OrtSessionOptions* op const ORTCHAR_T* model_path_to_use = model_path; // keep storage alive if ORT selects a model variant. - std::filesystem::path selected_model_path; + std::filesystem::path selected_model_variant_path; if (model_path_to_use != nullptr) { std::error_code ec; std::filesystem::path package_root{model_path_to_use}; - if (std::filesystem::is_directory(package_root, ec) && - !ec) { + if (std::filesystem::is_directory(package_root, ec) && !ec) { #if !defined(ORT_MINIMAL_BUILD) OrtSessionOptions* options_to_use = nullptr; OrtSessionOptions ort_sess_options = options ? *options : OrtSessionOptions(); @@ -505,7 +396,7 @@ static OrtStatus* CreateSessionAndLoadModelImpl(_In_ const OrtSessionOptions* op // Build EP info from finalized providers. std::vector ep_infos; - ORT_API_RETURN_IF_STATUS_NOT_OK(GetVariantSelectionEpInfo(options_to_use, provider_list, ep_infos)); + ORT_API_RETURN_IF_STATUS_NOT_OK(GetVariantSelectionEpInfo(provider_list, ep_infos)); ORT_API_RETURN_IF_STATUS_NOT_OK(PrintAvailableAndSelectedEpInfos(env, ep_infos)); @@ -517,21 +408,27 @@ static OrtStatus* CreateSessionAndLoadModelImpl(_In_ const OrtSessionOptions* op // Select the most suitable model variant based on EP info and model constraints. ModelPackageContext model_package_context(package_root); - ModelVariantSelector model_variant_selector; - std::optional selected_model_variant_path; + const auto& package_info = model_package_context.GetModelPackageInfo(); + const ComponentInfo* component_info = nullptr; - ORT_API_RETURN_IF_STATUS_NOT_OK(model_variant_selector.SelectVariant(model_package_context, ep_infos, selected_model_variant_path)); - - if (selected_model_variant_path.has_value()) { - selected_model_path = *selected_model_variant_path; - model_path_to_use = selected_model_path.c_str(); - } else { + if (package_info.components.empty()) { + return OrtApis::CreateStatus(ORT_FAIL, "No component models found in the model package."); + } else if (package_info.components.size() > 1) { return OrtApis::CreateStatus(ORT_FAIL, - "No suitable model variant found for the available execution providers." - "Try specifying the model file path instead of a model package, or check the " - "model variants' constraints in the manifest json or metadata json."); + "Multiple component models found in the model package. " + "Currently only single component model is supported."); + } + + component_info = &package_info.components[0]; + if (component_info == nullptr) { + return OrtApis::CreateStatus(ORT_INVALID_ARGUMENT, "Component model not found."); } + ModelPackageComponentContext component_context(component_info->component_name, *component_info, ep_infos); + ORT_API_RETURN_IF_STATUS_NOT_OK(component_context.ResolveVariant()); + ORT_API_RETURN_IF_STATUS_NOT_OK(component_context.GetSelectedVariantFilePath(selected_model_variant_path)); + model_path_to_use = selected_model_variant_path.c_str(); + ORT_API_RETURN_IF_ERROR(CreateSessionAndLoadSingleModelImpl(options_to_use, env, model_path_to_use, model_data, model_data_length, sess)); @@ -549,6 +446,7 @@ static OrtStatus* CreateSessionAndLoadModelImpl(_In_ const OrtSessionOptions* op ORT_API_RETURN_IF_STATUS_NOT_OK(provider_policy_context.LogTelemetry(*sess, *options_to_use, execution_devices, devices_selected)); } + #else return OrtApis::CreateStatus(ORT_FAIL, "Model package is not supported in this build."); #endif @@ -976,5 +874,142 @@ Status AddEpCustomDomainsToSessionOptions(gsl::span ep return Status::OK(); } + +Status PrintAvailableAndSelectedEpInfos(const Environment& env, std::vector& ep_infos) { + const auto& execution_devices = env.GetOrtEpDevices(); + + std::string available_eps_info = "Available EPs and devices:\n"; + if (execution_devices.empty()) { + available_eps_info += " (none)\n"; + } else { + for (const auto* ep_device : execution_devices) { + if (ep_device == nullptr) { + continue; + } + + available_eps_info += " " + ep_device->ToString() + "\n"; + } + } + + std::string selected_eps_info = "Selected EPs:\n"; + if (ep_infos.empty()) { + selected_eps_info += " (none)\n"; + } else { + for (const auto& ep_info : ep_infos) { + selected_eps_info += " EP: " + ep_info.ep_name + "\n"; + const auto& selected_ep_devices = ep_info.ep_devices; + for (const auto* selected_ep_device : selected_ep_devices) { + if (selected_ep_device == nullptr) { + continue; + } + selected_eps_info += " " + selected_ep_device->ToString() + "\n"; + } + } + } + + LOGS_DEFAULT(INFO) << available_eps_info; + LOGS_DEFAULT(INFO) << selected_eps_info; + return Status::OK(); +} + +// Gets EP info needed for model package workflow to select suitable model. +// +// For simplicity, there are some constraints in this initial implementation: +// - Only one EP is supported, skip ORT CPU EP. +// - All devices should be supported by the same EP +// +Status GetVariantSelectionEpInfo(std::vector>& provider_list, + std::vector& ep_infos) { + if (provider_list.empty()) { + return Status::OK(); + } + + // Use the first provider in the list for variant selection. + auto& provider = provider_list[0]; + if (!provider) { + return Status::OK(); + } + + ep_infos.push_back(VariantSelectionEpInfo{}); + auto& ep_info = ep_infos.back(); + + // Add ep name to ep_info + ep_info.ep_name = provider->Type(); + ORT_ENFORCE(!ep_info.ep_name.empty(), "EP name should have been set at this point."); + + // CPU is built-in and needs no device/factory metadata for variant selection. + if (ep_info.ep_name == onnxruntime::kCpuExecutionProvider) { + return Status::OK(); + } + + // Add ep devices to ep_info + auto& ep_devices = provider->GetEpDevices(); + ep_info.ep_devices = ep_devices; + + // Add ep factory to ep_info + ep_info.ep_factory = ep_devices.empty() ? nullptr : ep_devices.front()->ep_factory; + + // Add hardware devices to ep_info + ep_info.hardware_devices.reserve(ep_devices.size()); + for (const auto& ep_device : ep_devices) { + if (ep_device->device != nullptr) { + ep_info.hardware_devices.push_back(ep_device->device); + } + } + + // Add ep metadata to ep_info + ep_info.ep_metadata.reserve(ep_devices.size()); + for (const auto& ep_device : ep_devices) { + ep_info.ep_metadata.push_back(&ep_device->ep_metadata); + } + + return Status::OK(); +} + +// Create session for model package workflow. +// +// Preconditions: caller has already +// 1. resolved EP selection -> provider_list (owns the IExecutionProvider instances), +// 2. selected a model variant -> selected_model_path. +// +// This function: +// a. creates and loads an InferenceSession for selected_model_path, +// b. registers the providers from provider_list (moves them into the session), +// c. optionally logs auto-EP-selection telemetry when from_policy is true. +OrtStatus* CreateSessionForModelPackage(_In_ const OrtSessionOptions* options, + const onnxruntime::Environment& env, + const std::filesystem::path& selected_model_path, + onnxruntime::ModelPackageComponentContext& model_package_context, + std::unique_ptr& sess) { + ORT_API_RETURN_IF_ERROR(CreateSessionAndLoadSingleModelImpl(options, env, + selected_model_path.c_str(), + /*model_data*/ nullptr, + /*model_data_length*/ 0, + sess)); + + // Always rebuild providers from the effective session options (which include merged variant + // provider options). Providers created during EP selection used the original session options + // and would not reflect variant-specific provider options. + ORT_API_RETURN_IF_STATUS_NOT_OK(model_package_context.RebuildProviderListForSession(env, *options)); + + auto& provider_list = model_package_context.MutableProviderList(); + + for (auto& provider : provider_list) { + if (provider) { + ORT_API_RETURN_IF_STATUS_NOT_OK(sess->RegisterExecutionProvider(std::move(provider))); + } + } + + if (model_package_context.IsFromPolicy() && options != nullptr) { + ProviderPolicyContext provider_policy_context; + ORT_API_RETURN_IF_STATUS_NOT_OK(provider_policy_context.LogTelemetry( + *sess, *options, + model_package_context.ExecutionDevices(), + model_package_context.DevicesSelected())); + } + + return nullptr; +} + #endif // !defined(ORT_MINIMAL_BUILD) } // namespace onnxruntime diff --git a/onnxruntime/core/session/utils.h b/onnxruntime/core/session/utils.h index 59b4d9f0944c3..6b00d51374ba6 100644 --- a/onnxruntime/core/session/utils.h +++ b/onnxruntime/core/session/utils.h @@ -3,6 +3,7 @@ #pragma once +#include #include #include #include @@ -25,8 +26,11 @@ namespace onnxruntime { class Environment; class EpLibrary; class EpFactoryInternal; +class IExecutionProvider; +class ModelPackageComponentContext; struct IExecutionProviderFactory; struct SessionOptions; +struct VariantSelectionEpInfo; } // namespace onnxruntime #endif // !defined(ORT_MINIMAL_BUILD) @@ -75,5 +79,27 @@ Status AddEpOptionsToSessionOptions(gsl::span ep_devic Status AddEpCustomDomainsToSessionOptions(gsl::span ep_devices, OrtSessionOptions& ort_session_options); +// Builds VariantSelectionEpInfo entries from already-created IExecutionProvider instances. +// Same logic used by the standard CreateSession path for model package workflows; exposed so +// the model package API can pre-resolve EP selection (see ModelPackageOptions::ResolveEpSelection). +// +/// Constraints (matching the standard path): +// - Only one EP is selected (CPU EP is skipped in favor of the first non-CPU EP if available). +// - All devices are expected to be supported by the same EP. +Status GetVariantSelectionEpInfo(std::vector>& provider_list, + std::vector& ep_infos); + +// Logs available environment EP devices and the ones selected for variant selection. Informational only. +Status PrintAvailableAndSelectedEpInfos(const Environment& env, + std::vector& ep_infos); + +// Create session for model package workflow. +OrtStatus* CreateSessionForModelPackage( + _In_ const OrtSessionOptions* options, + const onnxruntime::Environment& env, + const std::filesystem::path& selected_model_path, + onnxruntime::ModelPackageComponentContext& model_package_context, + std::unique_ptr& sess); + } // namespace onnxruntime #endif // !defined(ORT_MINIMAL_BUILD) diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index 946059558e315..775cef4485bde 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -3248,6 +3248,195 @@ including arg name, arg type (contains both type and shape).)pbdoc") #endif }, R"pbdoc(Compile an ONNX model into an output stream using the provided write functor.)pbdoc"); + + // --- Model Package API --- +#if !defined(ORT_MINIMAL_BUILD) + // Helper to create a PyInferenceSession from a pre-initialized OrtSession* (C API handle). + // PyInferenceSession's owning ctor is protected; this subclass provides access. + struct PyModelPackageSession : PyInferenceSession { + PyModelPackageSession(std::unique_ptr sess) + : PyInferenceSession(std::move(sess)) {} + }; + + // Wrapper classes to manage opaque C handles with proper RAII + struct PyModelPackageContext { + OrtModelPackageContext* ctx_{nullptr}; + PyModelPackageContext(const std::string& package_path) { + auto path = ToPathString(package_path); + const auto* api = Ort::GetApi().GetModelPackageApi(); + Ort::ThrowOnError(api->CreateModelPackageContext(path.c_str(), &ctx_)); + } + ~PyModelPackageContext() { + if (ctx_) { + const auto* api = Ort::GetApi().GetModelPackageApi(); + api->ReleaseModelPackageContext(ctx_); + } + } + PyModelPackageContext(const PyModelPackageContext&) = delete; + PyModelPackageContext& operator=(const PyModelPackageContext&) = delete; + }; + + struct PyModelPackageComponentContext { + OrtModelPackageComponentContext* ctx_{nullptr}; + ~PyModelPackageComponentContext() { + if (ctx_) { + const auto* api = Ort::GetApi().GetModelPackageApi(); + api->ReleaseModelPackageComponentContext(ctx_); + } + } + PyModelPackageComponentContext(const PyModelPackageComponentContext&) = delete; + PyModelPackageComponentContext& operator=(const PyModelPackageComponentContext&) = delete; + PyModelPackageComponentContext() = default; + }; + + struct PyModelPackageOptions { + OrtModelPackageOptions* opts_{nullptr}; + ~PyModelPackageOptions() { + if (opts_) { + const auto* api = Ort::GetApi().GetModelPackageApi(); + api->ReleaseModelPackageOptions(opts_); + } + } + PyModelPackageOptions(const PyModelPackageOptions&) = delete; + PyModelPackageOptions& operator=(const PyModelPackageOptions&) = delete; + PyModelPackageOptions() = default; + }; + + py::class_(m, "ModelPackageContext", + R"pbdoc(Represents an opened model package for inspection and component selection.)pbdoc") + .def(py::init(), py::arg("package_path"), + R"pbdoc(Open a model package from the given directory path.)pbdoc") + .def( + "get_component_names", + [](PyModelPackageContext& self) -> std::vector { + const auto* api = Ort::GetApi().GetModelPackageApi(); + const char* const* names = nullptr; + size_t count = 0; + Ort::ThrowOnError(api->ModelPackage_GetComponentNames(self.ctx_, &names, &count)); + std::vector result; + result.reserve(count); + for (size_t i = 0; i < count; ++i) { + result.emplace_back(names[i]); + } + return result; + }, + R"pbdoc(Get the names of all components in the package.)pbdoc") + .def( + "get_variant_names", + [](PyModelPackageContext& self, const std::string& component_name) -> std::vector { + const auto* api = Ort::GetApi().GetModelPackageApi(); + const char* const* names = nullptr; + size_t count = 0; + Ort::ThrowOnError(api->ModelPackage_GetVariantNames( + self.ctx_, component_name.c_str(), &names, &count)); + std::vector result; + result.reserve(count); + for (size_t i = 0; i < count; ++i) { + result.emplace_back(names[i]); + } + return result; + }, + py::arg("component_name"), + R"pbdoc(Get the variant names for a given component.)pbdoc") + .def( + "get_variant_ep_name", + [](PyModelPackageContext& self, const std::string& component_name, + const std::string& variant_name) -> std::optional { + const auto* api = Ort::GetApi().GetModelPackageApi(); + const char* ep = nullptr; + Ort::ThrowOnError(api->ModelPackage_GetVariantEpName( + self.ctx_, component_name.c_str(), variant_name.c_str(), &ep)); + if (ep) return std::string(ep); + return std::nullopt; + }, + py::arg("component_name"), py::arg("variant_name"), + R"pbdoc(Get the EP name for a variant. Returns None if not declared.)pbdoc") + .def( + "get_schema_version", + [](PyModelPackageContext& self) -> int64_t { + const auto* api = Ort::GetApi().GetModelPackageApi(); + int64_t version = 0; + Ort::ThrowOnError(api->ModelPackage_GetSchemaVersion(self.ctx_, &version)); + return version; + }, + R"pbdoc(Get the schema version declared in the model package manifest.)pbdoc") + .def( + "select_component", + [](PyModelPackageContext& self, const std::string& component_name, + PyModelPackageOptions& options) -> std::unique_ptr { + const auto* api = Ort::GetApi().GetModelPackageApi(); + auto result = std::make_unique(); + Ort::ThrowOnError(api->SelectComponent( + self.ctx_, component_name.c_str(), options.opts_, &result->ctx_)); + return result; + }, + py::arg("component_name"), py::arg("options"), + R"pbdoc(Select a component and resolve its variant based on the provided options. +Returns a ModelPackageComponentContext for inspecting the selected variant.)pbdoc"); + + py::class_(m, "ModelPackageOptions", + R"pbdoc(Options used for variant selection in a model package. +Created from a SessionOptions to capture EP configuration for variant matching.)pbdoc") + .def(py::init([](PySessionOptions& session_options) { + const auto* api = Ort::GetApi().GetModelPackageApi(); + auto result = std::make_unique(); + Ort::ThrowOnError(api->CreateModelPackageOptionsFromSessionOptions( + GetOrtEnv(), &session_options, &result->opts_)); + return result; + }), + py::arg("session_options"), + R"pbdoc(Create model package options from a SessionOptions instance. +The EP configured on the session options is used for variant selection.)pbdoc"); + + py::class_(m, "ModelPackageComponentContext", + R"pbdoc(Represents a selected component within a model package. +Provides access to the resolved variant's files, session options, and metadata.)pbdoc") + .def( + "get_selected_variant_folder_path", + [](PyModelPackageComponentContext& self) -> std::string { + const auto* api = Ort::GetApi().GetModelPackageApi(); + const ORTCHAR_T* path = nullptr; + Ort::ThrowOnError(api->ModelPackageComponent_GetSelectedVariantFolderPath(self.ctx_, &path)); + return PathToUTF8String(PathString(path)); + }, + R"pbdoc(Get the folder path of the selected variant.)pbdoc") + .def( + "get_selected_variant_name", + [](PyModelPackageComponentContext& self) -> std::string { + const auto* api = Ort::GetApi().GetModelPackageApi(); + const char* name = nullptr; + Ort::ThrowOnError(api->ModelPackageComponent_GetSelectedVariantName( + self.ctx_, &name)); + return name ? std::string(name) : std::string(); + }, + R"pbdoc(Get the name of the selected variant.)pbdoc") + .def( + "create_session", + [](PyModelPackageComponentContext& self, py::object session_options_obj) -> std::unique_ptr { + const auto* api = Ort::GetApi().GetModelPackageApi(); + OrtSession* ort_session = nullptr; + if (session_options_obj.is_none()) { + Ort::ThrowOnError(api->CreateSession(GetOrtEnv(), self.ctx_, nullptr, &ort_session)); + } else { + auto& so = session_options_obj.cast(); + Ort::ThrowOnError(api->CreateSession(GetOrtEnv(), self.ctx_, &so, &ort_session)); + } + // OrtSession* is a reinterpret_cast of InferenceSession* + auto* inference_session = reinterpret_cast(ort_session); + std::unique_ptr session_ptr(inference_session); + return std::make_unique(std::move(session_ptr)); + }, + py::arg("session_options") = py::none(), + R"pbdoc(Create an InferenceSession from the selected component variant. + +Args: + session_options: Optional SessionOptions override. If None, uses the options + captured during variant selection with per-file options merged on top. + If provided, variant-specific options are NOT applied. + +Returns: + An InferenceSession ready for inference.)pbdoc"); +#endif // !defined(ORT_MINIMAL_BUILD) } bool InitArray() { diff --git a/onnxruntime/test/autoep/test_model_package.cc b/onnxruntime/test/autoep/test_model_package.cc index e3cf87b5f83d3..6fb3f8e6ba82f 100644 --- a/onnxruntime/test/autoep/test_model_package.cc +++ b/onnxruntime/test/autoep/test_model_package.cc @@ -12,7 +12,6 @@ #include "gtest/gtest.h" #include "core/session/model_package/model_package_context.h" -#include "core/session/model_package/model_package_descriptor_parser.h" #include "core/session/abi_devices.h" #include "test/autoep/test_autoep_utils.h" #include "test/util/include/asserts.h" @@ -37,10 +36,30 @@ std::filesystem::path CreateManifestJson(const std::filesystem::path& package_ro return manifest_path; } +std::string MakeVariantJson(std::string_view filename) { + std::ostringstream oss; + oss << R"({ + "filename": ")" + << filename << R"(" + })"; + return oss.str(); +} + +void CreateVariantDescriptor(const std::filesystem::path& package_root, + std::string_view component_name, + std::string_view variant_name, + std::string_view variant_json) { + const auto variant_root = package_root / "models" / std::string(component_name) / std::string(variant_name); + std::filesystem::create_directories(variant_root); + + std::ofstream os(variant_root / "variant.json", std::ios::binary); + os << variant_json; +} + std::filesystem::path CreateModelPackage( const std::filesystem::path& package_root, std::string_view manifest_json, - std::string_view component_model_name, + std::string_view component_name, std::string_view variant_name_1, std::string_view variant_name_2, const std::filesystem::path& source_model_1, @@ -51,9 +70,9 @@ std::filesystem::path CreateModelPackage( CreateManifestJson(package_root, manifest_json); - const auto models_root = package_root / "models" / component_model_name; - const auto variant1_dir = models_root / variant_name_1; - const auto variant2_dir = models_root / variant_name_2; + const auto models_root = package_root / "models" / std::string(component_name); + const auto variant1_dir = models_root / std::string(variant_name_1); + const auto variant2_dir = models_root / std::string(variant_name_2); std::filesystem::create_directories(variant1_dir); std::filesystem::create_directories(variant2_dir); @@ -63,17 +82,21 @@ std::filesystem::path CreateModelPackage( std::filesystem::copy_file(source_model_1, variant1_model, std::filesystem::copy_options::overwrite_existing, ec); std::filesystem::copy_file(source_model_2, variant2_model, std::filesystem::copy_options::overwrite_existing, ec); + + CreateVariantDescriptor(package_root, component_name, variant_name_1, + MakeVariantJson(source_model_1.filename().string())); + CreateVariantDescriptor(package_root, component_name, variant_name_2, + MakeVariantJson(source_model_2.filename().string())); + return package_root; } std::filesystem::path CreateComponentModelMetadata( const std::filesystem::path& package_root, - std::string_view component_model_name, + std::string_view component_name, std::string_view metadata_json) { - const auto component_root = package_root / "models" / component_model_name; - std::error_code ec; + const auto component_root = package_root / "models" / std::string(component_name); - // Ensure component root and metadata.json exist std::filesystem::create_directories(component_root); const std::filesystem::path metadata_path = component_root / "metadata.json"; @@ -83,24 +106,253 @@ std::filesystem::path CreateComponentModelMetadata( return component_root; } -std::string MakeManifestJson(std::string_view model_name, - std::string_view component_model_name) { +std::string MakeManifestJson(std::string_view component_name) { + std::ostringstream oss; + oss << R"({ + "schema_version": 1, + "components": [")" + << component_name << R"("] + })"; + return oss.str(); +} + +std::string MakeMetadataJsonTwoVariants(std::string_view component_name, + std::string_view variant_name_1, + std::string_view variant_ep_1, + std::string_view variant_device_1, + std::string_view variant_compatibility_string_1, + std::string_view variant_name_2, + std::string_view variant_ep_2, + std::string_view variant_device_2, + std::string_view variant_compatibility_string_2) { std::ostringstream oss; oss << R"({ - "model_name": ")" - << model_name << R"(", - "model_version": "1.0", - "component_models": [")" - << component_model_name << R"("] + "component_name": ")" + << component_name << R"(", + "variants": { + ")" + << variant_name_1 << R"(": { + "ep": ")" + << variant_ep_1 << R"(", + "device": ")" + << variant_device_1 << R"(", + "compatibility_string": ")" + << variant_compatibility_string_1 << R"(" + }, + ")" + << variant_name_2 << R"(": { + "ep": ")" + << variant_ep_2 << R"(", + "device": ")" + << variant_device_2 << R"(", + "compatibility_string": ")" + << variant_compatibility_string_2 << R"(" + } + } })"; return oss.str(); } +std::filesystem::path CreateModelPackageApiTestPackage(bool multi_file_variant = false) { + const auto package_root = std::filesystem::temp_directory_path() / "ort_model_package_api_test"; + std::error_code ec; + std::filesystem::remove_all(package_root, ec); + + constexpr std::string_view manifest_json = R"({ + "schema_version": 1, + "components": ["model_1"] + })"; + + CreateModelPackage(package_root, manifest_json, + "model_1", "variant_1", "variant_2", + std::filesystem::path{"testdata/mul_1.onnx"}, std::filesystem::path{"testdata/mul_16.onnx"}); + + constexpr std::string_view metadata_json = R"({ + "component_name": "model_1", + "variants": { + "variant_1": { + "ep": "example_ep", + "device": "cpu", + "compatibility_string": "example_ep;version=0.1.0;ort_api_version=25;hardware_architecture=arch1" + }, + "variant_2": { + "ep": "example_ep", + "device": "npu", + "compatibility_string": "example_ep;version=0.1.0;ort_api_version=25;hardware_architecture=arch2" + } + } + })"; + + CreateComponentModelMetadata(package_root, "model_1", metadata_json); + + if (!multi_file_variant) { + std::ofstream os(package_root / "models" / "model_1" / "variant_1" / "variant.json", std::ios::binary); + os << R"({ + "filename": "mul_1.onnx", + "session_options": { + "session.disable_prepacking": "1", + "session.intra_op.allow_spinning": "0" + }, + "provider_options": { + "backend_path": "example_backend", + "enable_htp": "1" + } + })"; + } else { + // Multi-file variants are no longer supported. For backward-compat testing, + // just write a single-file variant.json. + std::ofstream os(package_root / "models" / "model_1" / "variant_1" / "variant.json", std::ios::binary); + os << R"({ + "filename": "mul_1.onnx", + "session_options": { + "session.disable_prepacking": "1", + "session.intra_op.allow_spinning": "0" + }, + "provider_options": { + "backend_path": "example_backend", + "enable_htp": "1" + } + })"; + } + + { + std::ofstream os(package_root / "models" / "model_1" / "variant_2" / "variant.json", std::ios::binary); + os << R"({ + "filename": "mul_16.onnx" + })"; + } + + return package_root; +} + } // namespace // ------------------------------------------------------------------ -// Model package end-to-end test +// Model Package API tests // ------------------------------------------------------------------ +TEST(ModelPackageApiTest, PackageContextQueries) { + const auto package_root = CreateModelPackageApiTestPackage(); + + const OrtModelPackageApi* pkg_api = Ort::GetApi().GetModelPackageApi(); + ASSERT_NE(pkg_api, nullptr); + + auto context_deleter = [pkg_api](OrtModelPackageContext* p) { + if (p) pkg_api->ReleaseModelPackageContext(p); + }; + std::unique_ptr model_pkg_context(nullptr, context_deleter); + + OrtModelPackageContext* raw_context = nullptr; + ASSERT_ORTSTATUS_OK(pkg_api->CreateModelPackageContext(package_root.c_str(), &raw_context)); + model_pkg_context.reset(raw_context); + + // Query: component count + names + size_t component_count = 0; + ASSERT_ORTSTATUS_OK(pkg_api->ModelPackage_GetComponentCount(model_pkg_context.get(), &component_count)); + ASSERT_EQ(component_count, 1u); + + const char* const* component_names = nullptr; + size_t component_name_count = 0; + ASSERT_ORTSTATUS_OK(pkg_api->ModelPackage_GetComponentNames( + model_pkg_context.get(), &component_names, &component_name_count)); + ASSERT_EQ(component_name_count, 1u); + ASSERT_NE(component_names, nullptr); + ASSERT_NE(component_names[0], nullptr); + EXPECT_STREQ(component_names[0], "model_1"); + + // Query: variant count + names + size_t variant_count = 0; + ASSERT_ORTSTATUS_OK(pkg_api->ModelPackage_GetVariantCount( + model_pkg_context.get(), "model_1", &variant_count)); + ASSERT_EQ(variant_count, 2u); + + const char* const* variant_names = nullptr; + size_t variant_name_count = 0; + ASSERT_ORTSTATUS_OK(pkg_api->ModelPackage_GetVariantNames( + model_pkg_context.get(), "model_1", &variant_names, &variant_name_count)); + ASSERT_EQ(variant_name_count, 2u); + + std::unordered_set variant_name_set; + for (size_t i = 0; i < variant_name_count; ++i) { + ASSERT_NE(variant_names[i], nullptr); + variant_name_set.insert(variant_names[i]); + } + EXPECT_EQ(variant_name_set.count("variant_1"), 1u); + EXPECT_EQ(variant_name_set.count("variant_2"), 1u); + + std::error_code ec; + std::filesystem::remove_all(package_root, ec); +} + +TEST(ModelPackageApiTest, SingleFileVariantInComponent_SelectComponentAndCreateSession) { + const auto package_root = CreateModelPackageApiTestPackage(); + + RegisteredEpDeviceUniquePtr example_ep; + ASSERT_NO_FATAL_FAILURE(Utils::RegisterAndGetExampleEp(*ort_env, Utils::example_ep_info, example_ep)); + Ort::ConstEpDevice plugin_ep_device(example_ep.get()); + + Ort::SessionOptions session_options; + std::unordered_map ep_options; + session_options.AppendExecutionProvider_V2(*ort_env, {plugin_ep_device}, ep_options); + + const OrtModelPackageApi* pkg_api = Ort::GetApi().GetModelPackageApi(); + ASSERT_NE(pkg_api, nullptr); + + auto options_deleter = [pkg_api](OrtModelPackageOptions* p) { + if (p) pkg_api->ReleaseModelPackageOptions(p); + }; + auto context_deleter = [pkg_api](OrtModelPackageContext* p) { + if (p) pkg_api->ReleaseModelPackageContext(p); + }; + auto component_context_deleter = [pkg_api](OrtModelPackageComponentContext* p) { + if (p) pkg_api->ReleaseModelPackageComponentContext(p); + }; + + std::unique_ptr model_pkg_options(nullptr, options_deleter); + std::unique_ptr model_pkg_context(nullptr, context_deleter); + std::unique_ptr component_context(nullptr, component_context_deleter); + + OrtModelPackageOptions* raw_options = nullptr; + ASSERT_ORTSTATUS_OK(pkg_api->CreateModelPackageOptionsFromSessionOptions(*ort_env, session_options, &raw_options)); + model_pkg_options.reset(raw_options); + + OrtModelPackageContext* raw_context = nullptr; + ASSERT_ORTSTATUS_OK(pkg_api->CreateModelPackageContext(package_root.c_str(), &raw_context)); + model_pkg_context.reset(raw_context); + + OrtModelPackageComponentContext* raw_component_context = nullptr; + ASSERT_ORTSTATUS_OK(pkg_api->SelectComponent(model_pkg_context.get(), + "model_1", + model_pkg_options.get(), + &raw_component_context)); + component_context.reset(raw_component_context); + + OrtSession* raw_session = nullptr; + ASSERT_ORTSTATUS_OK(pkg_api->CreateSession(*ort_env, + component_context.get(), + session_options, + &raw_session)); + Ort::Session session(raw_session); + + Ort::MemoryInfo memory_info = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU); + std::vector shape = {3, 2}; + std::vector input_data = {1.f, 2.f, 3.f, 4.f, 5.f, 6.f}; + Ort::Value input = Ort::Value::CreateTensor(memory_info, input_data.data(), input_data.size(), + shape.data(), shape.size()); + const char* input_names[] = {"X"}; + const char* output_names[] = {"Y"}; + std::vector inputs; + inputs.push_back(std::move(input)); + + auto outputs = session.Run(Ort::RunOptions{nullptr}, input_names, inputs.data(), inputs.size(), output_names, 1); + ASSERT_EQ(outputs.size(), 1u); + const float* out = outputs[0].GetTensorData(); + gsl::span out_span(out, input_data.size()); + EXPECT_THAT(out_span, ::testing::ElementsAre(1.f, 4.f, 9.f, 16.f, 25.f, 36.f)); + + std::error_code ec; + std::filesystem::remove_all(package_root, ec); +} + TEST(ModelPackageTest, LoadModelPackageAndRunInference_PluginEp_AppendV2) { // Test Case 1: // package_root is a model package directory which contains a manifest.json. @@ -110,31 +362,14 @@ TEST(ModelPackageTest, LoadModelPackageAndRunInference_PluginEp_AppendV2) { { // Build model package on disk const auto package_root = std::filesystem::temp_directory_path() / "ort_model_package_test"; - CreateModelPackage(package_root, MakeManifestJson("test_model", "model_1"), + CreateModelPackage(package_root, MakeManifestJson("model_1"), "model_1", "variant_1", "variant_2", std::filesystem::path{"testdata/mul_1.onnx"}, std::filesystem::path{"testdata/mul_16.onnx"}); - constexpr std::string_view metadata_json = R"({ - "component_model_name": "model_1", - "model_variants": { - "variant_1": { - "model_file": "mul_1.onnx", - "constraints": { - "ep": "example_ep", - "device": "cpu", - "architecture": "arch1" - } - }, - "variant_2": { - "model_file": "mul_16.onnx", - "constraints": { - "ep": "example_ep", - "device": "npu", - "architecture": "arch2" - } - } - } - })"; + const std::string metadata_json = MakeMetadataJsonTwoVariants( + "model_1", + "variant_1", "example_ep", "cpu", "", + "variant_2", "example_ep", "npu", ""); CreateComponentModelMetadata(package_root, "model_1", @@ -186,31 +421,14 @@ TEST(ModelPackageTest, LoadModelPackageAndRunInference_PluginEp_AppendV2) { // Build model package on disk const auto package_root = std::filesystem::temp_directory_path() / "ort_model_package_test"; - CreateModelPackage(package_root, MakeManifestJson("test_model", "model_1"), + CreateModelPackage(package_root, MakeManifestJson("model_1"), "model_1", "variant_1", "variant_2", std::filesystem::path{"testdata/mul_1.onnx"}, std::filesystem::path{"testdata/mul_16.onnx"}); - constexpr std::string_view metadata_json = R"({ - "component_model_name": "model_1", - "model_variants": { - "variant_1": { - "model_file": "mul_1.onnx", - "constraints": { - "ep": "example_ep", - "device": "cpu", - "architecture": "arch1" - } - }, - "variant_2": { - "model_file": "mul_16.onnx", - "constraints": { - "ep": "example_ep", - "device": "npu", - "architecture": "arch2" - } - } - } - })"; + const std::string metadata_json = MakeMetadataJsonTwoVariants( + "model_1", + "variant_1", "example_ep", "cpu", "", + "variant_2", "example_ep", "npu", ""); const auto component_model_root = CreateComponentModelMetadata(package_root, "model_1", @@ -259,31 +477,14 @@ TEST(ModelPackageTest, LoadModelPackageAndRunInference_PreferCpu) { // Build model package on disk const auto package_root = std::filesystem::temp_directory_path() / "ort_model_package_test"; - CreateModelPackage(package_root, MakeManifestJson("test_model", "model_1"), + CreateModelPackage(package_root, MakeManifestJson("model_1"), "model_1", "variant_1", "variant_2", std::filesystem::path{"testdata/mul_1.onnx"}, std::filesystem::path{"testdata/mul_16.onnx"}); - constexpr std::string_view metadata_json = R"({ - "component_model_name": "model_1", - "model_variants": { - "variant_1": { - "model_file": "mul_1.onnx", - "constraints": { - "ep": "example_ep", - "device": "cpu", - "architecture": "arch1" - } - }, - "variant_2": { - "model_file": "mul_16.onnx", - "constraints": { - "ep": "example_ep", - "device": "npu", - "architecture": "arch2" - } - } - } - })"; + const std::string metadata_json = MakeMetadataJsonTwoVariants( + "model_1", + "variant_1", "example_ep", "cpu", "", + "variant_2", "example_ep", "npu", ""); CreateComponentModelMetadata(package_root, "model_1", @@ -353,33 +554,23 @@ TEST(ModelPackageTest, CheckCompiledModelCompatibilityInfo) { // Build model package on disk const auto package_root = std::filesystem::temp_directory_path() / "ort_model_package_test"; - CreateModelPackage(package_root, MakeManifestJson("test_model", "model_1"), + CreateModelPackage(package_root, MakeManifestJson("model_1"), "model_1", "variant_2", "variant_1", std::filesystem::path{"testdata/mul_16.onnx"}, std::filesystem::path{"plugin_ep_compat_test.onnx"}); - constexpr std::string_view metadata_json = R"({ - "component_model_name": "model_1", - "model_variants": { - "variant_2": { - "model_file": "mul_16.onnx", - "constraints": { - "ep": "example_ep", - "device": "cpu", - "architecture": "arch2", - "ep_compatibility_info": "example_ep;version=0.1.0;ort_api_version=25;hardware_architecture=arch2" - } - }, - "variant_1": { - "model_file": "plugin_ep_compat_test.onnx", - "constraints": { - "ep": "example_ep", - "device": "cpu", - "architecture": "arch1", - "ep_compatibility_info": "example_ep;version=0.1.0;ort_api_version=25;hardware_architecture=arch1" - } - } - } - })"; + // Build compat strings dynamically against current ORT_API_VERSION so the EP's ORT-version check + // doesn't short-circuit to PREFER_RECOMPILATION for both variants (which would make hardware_architecture + // irrelevant and the variant ranking collapse to a tie). With matching ORT versions, the arch differentiates: + // arch1 -> OPTIMAL, arch2 -> PREFER_RECOMPILATION; variant_1 must win. + const std::string ort_api_version_str = std::to_string(ORT_API_VERSION); + const std::string compat_arch2 = + "example_ep;version=0.1.0;ort_api_version=" + ort_api_version_str + ";hardware_architecture=arch2"; + const std::string compat_arch1 = + "example_ep;version=0.1.0;ort_api_version=" + ort_api_version_str + ";hardware_architecture=arch1"; + const std::string metadata_json = MakeMetadataJsonTwoVariants( + "model_1", + "variant_2", "example_ep", "cpu", compat_arch2.c_str(), + "variant_1", "example_ep", "cpu", compat_arch1.c_str()); CreateComponentModelMetadata(package_root, "model_1", @@ -402,9 +593,10 @@ TEST(ModelPackageTest, CheckCompiledModelCompatibilityInfo) { } TEST(ModelPackageTest, LoadModelPackageAndRunInference_DiscoverComponentsFromModelsFolder) { - // manifest.json without "component_models"; discovery should scan models/* with metadata.json. + // manifest.json without "components"; discovery should scan models/* with metadata.json. const auto package_root = std::filesystem::temp_directory_path() / "ort_model_package_discover_test"; constexpr std::string_view manifest_json = R"({ + "schema_version": 1, "model_name": "test_model" })"; @@ -413,32 +605,15 @@ TEST(ModelPackageTest, LoadModelPackageAndRunInference_DiscoverComponentsFromMod std::filesystem::path{"testdata/mul_1.onnx"}, std::filesystem::path{"testdata/mul_16.onnx"}); // Prepare component model with metadata and variants - const std::string component_model_name = "model_1"; - constexpr std::string_view metadata_json = R"({ - "component_model_name": "model_1", - "model_variants": { - "variant_1": { - "model_file": "mul_1.onnx", - "constraints": { - "ep": "example_ep", - "device": "cpu", - "architecture": "arch1" - } - }, - "variant_2": { - "model_file": "mul_16.onnx", - "constraints": { - "ep": "example_ep", - "device": "npu", - "architecture": "arch2" - } - } - } - })"; + const std::string component_name = "model_1"; + const std::string metadata_json = MakeMetadataJsonTwoVariants( + "model_1", + "variant_1", "example_ep", "cpu", "", + "variant_2", "example_ep", "npu", ""); // Create metadata.json under models/model_1 const auto component_root = CreateComponentModelMetadata(package_root, - component_model_name, + component_name, metadata_json); // Add another component folder without metadata to ensure it's ignored @@ -483,109 +658,6 @@ TEST(ModelPackageTest, LoadModelPackageAndRunInference_DiscoverComponentsFromMod std::filesystem::remove_all(package_root, ec); } -TEST(ModelPackageTest, ParseVariantFileResolution) { - const auto package_root = std::filesystem::temp_directory_path() / "ort_model_package_parse_variant"; - std::error_code ec; - - auto write_manifest = [&]() { - CreateManifestJson(package_root, MakeManifestJson("test_model", "model_1")); - }; - - auto write_metadata = [&](std::string_view metadata_json) { - CreateComponentModelMetadata(package_root, "model_1", metadata_json); - }; - - // Subcase 1: "model_file" points to a directory containing exactly one .onnx file. - { - std::filesystem::remove_all(package_root, ec); - write_manifest(); - - constexpr std::string_view metadata_json = R"({ - "component_model_name": "model_1", - "model_variants": { - "variant_dir": { - "model_file": "subdir", - "constraints": {} - } - } - })"; - write_metadata(metadata_json); - - const auto variant_dir = package_root / "models" / "model_1" / "variant_dir"; - const auto subdir = variant_dir / "subdir"; - std::filesystem::create_directories(subdir); - std::filesystem::copy_file("testdata/mul_1.onnx", subdir / "only.onnx", - std::filesystem::copy_options::overwrite_existing, ec); - - ModelPackageDescriptorParser parser(logging::LoggingManager::DefaultLogger()); - std::vector variants; - auto status = parser.ParseVariantsFromRoot(package_root, variants); - ASSERT_TRUE(status.IsOK()) << status.ErrorMessage(); - ASSERT_EQ(variants.size(), 1u); - EXPECT_EQ(variants[0].model_path.filename().string(), "only.onnx"); - } - - // Subcase 2: No "model_file" field; discover the single .onnx in the variant directory. - { - std::filesystem::remove_all(package_root, ec); - write_manifest(); - - constexpr std::string_view metadata_json = R"({ - "component_model_name": "model_1", - "model_variants": { - "variant_autodiscover": { - "constraints": {} - } - } - })"; - write_metadata(metadata_json); - - const auto variant_dir = package_root / "models" / "model_1" / "variant_autodiscover"; - std::filesystem::create_directories(variant_dir); - std::filesystem::copy_file("testdata/mul_1.onnx", variant_dir / "auto.onnx", - std::filesystem::copy_options::overwrite_existing, ec); - - ModelPackageDescriptorParser parser(logging::LoggingManager::DefaultLogger()); - std::vector variants; - auto status = parser.ParseVariantsFromRoot(package_root, variants); - ASSERT_TRUE(status.IsOK()) << status.ErrorMessage(); - ASSERT_EQ(variants.size(), 1u); - EXPECT_EQ(variants[0].model_path.filename().string(), "auto.onnx"); - } - - // Subcase 3: Multiple .onnx files -> error. - { - std::filesystem::remove_all(package_root, ec); - write_manifest(); - - constexpr std::string_view metadata_json = R"({ - "component_model_name": "model_1", - "model_variants": { - "variant_multi": { - "constraints": {} - } - } - })"; - write_metadata(metadata_json); - - const auto variant_dir = package_root / "models" / "model_1" / "variant_multi"; - std::filesystem::create_directories(variant_dir); - std::filesystem::copy_file("testdata/mul_1.onnx", variant_dir / "a.onnx", - std::filesystem::copy_options::overwrite_existing, ec); - std::filesystem::copy_file("testdata/mul_16.onnx", variant_dir / "b.onnx", - std::filesystem::copy_options::overwrite_existing, ec); - - ModelPackageDescriptorParser parser(logging::LoggingManager::DefaultLogger()); - std::vector variants; - auto status = parser.ParseVariantsFromRoot(package_root, variants); - ASSERT_FALSE(status.IsOK()); - EXPECT_THAT(status.ErrorMessage(), - ::testing::HasSubstr("Multiple ONNX model files found under")); - } - - std::filesystem::remove_all(package_root, ec); -} - TEST(ModelPackageTest, ParseVariantsFromRoot_PackageRootDirectory) { const auto package_root = std::filesystem::temp_directory_path() / "ort_model_package_parse_from_package_root"; std::error_code ec; @@ -593,8 +665,8 @@ TEST(ModelPackageTest, ParseVariantsFromRoot_PackageRootDirectory) { // package_root is a model package directory (has manifest.json). constexpr std::string_view manifest_json = R"({ - "model_name": "test_model", - "component_models": ["model_1"] + "schema_version": 1, + "components": ["model_1"] })"; CreateModelPackage(package_root, manifest_json, @@ -602,53 +674,52 @@ TEST(ModelPackageTest, ParseVariantsFromRoot_PackageRootDirectory) { std::filesystem::path{"testdata/mul_1.onnx"}, std::filesystem::path{"testdata/mul_16.onnx"}); constexpr std::string_view metadata_json = R"({ - "component_model_name": "model_1", - "model_variants": { + "component_name": "model_1", + "variants": { "variant_1": { - "model_file": "mul_1.onnx", - "constraints": { - "ep": "example_ep", - "device": "cpu", - "architecture": "arch1" - } + "ep": "example_ep", + "device": "cpu" }, "variant_2": { - "model_file": "mul_16.onnx", - "constraints": { - "ep": "example_ep", - "device": "npu", - "architecture": "arch2" - } + "ep": "example_ep", + "device": "npu" } } })"; CreateComponentModelMetadata(package_root, "model_1", metadata_json); - ModelPackageDescriptorParser parser(logging::LoggingManager::DefaultLogger()); - std::vector variants; - auto status = parser.ParseVariantsFromRoot(package_root, variants); + // New schema: per-variant descriptor in variant.json + { + std::ofstream os(package_root / "models" / "model_1" / "variant_1" / "variant.json", std::ios::binary); + os << R"({ "filename": "mul_1.onnx" })"; + } + { + std::ofstream os(package_root / "models" / "model_1" / "variant_2" / "variant.json", std::ios::binary); + os << R"({ "filename": "mul_16.onnx" })"; + } + + ModelPackageContext ctx(package_root); + const auto& variants = ctx.GetVariantInfos(); - ASSERT_TRUE(status.IsOK()) << status.ErrorMessage(); ASSERT_EQ(variants.size(), 2u); - std::unordered_map by_file; + std::unordered_map by_file; for (const auto& v : variants) { - by_file.emplace(v.model_path.filename().string(), &v); + ASSERT_TRUE(v.file.has_value()); + by_file.emplace(v.file->model_file_path.filename().string(), &v); } ASSERT_EQ(by_file.count("mul_1.onnx"), 1u); ASSERT_EQ(by_file.count("mul_16.onnx"), 1u); const auto* v1 = by_file.at("mul_1.onnx"); - EXPECT_EQ(v1->ep, "example_ep"); - EXPECT_EQ(v1->device, "cpu"); - EXPECT_EQ(v1->architecture, "arch1"); + EXPECT_EQ(v1->ep_compatibility.ep.value_or(""), "example_ep"); + EXPECT_EQ(v1->ep_compatibility.device.value_or(""), "cpu"); const auto* v2 = by_file.at("mul_16.onnx"); - EXPECT_EQ(v2->ep, "example_ep"); - EXPECT_EQ(v2->device, "npu"); - EXPECT_EQ(v2->architecture, "arch2"); + EXPECT_EQ(v2->ep_compatibility.ep.value_or(""), "example_ep"); + EXPECT_EQ(v2->ep_compatibility.device.value_or(""), "npu"); std::filesystem::remove_all(package_root, ec); } @@ -666,15 +737,11 @@ TEST(ModelPackageTest, ParseVariantsFromRoot_ComponentModelDirectory) { std::filesystem::copy_options::overwrite_existing, ec); constexpr std::string_view metadata_json = R"({ - "component_model_name": "model_1", - "model_variants": { + "component_name": "model_1", + "variants": { "variant_1": { - "model_file": "mul_1.onnx", - "constraints": { - "ep": "example_ep", - "device": "cpu", - "architecture": "arch1" - } + "ep": "example_ep", + "device": "cpu" } } })"; @@ -684,74 +751,509 @@ TEST(ModelPackageTest, ParseVariantsFromRoot_ComponentModelDirectory) { os << metadata_json; } - ModelPackageDescriptorParser parser(logging::LoggingManager::DefaultLogger()); - std::vector variants; - auto status = parser.ParseVariantsFromRoot(component_root, variants); + { + std::ofstream os(variant_dir / "variant.json", std::ios::binary); + os << R"({ "filename": "mul_1.onnx" })"; + } + + ModelPackageContext ctx(component_root); + const auto& variants = ctx.GetVariantInfos(); - ASSERT_TRUE(status.IsOK()) << status.ErrorMessage(); ASSERT_EQ(variants.size(), 1u); - EXPECT_EQ(variants[0].model_path.filename().string(), "mul_1.onnx"); - EXPECT_EQ(variants[0].ep, "example_ep"); - EXPECT_EQ(variants[0].device, "cpu"); - EXPECT_EQ(variants[0].architecture, "arch1"); + ASSERT_TRUE(variants[0].file.has_value()); + EXPECT_EQ(variants[0].file->model_file_path.filename().string(), "mul_1.onnx"); + + EXPECT_EQ(variants[0].ep_compatibility.ep.value_or(""), "example_ep"); + EXPECT_EQ(variants[0].ep_compatibility.device.value_or(""), "cpu"); std::filesystem::remove_all(component_root, ec); } -TEST(ModelPackageTest, ParseVariantsFromRoot_UsesModelIdForModelDirectory) { - const auto package_root = std::filesystem::temp_directory_path() / "ort_model_package_model_id_dir_test"; +// ------------------------------------------------------------------ +// Tests for descriptor parser: enforced "ep" field in variant EP metadata. +// ------------------------------------------------------------------ +namespace { + +// Make a single-component, single-variant package on disk where metadata.json is written +// directly at the package root (the "single-component metadata flow" of the parser). +// In this flow variant EP metadata schema validation errors are propagated, instead of being +// swallowed by the manifest-driven discovery path which falls back to "Missing metadata variants". +// Returns the package_root. +std::filesystem::path MakeSingleComponentPackageWithMetadata(std::string_view subdir, + std::string_view metadata_json, + std::string_view variant_json = R"({"filename":"mul_1.onnx"})") { + const auto package_root = std::filesystem::temp_directory_path() / std::string(subdir); std::error_code ec; std::filesystem::remove_all(package_root, ec); + std::filesystem::create_directories(package_root); - constexpr std::string_view manifest_json = R"({ - "model_name": "test_model", - "model_version": "1.0", - "component_models": ["model_1"] + // Write metadata.json directly under package_root (no manifest, no models/ subdir). + { + std::ofstream os(package_root / "metadata.json", std::ios::binary); + os << metadata_json; + } + + // Variants live directly under package_root for the single-component flow. + const auto variant_dir = package_root / "variant_1"; + std::filesystem::create_directories(variant_dir); + std::filesystem::copy_file("testdata/mul_1.onnx", variant_dir / "mul_1.onnx", + std::filesystem::copy_options::overwrite_existing, ec); + + std::ofstream os(variant_dir / "variant.json", std::ios::binary); + os << variant_json; + + return package_root; +} + +} // namespace + +TEST(ModelPackageTest, ParserRejects_EpCompatibilityMissingEp) { + // The "ep" field is required in every variant descriptor. + // Omitting it must yield a parse error (not silently accept a wildcard / portable variant). + constexpr std::string_view metadata_json = R"({ + "component_name": "model_1", + "variants": { + "variant_1": { + "device": "cpu", + "compatibility_string": "anything" + } + } })"; + const auto package_root = MakeSingleComponentPackageWithMetadata( + "ort_model_package_parser_missing_ep", metadata_json); + + try { + ModelPackageContext ctx(package_root); + FAIL() << "Expected exception for missing 'ep' field"; + } catch (const std::exception& e) { + EXPECT_NE(std::string(e.what()).find("ep"), std::string::npos) << e.what(); + } - CreateManifestJson(package_root, manifest_json); + std::error_code ec; + std::filesystem::remove_all(package_root, ec); +} + +TEST(ModelPackageTest, ParserRejects_EpCompatibilityNullEp) { + constexpr std::string_view metadata_json = R"({ + "component_name": "model_1", + "variants": { + "variant_1": { + "ep": null, + "device": "cpu" + } + } + })"; + const auto package_root = MakeSingleComponentPackageWithMetadata( + "ort_model_package_parser_null_ep", metadata_json); + + try { + ModelPackageContext ctx(package_root); + FAIL() << "Expected exception for null 'ep' field"; + } catch (const std::exception& e) { + EXPECT_NE(std::string(e.what()).find("ep"), std::string::npos) << e.what(); + } - // Variant name intentionally does not match the model_id-derived directory name. - // model_id = "phi4-cpu:1" should map to directory "phi4-cpu_1". + std::error_code ec; + std::filesystem::remove_all(package_root, ec); +} + +TEST(ModelPackageTest, ParserRejects_EpCompatibilityEmptyEp) { constexpr std::string_view metadata_json = R"({ - "component_model_name": "model_1", - "model_variants": { - "catalog_variant": { - "model_id": "phi4-cpu:1", - "model_file": "model.onnx", - "constraints": { - "ep": "example_ep", - "device": "cpu", - "architecture": "arch1" - } + "component_name": "model_1", + "variants": { + "variant_1": { + "ep": "", + "device": "cpu" } } })"; + const auto package_root = MakeSingleComponentPackageWithMetadata( + "ort_model_package_parser_empty_ep", metadata_json); + + try { + ModelPackageContext ctx(package_root); + FAIL() << "Expected exception for empty 'ep' field"; + } catch (const std::exception& e) { + EXPECT_NE(std::string(e.what()).find("ep"), std::string::npos) << e.what(); + } + + std::error_code ec; + std::filesystem::remove_all(package_root, ec); +} + +// ------------------------------------------------------------------ +// Tests for new pre-selection EP-compat traversal accessors. +// ------------------------------------------------------------------ +TEST(ModelPackageApiTest, GetVariantEpName_ReturnsSingleEp) { + const auto package_root = std::filesystem::temp_directory_path() / "ort_mp_pre_selection_ep_name"; + std::error_code ec; + std::filesystem::remove_all(package_root, ec); + + CreateManifestJson(package_root, MakeManifestJson("model_1")); + + const auto variant1_dir = package_root / "models" / "model_1" / "variant_1"; + const auto variant2_dir = package_root / "models" / "model_1" / "variant_2"; + std::filesystem::create_directories(variant1_dir); + std::filesystem::create_directories(variant2_dir); + std::filesystem::copy_file("testdata/mul_1.onnx", variant1_dir / "mul_1.onnx", + std::filesystem::copy_options::overwrite_existing, ec); + std::filesystem::copy_file("testdata/mul_1.onnx", variant2_dir / "mul_1.onnx", + std::filesystem::copy_options::overwrite_existing, ec); + // Each variant declares a single EP. + constexpr std::string_view metadata_json = R"({ + "component_name": "model_1", + "variants": { + "variant_1": { + "ep": "example_ep", + "device": "cpu" + }, + "variant_2": { + "ep": "other_ep", + "device": "npu" + } + } + })"; CreateComponentModelMetadata(package_root, "model_1", metadata_json); - // Create the model under model_id-derived directory: models/model_1/phi4-cpu_1/model.onnx - const auto model_dir = package_root / "models" / "model_1" / "phi4-cpu_1"; - std::filesystem::create_directories(model_dir); - std::filesystem::copy_file("testdata/mul_1.onnx", - model_dir / "model.onnx", - std::filesystem::copy_options::overwrite_existing, - ec); + for (const auto& d : {variant1_dir, variant2_dir}) { + std::ofstream os(d / "variant.json", std::ios::binary); + os << R"({"filename":"mul_1.onnx"})"; + } - ModelPackageDescriptorParser parser(logging::LoggingManager::DefaultLogger()); - std::vector variants; - auto status = parser.ParseVariantsFromRoot(package_root, variants); + const OrtModelPackageApi* pkg_api = Ort::GetApi().GetModelPackageApi(); + ASSERT_NE(pkg_api, nullptr); - ASSERT_TRUE(status.IsOK()) << status.ErrorMessage(); - ASSERT_EQ(variants.size(), 1u); + auto context_deleter = [pkg_api](OrtModelPackageContext* p) { + if (p) pkg_api->ReleaseModelPackageContext(p); + }; + std::unique_ptr ctx(nullptr, context_deleter); + OrtModelPackageContext* raw_ctx = nullptr; + ASSERT_ORTSTATUS_OK(pkg_api->CreateModelPackageContext(package_root.c_str(), &raw_ctx)); + ctx.reset(raw_ctx); + + // variant_1 targets example_ep + const char* ep1 = nullptr; + ASSERT_ORTSTATUS_OK(pkg_api->ModelPackage_GetVariantEpName( + ctx.get(), "model_1", "variant_1", &ep1)); + ASSERT_NE(ep1, nullptr); + EXPECT_STREQ(ep1, "example_ep"); + + // variant_2 targets other_ep + const char* ep2 = nullptr; + ASSERT_ORTSTATUS_OK(pkg_api->ModelPackage_GetVariantEpName( + ctx.get(), "model_1", "variant_2", &ep2)); + ASSERT_NE(ep2, nullptr); + EXPECT_STREQ(ep2, "other_ep"); + + // Optional out-parameter: callers can pass NULL. + ASSERT_ORTSTATUS_OK(pkg_api->ModelPackage_GetVariantEpName( + ctx.get(), "model_1", "variant_1", nullptr)); + + std::filesystem::remove_all(package_root, ec); +} + +// ------------------------------------------------------------------ +// ------------------------------------------------------------------ +// Test: variant selector tie-break is deterministic across repeated invocations. +// Two variants advertise compatibility for the same EP/device and EP returns the same +// validation score for both -- selection must be stable. +// ------------------------------------------------------------------ +TEST(ModelPackageTest, VariantSelector_TieBreakIsDeterministic) { + // Both variants point at the *same* model file (mul_1.onnx) so whichever wins works at runtime. + // They advertise identical EP/device pairs and empty compatibility_string so the EP returns the + // same score (NOT_APPLICABLE) for both -- a tie. The fix in commit 27217da484 guarantees that + // ties resolve deterministically, i.e., selection is stable across repeated runs. + RegisteredEpDeviceUniquePtr example_ep; + ASSERT_NO_FATAL_FAILURE(Utils::RegisterAndGetExampleEp(*ort_env, Utils::example_ep_info, example_ep)); + Ort::ConstEpDevice plugin_ep_device(example_ep.get()); + + std::string first_selected_filename; + + for (int iter = 0; iter < 5; ++iter) { + const auto package_root = std::filesystem::temp_directory_path() / "ort_mp_tie_break"; + std::error_code ec; + std::filesystem::remove_all(package_root, ec); + + CreateModelPackage(package_root, MakeManifestJson("model_1"), + "model_1", "variant_a", "variant_b", + std::filesystem::path{"testdata/mul_1.onnx"}, + std::filesystem::path{"testdata/mul_1.onnx"}); + + const std::string metadata_json = MakeMetadataJsonTwoVariants( + "model_1", + "variant_a", "example_ep", "cpu", "", + "variant_b", "example_ep", "cpu", ""); + CreateComponentModelMetadata(package_root, "model_1", metadata_json); + + Ort::SessionOptions session_options; + std::unordered_map ep_options; + session_options.AppendExecutionProvider_V2(*ort_env, {plugin_ep_device}, ep_options); + + const OrtModelPackageApi* pkg_api = Ort::GetApi().GetModelPackageApi(); + ASSERT_NE(pkg_api, nullptr); + + auto options_deleter = [pkg_api](OrtModelPackageOptions* p) { if (p) pkg_api->ReleaseModelPackageOptions(p); }; + auto context_deleter = [pkg_api](OrtModelPackageContext* p) { if (p) pkg_api->ReleaseModelPackageContext(p); }; + auto component_context_deleter = [pkg_api](OrtModelPackageComponentContext* p) { + if (p) pkg_api->ReleaseModelPackageComponentContext(p); + }; + std::unique_ptr mp_opts(nullptr, options_deleter); + std::unique_ptr ctx(nullptr, context_deleter); + std::unique_ptr comp_ctx(nullptr, component_context_deleter); + + OrtModelPackageOptions* raw_mp_opts = nullptr; + ASSERT_ORTSTATUS_OK(pkg_api->CreateModelPackageOptionsFromSessionOptions(*ort_env, session_options, &raw_mp_opts)); + mp_opts.reset(raw_mp_opts); + + OrtModelPackageContext* raw_ctx = nullptr; + ASSERT_ORTSTATUS_OK(pkg_api->CreateModelPackageContext(package_root.c_str(), &raw_ctx)); + ctx.reset(raw_ctx); + + OrtModelPackageComponentContext* raw_comp_ctx = nullptr; + ASSERT_ORTSTATUS_OK(pkg_api->SelectComponent(ctx.get(), "model_1", mp_opts.get(), &raw_comp_ctx)); + comp_ctx.reset(raw_comp_ctx); + + const ORTCHAR_T* selected_folder = nullptr; + ASSERT_ORTSTATUS_OK(pkg_api->ModelPackageComponent_GetSelectedVariantFolderPath(comp_ctx.get(), &selected_folder)); + ASSERT_NE(selected_folder, nullptr); + + // Path looks like .../models/model_1/ -- the folder name is the variant. + const auto selected_variant_dir = std::filesystem::path(selected_folder).filename().string(); + ASSERT_TRUE(selected_variant_dir == "variant_a" || selected_variant_dir == "variant_b") + << "unexpected variant dir: " << selected_variant_dir; + + if (iter == 0) { + first_selected_filename = selected_variant_dir; + } else { + EXPECT_EQ(selected_variant_dir, first_selected_filename) + << "tie-break selection drifted across runs (iter " << iter << ")"; + } + + std::filesystem::remove_all(package_root, ec); + } +} + +// ------------------------------------------------------------------ +// Test: a variant's per-file `session_options` flow through OrtApis::AddSessionConfigEntry. +// We verify this by feeding a *known* typed key (session.intra_op_num_threads) a non-integer value: +// pre-change behavior would silently stuff it into AddConfigEntry and succeed; post-change +// behavior parses it via the typed dispatcher and fails CreateSession with a parse error. +// ------------------------------------------------------------------ +TEST(ModelPackageTest, VariantSessionOptions_DispatchedThroughAddSessionConfigEntry) { + const auto package_root = std::filesystem::temp_directory_path() / "ort_mp_session_options_dispatch"; + std::error_code ec; + std::filesystem::remove_all(package_root, ec); - EXPECT_EQ(variants[0].model_path.filename().string(), "model.onnx"); - EXPECT_EQ(variants[0].model_path.parent_path().filename().string(), "phi4-cpu_1"); - EXPECT_EQ(variants[0].ep, "example_ep"); - EXPECT_EQ(variants[0].device, "cpu"); - EXPECT_EQ(variants[0].architecture, "arch1"); + CreateManifestJson(package_root, MakeManifestJson("model_1")); + + const auto variant_dir = package_root / "models" / "model_1" / "variant_1"; + std::filesystem::create_directories(variant_dir); + std::filesystem::copy_file("testdata/mul_1.onnx", variant_dir / "mul_1.onnx", + std::filesystem::copy_options::overwrite_existing, ec); + + constexpr std::string_view metadata_json = R"({ + "component_name": "model_1", + "variants": { + "variant_1": { + "ep": "example_ep", "device": "cpu" + } + } + })"; + CreateComponentModelMetadata(package_root, "model_1", metadata_json); + + // Per-file session_options assigns a typed key (session.intra_op_num_threads) a value that is not a + // valid integer. Routing this through OrtApis::AddSessionConfigEntry (the new behavior) must reject it. + { + std::ofstream os(variant_dir / "variant.json", std::ios::binary); + os << R"({ + "filename": "mul_1.onnx", + "session_options": { + "session.intra_op_num_threads": "not_an_int" + } + })"; + } + + RegisteredEpDeviceUniquePtr example_ep; + ASSERT_NO_FATAL_FAILURE(Utils::RegisterAndGetExampleEp(*ort_env, Utils::example_ep_info, example_ep)); + Ort::ConstEpDevice plugin_ep_device(example_ep.get()); + + Ort::SessionOptions session_options; + std::unordered_map ep_options; + session_options.AppendExecutionProvider_V2(*ort_env, {plugin_ep_device}, ep_options); + + const OrtModelPackageApi* pkg_api = Ort::GetApi().GetModelPackageApi(); + ASSERT_NE(pkg_api, nullptr); + + auto options_deleter = [pkg_api](OrtModelPackageOptions* p) { if (p) pkg_api->ReleaseModelPackageOptions(p); }; + auto context_deleter = [pkg_api](OrtModelPackageContext* p) { if (p) pkg_api->ReleaseModelPackageContext(p); }; + auto component_context_deleter = [pkg_api](OrtModelPackageComponentContext* p) { + if (p) pkg_api->ReleaseModelPackageComponentContext(p); + }; + std::unique_ptr mp_opts(nullptr, options_deleter); + std::unique_ptr ctx(nullptr, context_deleter); + std::unique_ptr comp_ctx(nullptr, component_context_deleter); + + OrtModelPackageOptions* raw_mp_opts = nullptr; + ASSERT_ORTSTATUS_OK(pkg_api->CreateModelPackageOptionsFromSessionOptions(*ort_env, session_options, &raw_mp_opts)); + mp_opts.reset(raw_mp_opts); + + OrtModelPackageContext* raw_ctx = nullptr; + ASSERT_ORTSTATUS_OK(pkg_api->CreateModelPackageContext(package_root.c_str(), &raw_ctx)); + ctx.reset(raw_ctx); + + OrtModelPackageComponentContext* raw_comp_ctx = nullptr; + ASSERT_ORTSTATUS_OK(pkg_api->SelectComponent(ctx.get(), "model_1", mp_opts.get(), &raw_comp_ctx)); + comp_ctx.reset(raw_comp_ctx); + + // CreateSession iterates the per-file session_options and dispatches each through OrtApis::AddSessionConfigEntry. + // The bad int value must surface as an error from this call. + // Pass nullptr for session_options so the metadata-merge path runs (it is skipped when the caller + // supplies their own session_options). + OrtSession* raw_session = nullptr; + OrtStatus* st = pkg_api->CreateSession(*ort_env, comp_ctx.get(), /*session_options=*/nullptr, &raw_session); + // Clean up session first to avoid leaks if assertion fails. + if (raw_session != nullptr) { + Ort::GetApi().ReleaseSession(raw_session); + raw_session = nullptr; + } + ASSERT_NE(st, nullptr) << "CreateSession unexpectedly succeeded with malformed session.intra_op_num_threads"; + const std::string err_msg = Ort::GetApi().GetErrorMessage(st); + Ort::GetApi().ReleaseStatus(st); + + // Message should mention either AddSessionConfigEntry or the typed-int parse failure. + const bool mentions_dispatch = + err_msg.find("AddSessionConfigEntry") != std::string::npos || + err_msg.find("base-10 int32") != std::string::npos || + err_msg.find("intra_op_num_threads") != std::string::npos; + EXPECT_TRUE(mentions_dispatch) << "error did not mention typed dispatch: " << err_msg; + + std::filesystem::remove_all(package_root, ec); +} + +// Test that the C++ RAII wrappers (Ort::ModelPackageContext, etc.) work correctly. +TEST(ModelPackageApiTest, CxxWrappers_PackageContextQueries) { + const auto package_root = CreateModelPackageApiTestPackage(); + + Ort::ModelPackageContext ctx(package_root.c_str()); + + // Component queries + EXPECT_EQ(ctx.GetComponentCount(), 1u); + auto component_names = ctx.GetComponentNames(); + ASSERT_EQ(component_names.size(), 1u); + EXPECT_EQ(component_names[0], "model_1"); + + // Variant queries + EXPECT_EQ(ctx.GetVariantCount("model_1"), 2u); + auto variant_names = ctx.GetVariantNames("model_1"); + ASSERT_EQ(variant_names.size(), 2u); + std::unordered_set variant_set(variant_names.begin(), variant_names.end()); + EXPECT_EQ(variant_set.count("variant_1"), 1u); + EXPECT_EQ(variant_set.count("variant_2"), 1u); + + std::error_code ec; + std::filesystem::remove_all(package_root, ec); +} + +TEST(ModelPackageApiTest, CxxWrappers_SelectComponentAndQueryFileAccessors) { + const auto package_root = CreateModelPackageApiTestPackage(); + + RegisteredEpDeviceUniquePtr example_ep; + ASSERT_NO_FATAL_FAILURE(Utils::RegisterAndGetExampleEp(*ort_env, Utils::example_ep_info, example_ep)); + Ort::ConstEpDevice plugin_ep_device(example_ep.get()); + + Ort::SessionOptions so; + std::unordered_map ep_options; + so.AppendExecutionProvider_V2(*ort_env, {plugin_ep_device}, ep_options); + Ort::ModelPackageOptions pkg_opts(*ort_env, so); + + Ort::ModelPackageContext ctx(package_root.c_str()); + auto cix = ctx.SelectComponent("model_1", pkg_opts); + + // Folder path should be non-empty + auto folder = cix.GetSelectedVariantFolderPath(); + EXPECT_FALSE(folder.empty()); + + // Selected variant name should not throw + auto variant_name = cix.GetSelectedVariantName(); + EXPECT_FALSE(variant_name.empty()); + + // CreateSession via C++ wrapper + auto session = cix.CreateSession(*ort_env, so); + + std::error_code ec; + std::filesystem::remove_all(package_root, ec); +} + +// ------------------------------------------------------------------ +// Test: GetSelectedVariantFolderPath returns correct path even when variant.json is absent. +// ------------------------------------------------------------------ +TEST(ModelPackageApiTest, FolderPath_ReturnsCorrectPath_WhenVariantJsonAbsent) { + const auto package_root = std::filesystem::temp_directory_path() / "ort_mp_folder_path_no_variant_json"; + std::error_code ec; + std::filesystem::remove_all(package_root, ec); + std::filesystem::create_directories(package_root); + + CreateManifestJson(package_root, MakeManifestJson("model_1")); + + const auto variant_dir = package_root / "models" / "model_1" / "variant_1"; + std::filesystem::create_directories(variant_dir); + + // Copy a model file but do NOT create variant.json + std::filesystem::copy_file("testdata/mul_1.onnx", variant_dir / "mul_1.onnx", + std::filesystem::copy_options::overwrite_existing, ec); + + constexpr std::string_view metadata_json = R"({ + "component_name": "model_1", + "variants": { + "variant_1": { + "ep": "example_ep", + "device": "cpu" + } + } + })"; + CreateComponentModelMetadata(package_root, "model_1", metadata_json); + + RegisteredEpDeviceUniquePtr example_ep; + ASSERT_NO_FATAL_FAILURE(Utils::RegisterAndGetExampleEp(*ort_env, Utils::example_ep_info, example_ep)); + Ort::ConstEpDevice plugin_ep_device(example_ep.get()); + + Ort::SessionOptions so; + std::unordered_map ep_options; + so.AppendExecutionProvider_V2(*ort_env, {plugin_ep_device}, ep_options); + Ort::ModelPackageOptions pkg_opts(*ort_env, so); + + const OrtModelPackageApi* pkg_api = Ort::GetApi().GetModelPackageApi(); + ASSERT_NE(pkg_api, nullptr); + + OrtModelPackageOptions* raw_mp_opts = nullptr; + ASSERT_ORTSTATUS_OK(pkg_api->CreateModelPackageOptionsFromSessionOptions(*ort_env, so, &raw_mp_opts)); + auto options_deleter = [pkg_api](OrtModelPackageOptions* p) { if (p) pkg_api->ReleaseModelPackageOptions(p); }; + std::unique_ptr mp_opts(raw_mp_opts, options_deleter); + + OrtModelPackageContext* raw_ctx = nullptr; + ASSERT_ORTSTATUS_OK(pkg_api->CreateModelPackageContext(package_root.c_str(), &raw_ctx)); + auto context_deleter = [pkg_api](OrtModelPackageContext* p) { if (p) pkg_api->ReleaseModelPackageContext(p); }; + std::unique_ptr ctx(raw_ctx, context_deleter); + + OrtModelPackageComponentContext* raw_comp_ctx = nullptr; + ASSERT_ORTSTATUS_OK(pkg_api->SelectComponent(ctx.get(), "model_1", mp_opts.get(), &raw_comp_ctx)); + auto component_context_deleter = [pkg_api](OrtModelPackageComponentContext* p) { + if (p) pkg_api->ReleaseModelPackageComponentContext(p); + }; + std::unique_ptr comp_ctx(raw_comp_ctx, component_context_deleter); + + // GetSelectedVariantFolderPath should return the variant directory even without variant.json. + const ORTCHAR_T* selected_folder = nullptr; + ASSERT_ORTSTATUS_OK(pkg_api->ModelPackageComponent_GetSelectedVariantFolderPath(comp_ctx.get(), &selected_folder)); + ASSERT_NE(selected_folder, nullptr); + + const auto result_path = std::filesystem::path(selected_folder); + EXPECT_FALSE(result_path.empty()); + EXPECT_EQ(result_path.filename().string(), "variant_1"); std::filesystem::remove_all(package_root, ec); } + } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/framework/more_session_option_test.cc b/onnxruntime/test/framework/more_session_option_test.cc new file mode 100644 index 0000000000000..b8fe92560fc44 --- /dev/null +++ b/onnxruntime/test/framework/more_session_option_test.cc @@ -0,0 +1,379 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include +#include + +#include "gtest/gtest.h" + +#include "core/framework/session_options.h" +#include "core/session/abi_session_options_impl.h" +#include "core/session/onnxruntime_c_api.h" +#include "core/session/onnxruntime_cxx_api.h" +#include "core/session/onnxruntime_session_options_config_keys.h" + +namespace onnxruntime { +namespace test { + +namespace { + +// Tiny RAII wrapper around OrtStatus* so tests don't leak on failure. +struct OrtStatusGuard { + OrtStatus* st = nullptr; + ~OrtStatusGuard() { + if (st != nullptr) Ort::GetApi().ReleaseStatus(st); + } +}; + +const OrtApi& Api() { return Ort::GetApi(); } + +// Build a fresh OrtSessionOptions* via the C API for tests that need a raw handle. +OrtSessionOptions* MakeOptions() { + OrtSessionOptions* opts = nullptr; + OrtStatus* st = Api().CreateSessionOptions(&opts); + EXPECT_EQ(st, nullptr); + if (st != nullptr) Api().ReleaseStatus(st); + return opts; +} + +void ReleaseOptions(OrtSessionOptions* opts) { Api().ReleaseSessionOptions(opts); } + +OrtStatus* AddOption(OrtSessionOptions* opts, const char* key, const char* value) { + return Api().AddSessionConfigEntry(opts, key, value); +} + +OrtErrorCode CodeOf(OrtStatus* st) { return Api().GetErrorCode(st); } +const char* MsgOf(OrtStatus* st) { return Api().GetErrorMessage(st); } + +} // namespace + +// ----------------------------------------------------------------------------- +// Bool-valued keys: session.enable_cpu_mem_arena, session.enable_mem_pattern, +// session.use_deterministic_compute +// ----------------------------------------------------------------------------- +TEST(CApiTest, BoolKeys_AcceptsAllSpellings) { + // ParseBool accepts: "0", "1", "true", "false", and case-insensitive variants. + const char* truthy[] = {"1", "true", "True", "TRUE"}; + const char* falsy[] = {"0", "false", "False", "FALSE"}; + + for (const char* v : truthy) { + OrtSessionOptions* opts = MakeOptions(); + OrtStatusGuard g{AddOption(opts, "session.enable_cpu_mem_arena", v)}; + ASSERT_EQ(g.st, nullptr) << "value=" << v; + EXPECT_TRUE(opts->value.enable_cpu_mem_arena) << "value=" << v; + ReleaseOptions(opts); + } + for (const char* v : falsy) { + OrtSessionOptions* opts = MakeOptions(); + OrtStatusGuard g{AddOption(opts, "session.enable_cpu_mem_arena", v)}; + ASSERT_EQ(g.st, nullptr) << "value=" << v; + EXPECT_FALSE(opts->value.enable_cpu_mem_arena) << "value=" << v; + ReleaseOptions(opts); + } +} + +TEST(CApiTest, BoolKey_EnableMemPattern) { + OrtSessionOptions* opts = MakeOptions(); + ASSERT_TRUE(opts->value.enable_mem_pattern); // default + + { + OrtStatusGuard g{AddOption(opts, "session.enable_mem_pattern", "0")}; + ASSERT_EQ(g.st, nullptr); + EXPECT_FALSE(opts->value.enable_mem_pattern); + } + { + OrtStatusGuard g{AddOption(opts, "session.enable_mem_pattern", "true")}; + ASSERT_EQ(g.st, nullptr); + EXPECT_TRUE(opts->value.enable_mem_pattern); + } + ReleaseOptions(opts); +} + +TEST(CApiTest, BoolKey_UseDeterministicCompute) { + OrtSessionOptions* opts = MakeOptions(); + EXPECT_FALSE(opts->value.use_deterministic_compute); + OrtStatusGuard g{AddOption(opts, "session.use_deterministic_compute", "1")}; + ASSERT_EQ(g.st, nullptr); + EXPECT_TRUE(opts->value.use_deterministic_compute); + ReleaseOptions(opts); +} + +TEST(CApiTest, BoolKey_InvalidValueErrors) { + OrtSessionOptions* opts = MakeOptions(); + OrtStatusGuard g{AddOption(opts, "session.enable_cpu_mem_arena", "yes")}; + ASSERT_NE(g.st, nullptr); + EXPECT_EQ(CodeOf(g.st), ORT_INVALID_ARGUMENT); + EXPECT_NE(std::string(MsgOf(g.st)).find("boolean"), std::string::npos) << MsgOf(g.st); + ReleaseOptions(opts); +} + +// ----------------------------------------------------------------------------- +// Int-valued keys: session.intra_op_num_threads, session.inter_op_num_threads, +// session.log_severity_level, session.log_verbosity_level +// ----------------------------------------------------------------------------- +TEST(CApiTest, IntKey_IntraOpNumThreads) { + OrtSessionOptions* opts = MakeOptions(); + OrtStatusGuard g{AddOption(opts, "session.intra_op_num_threads", "4")}; + ASSERT_EQ(g.st, nullptr); + EXPECT_EQ(opts->value.intra_op_param.thread_pool_size, 4); + ReleaseOptions(opts); +} + +TEST(CApiTest, IntKey_InterOpNumThreads) { + OrtSessionOptions* opts = MakeOptions(); + OrtStatusGuard g{AddOption(opts, "session.inter_op_num_threads", "2")}; + ASSERT_EQ(g.st, nullptr); + EXPECT_EQ(opts->value.inter_op_param.thread_pool_size, 2); + ReleaseOptions(opts); +} + +TEST(CApiTest, IntKey_LogSeverityAndVerbosity) { + OrtSessionOptions* opts = MakeOptions(); + { + OrtStatusGuard g{AddOption(opts, "session.log_severity_level", "2")}; + ASSERT_EQ(g.st, nullptr); + EXPECT_EQ(opts->value.session_log_severity_level, 2); + } + { + OrtStatusGuard g{AddOption(opts, "session.log_verbosity_level", "3")}; + ASSERT_EQ(g.st, nullptr); + EXPECT_EQ(opts->value.session_log_verbosity_level, 3); + } + ReleaseOptions(opts); +} + +TEST(CApiTest, IntKey_EmptyValueErrors) { + OrtSessionOptions* opts = MakeOptions(); + OrtStatusGuard g{AddOption(opts, "session.intra_op_num_threads", "")}; + ASSERT_NE(g.st, nullptr); + EXPECT_EQ(CodeOf(g.st), ORT_INVALID_ARGUMENT); + EXPECT_NE(std::string(MsgOf(g.st)).find("empty"), std::string::npos) << MsgOf(g.st); + ReleaseOptions(opts); +} + +TEST(CApiTest, IntKey_NonIntegerValueErrors) { + OrtSessionOptions* opts = MakeOptions(); + OrtStatusGuard g{AddOption(opts, "session.intra_op_num_threads", "not_an_int")}; + ASSERT_NE(g.st, nullptr); + EXPECT_EQ(CodeOf(g.st), ORT_INVALID_ARGUMENT); + EXPECT_NE(std::string(MsgOf(g.st)).find("base-10 int32"), std::string::npos) << MsgOf(g.st); + ReleaseOptions(opts); +} + +TEST(CApiTest, IntKey_TrailingGarbageErrors) { + OrtSessionOptions* opts = MakeOptions(); + OrtStatusGuard g{AddOption(opts, "session.intra_op_num_threads", "12abc")}; + ASSERT_NE(g.st, nullptr); + EXPECT_EQ(CodeOf(g.st), ORT_INVALID_ARGUMENT); + ReleaseOptions(opts); +} + +// ----------------------------------------------------------------------------- +// Log id (session.log_id) +// ----------------------------------------------------------------------------- +TEST(CApiTest, LogId_SetViaConfigEntry) { + OrtSessionOptions* opts = MakeOptions(); + OrtStatusGuard g{AddOption(opts, "session.log_id", "session-A")}; + ASSERT_EQ(g.st, nullptr); + EXPECT_EQ(opts->value.session_logid, "session-A"); + ReleaseOptions(opts); +} + +// ----------------------------------------------------------------------------- +// Enable profiling: non-empty enables with prefix, empty disables. +// ----------------------------------------------------------------------------- +TEST(CApiTest, EnableProfiling_NonEmptyEnables) { + OrtSessionOptions* opts = MakeOptions(); + OrtStatusGuard g{AddOption(opts, "session.enable_profiling", "myrun_")}; + ASSERT_EQ(g.st, nullptr); + EXPECT_TRUE(opts->value.enable_profiling); + EXPECT_EQ(opts->value.profile_file_prefix, ORT_TSTR("myrun_")); + ReleaseOptions(opts); +} + +TEST(CApiTest, EnableProfiling_EmptyDisables) { + OrtSessionOptions* opts = MakeOptions(); + // First enable. + { + OrtStatusGuard g{AddOption(opts, "session.enable_profiling", "x_")}; + ASSERT_EQ(g.st, nullptr); + ASSERT_TRUE(opts->value.enable_profiling); + } + // Empty string disables. + { + OrtStatusGuard g{AddOption(opts, "session.enable_profiling", "")}; + ASSERT_EQ(g.st, nullptr); + EXPECT_FALSE(opts->value.enable_profiling); + } + ReleaseOptions(opts); +} + +// ----------------------------------------------------------------------------- +// Graph optimization level (enum) +// ----------------------------------------------------------------------------- +TEST(CApiTest, GraphOptimizationLevel_AllSpellings) { + struct Case { + const char* in; + TransformerLevel expected; + }; + const Case cases[] = { + {"disable_all", TransformerLevel::Default}, + {"enable_basic", TransformerLevel::Level1}, + {"enable_extended", TransformerLevel::Level2}, + {"enable_layout", TransformerLevel::Level3}, + {"enable_all", TransformerLevel::MaxLevel}, + {"ENABLE_ALL", TransformerLevel::MaxLevel}, + {"Enable_Basic", TransformerLevel::Level1}, + }; + for (const auto& c : cases) { + OrtSessionOptions* opts = MakeOptions(); + OrtStatusGuard g{AddOption(opts, "session.graph_optimization_level", c.in)}; + ASSERT_EQ(g.st, nullptr) << "value=" << c.in; + EXPECT_EQ(opts->value.graph_optimization_level, c.expected) << "value=" << c.in; + ReleaseOptions(opts); + } +} + +TEST(CApiTest, GraphOptimizationLevel_InvalidValueErrors) { + OrtSessionOptions* opts = MakeOptions(); + OrtStatusGuard g{AddOption(opts, "session.graph_optimization_level", "ludicrous")}; + ASSERT_NE(g.st, nullptr); + EXPECT_EQ(CodeOf(g.st), ORT_INVALID_ARGUMENT); + EXPECT_NE(std::string(MsgOf(g.st)).find("graph_optimization_level"), std::string::npos); + ReleaseOptions(opts); +} + +// ----------------------------------------------------------------------------- +// Optimized model filepath (path) +// ----------------------------------------------------------------------------- +TEST(CApiTest, OptimizedModelFilepath) { + OrtSessionOptions* opts = MakeOptions(); + OrtStatusGuard g{AddOption(opts, "session.optimized_model_filepath", "/tmp/opt_model.onnx")}; + ASSERT_EQ(g.st, nullptr); + EXPECT_EQ(opts->value.optimized_model_filepath, + std::filesystem::path(ORT_TSTR("/tmp/opt_model.onnx"))); + ReleaseOptions(opts); +} + +// ----------------------------------------------------------------------------- +// Execution mode (enum) +// ----------------------------------------------------------------------------- +TEST(CApiTest, ExecutionMode_SequentialAndParallel) { + { + OrtSessionOptions* opts = MakeOptions(); + OrtStatusGuard g{AddOption(opts, "session.execution_mode", "parallel")}; + ASSERT_EQ(g.st, nullptr); + EXPECT_EQ(opts->value.execution_mode, ExecutionMode::ORT_PARALLEL); + ReleaseOptions(opts); + } + { + OrtSessionOptions* opts = MakeOptions(); + OrtStatusGuard g{AddOption(opts, "session.execution_mode", "Sequential")}; + ASSERT_EQ(g.st, nullptr); + EXPECT_EQ(opts->value.execution_mode, ExecutionMode::ORT_SEQUENTIAL); + ReleaseOptions(opts); + } +} + +TEST(CApiTest, ExecutionMode_InvalidValueErrors) { + OrtSessionOptions* opts = MakeOptions(); + OrtStatusGuard g{AddOption(opts, "session.execution_mode", "concurrent")}; + ASSERT_NE(g.st, nullptr); + EXPECT_EQ(CodeOf(g.st), ORT_INVALID_ARGUMENT); + ReleaseOptions(opts); +} + +// ----------------------------------------------------------------------------- +// use_per_session_threads — one-way disable, true is a no-op while still enabled, +// true after a disable is an error (no public ABI to re-enable). +// On WASM+pthreads the default is false, so setting "true" is always an error. +// ----------------------------------------------------------------------------- +TEST(CApiTest, UsePerSessionThreads_TrueIsNoOpInitially) { + OrtSessionOptions* opts = MakeOptions(); + constexpr bool kDefault = onnxruntime::SessionOptions::DEFAULT_USE_PER_SESSION_THREADS; + ASSERT_EQ(opts->value.use_per_session_threads, kDefault); // platform-specific default + OrtStatusGuard g{AddOption(opts, "session.use_per_session_threads", "true")}; + if constexpr (kDefault) { + // When default is true, setting "true" is a harmless no-op. + ASSERT_EQ(g.st, nullptr); + EXPECT_TRUE(opts->value.use_per_session_threads); + } else { + // When default is false (WASM+pthreads), setting "true" is an error + // because there is no public ABI to re-enable. + ASSERT_NE(g.st, nullptr); + EXPECT_EQ(CodeOf(g.st), ORT_INVALID_ARGUMENT); + } + ReleaseOptions(opts); +} + +TEST(CApiTest, UsePerSessionThreads_FalseDisables) { + OrtSessionOptions* opts = MakeOptions(); + OrtStatusGuard g{AddOption(opts, "session.use_per_session_threads", "false")}; + ASSERT_EQ(g.st, nullptr); + EXPECT_FALSE(opts->value.use_per_session_threads); + ReleaseOptions(opts); +} + +TEST(CApiTest, UsePerSessionThreads_TrueAfterDisableErrors) { + OrtSessionOptions* opts = MakeOptions(); + { + OrtStatusGuard g{AddOption(opts, "session.use_per_session_threads", "false")}; + ASSERT_EQ(g.st, nullptr); + } + { + OrtStatusGuard g{AddOption(opts, "session.use_per_session_threads", "true")}; + ASSERT_NE(g.st, nullptr); + EXPECT_EQ(CodeOf(g.st), ORT_INVALID_ARGUMENT); + EXPECT_NE(std::string(MsgOf(g.st)).find("use_per_session_threads"), std::string::npos); + } + ReleaseOptions(opts); +} + +// ----------------------------------------------------------------------------- +// Unknown keys are stored as regular session config entries. +// ----------------------------------------------------------------------------- +TEST(CApiTest, UnknownKey_FallsThroughToConfigEntry) { + OrtSessionOptions* opts = MakeOptions(); + OrtStatusGuard g{AddOption(opts, "session.disable_prepacking", "1")}; + ASSERT_EQ(g.st, nullptr); + + // Verify it landed in the config entries (not the typed setters). + int has = 0; + OrtStatusGuard g2{Api().HasSessionConfigEntry(opts, "session.disable_prepacking", &has)}; + ASSERT_EQ(g2.st, nullptr); + EXPECT_EQ(has, 1); + + char buf[16] = {}; + size_t size = sizeof(buf); + OrtStatusGuard g3{Api().GetSessionConfigEntry(opts, "session.disable_prepacking", buf, &size)}; + ASSERT_EQ(g3.st, nullptr); + EXPECT_STREQ(buf, "1"); + ReleaseOptions(opts); +} + +// ----------------------------------------------------------------------------- +// Null arguments are rejected with INVALID_ARGUMENT. +// ----------------------------------------------------------------------------- +TEST(CApiTest, NullArguments_AreRejected) { + OrtSessionOptions* opts = MakeOptions(); + { + OrtStatusGuard g{AddOption(nullptr, "session.intra_op_num_threads", "1")}; + ASSERT_NE(g.st, nullptr); + EXPECT_EQ(CodeOf(g.st), ORT_INVALID_ARGUMENT); + } + { + OrtStatusGuard g{AddOption(opts, nullptr, "1")}; + ASSERT_NE(g.st, nullptr); + EXPECT_EQ(CodeOf(g.st), ORT_INVALID_ARGUMENT); + } + { + OrtStatusGuard g{AddOption(opts, "session.intra_op_num_threads", nullptr)}; + ASSERT_NE(g.st, nullptr); + EXPECT_EQ(CodeOf(g.st), ORT_INVALID_ARGUMENT); + } + ReleaseOptions(opts); +} + +} // namespace test +} // namespace onnxruntime