Skip to content

ORT 1.23.1 cherrypick 1 #26138

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
adrianlizarraga wants to merge 13 commits into from
Closed

ORT 1.23.1 cherrypick 1 #26138

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
fae1608
Update version to 1.23.1
adrianlizarraga Sep 23, 2025
f300a62
npm run format
adrianlizarraga Sep 23, 2025
18e066d
[CPU] MoE Kernel (#25958)
apsonawane Sep 10, 2025
428359c
[CPU] Block-wise QMoE kernel for CPU (#26009)
apsonawane Sep 15, 2025
ba1bc31
manual cherry-pick Fix Attention GQA implementation on CPU #25966
adrianlizarraga Sep 23, 2025
4bd22e7
Address edge GetMemInfo edge cases (#26021)
yuslepukhin Sep 16, 2025
b439ecf
Implement new Python APIs (#25999)
yuslepukhin Sep 17, 2025
7182b71
MemcpyFromHost and MemcpyToHost support for plugin EPs (#26088)
edgchen1 Sep 23, 2025
0b85aec
Use old include path for internal testing EP header
adrianlizarraga Sep 24, 2025
212a91c
remove extra include
adrianlizarraga Sep 24, 2025
eeebe89
[TRT RTX EP] Fix bug for generating the correct subgraph in GetCapabi…
chilo-ms Sep 24, 2025
f8f2e99
Mark unused param in DequantizeBlockWithMlas
adrianlizarraga Sep 24, 2025
e031a53
Disable unit tests that use test models created with new ONNX IR.
adrianlizarraga Sep 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion VERSION_NUMBER
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1.23.0
1.23.1
4 changes: 4 additions & 0 deletions cmake/onnxruntime_unittests.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1800,6 +1800,7 @@ endif()
if (WIN32 AND onnxruntime_BUILD_SHARED_LIB AND
NOT CMAKE_SYSTEM_NAME STREQUAL "Emscripten" AND
NOT onnxruntime_MINIMAL_BUILD)
# example_plugin_ep
file(GLOB onnxruntime_autoep_test_library_src "${TEST_SRC_DIR}/autoep/library/*.h"
"${TEST_SRC_DIR}/autoep/library/*.cc")
onnxruntime_add_shared_library_module(example_plugin_ep ${onnxruntime_autoep_test_library_src})
Expand All @@ -1822,6 +1823,9 @@ if (WIN32 AND onnxruntime_BUILD_SHARED_LIB AND
set_property(TARGET example_plugin_ep APPEND_STRING PROPERTY LINK_FLAGS
${ONNXRUNTIME_AUTOEP_LIB_LINK_FLAG})

set_target_properties(example_plugin_ep PROPERTIES FOLDER "ONNXRuntimeTest")
source_group(TREE ${TEST_SRC_DIR} FILES ${onnxruntime_autoep_test_library_src})

# test library
file(GLOB onnxruntime_autoep_test_SRC "${ONNXRUNTIME_AUTOEP_TEST_SRC_DIR}/*.h"
"${ONNXRUNTIME_AUTOEP_TEST_SRC_DIR}/*.cc")
Expand Down
3 changes: 3 additions & 0 deletions docs/OperatorKernels.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -253,6 +253,8 @@ Do not modify directly.*
|||[9, 12]|**T** = tensor(float)|
|||[1, 8]|**T** = tensor(float)|
|MelWeightMatrix|*in* num_mel_bins:**T1**<br> *in* dft_length:**T1**<br> *in* sample_rate:**T1**<br> *in* lower_edge_hertz:**T2**<br> *in* upper_edge_hertz:**T2**<br> *out* output:**T3**|17+|**T1** = tensor(int32), tensor(int64)<br/> **T2** = tensor(float)<br/> **T3** = tensor(double), tensor(float), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)|
|MemcpyFromHost|*in* X:**T**<br> *out* Y:**T**|1+|**T** = seq(tensor(bfloat16)), seq(tensor(bool)), seq(tensor(double)), seq(tensor(float)), seq(tensor(float16)), seq(tensor(float8e4m3fn)), seq(tensor(float8e4m3fnuz)), seq(tensor(float8e5m2)), seq(tensor(float8e5m2fnuz)), seq(tensor(int16)), seq(tensor(int32)), seq(tensor(int64)), seq(tensor(int8)), seq(tensor(uint16)), seq(tensor(uint32)), seq(tensor(uint64)), seq(tensor(uint8)), tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(float8e4m3fn), tensor(float8e4m3fnuz), tensor(float8e5m2), tensor(float8e5m2fnuz), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)|
|MemcpyToHost|*in* X:**T**<br> *out* Y:**T**|1+|**T** = seq(tensor(bfloat16)), seq(tensor(bool)), seq(tensor(double)), seq(tensor(float)), seq(tensor(float16)), seq(tensor(float8e4m3fn)), seq(tensor(float8e4m3fnuz)), seq(tensor(float8e5m2)), seq(tensor(float8e5m2fnuz)), seq(tensor(int16)), seq(tensor(int32)), seq(tensor(int64)), seq(tensor(int8)), seq(tensor(uint16)), seq(tensor(uint32)), seq(tensor(uint64)), seq(tensor(uint8)), tensor(bfloat16), tensor(bool), tensor(double), tensor(float), tensor(float16), tensor(float8e4m3fn), tensor(float8e4m3fnuz), tensor(float8e5m2), tensor(float8e5m2fnuz), tensor(int16), tensor(int32), tensor(int64), tensor(int8), tensor(uint16), tensor(uint32), tensor(uint64), tensor(uint8)|
|Min|*in* data_0:**T**<br> *out* min:**T**|13+|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)|
|||12|**T** = tensor(double), tensor(float), tensor(float16), tensor(int32), tensor(int64), tensor(uint32), tensor(uint64)|
|||[8, 11]|**T** = tensor(double), tensor(float)|
Expand Down Expand Up @@ -547,6 +549,7 @@ Do not modify directly.*
|MatMulIntegerToFloat|*in* A:**T1**<br> *in* B:**T2**<br> *in* a_scale:**T3**<br> *in* b_scale:**T3**<br> *in* a_zero_point:**T1**<br> *in* b_zero_point:**T2**<br> *in* bias:**T3**<br> *out* Y:**T3**|1+|**T1** = tensor(int8), tensor(uint8)<br/> **T2** = tensor(int8), tensor(uint8)<br/> **T3** = tensor(float)|
|MatMulNBits|*in* A:**T1**<br> *in* B:**T2**<br> *in* scales:**T1**<br> *in* zero_points:**T3**<br> *in* g_idx:**T4**<br> *in* bias:**T1**<br> *out* Y:**T1**|1+|**T1** = tensor(float), tensor(float16)<br/> **T2** = tensor(uint8)<br/> **T3** = tensor(float), tensor(float16), tensor(uint8)<br/> **T4** = tensor(int32)|
|MaxpoolWithMask|*in* X:**T**<br> *in* M:**tensor(int32)**<br> *out* Y:**T**|1+|**T** = tensor(float)|
|MoE|*in* input:**T**<br> *in* router_probs:**T**<br> *in* fc1_experts_weights:**T**<br> *in* fc1_experts_bias:**T**<br> *in* fc2_experts_weights:**T**<br> *in* fc2_experts_bias:**T**<br> *in* fc3_experts_weights:**T**<br> *in* fc3_experts_bias:**T**<br> *out* output:**T**|1+|**T** = tensor(float)|
|MultiHeadAttention|*in* query:**T**<br> *in* key:**T**<br> *in* value:**T**<br> *in* bias:**T**<br> *in* key_padding_mask:**M**<br> *in* attention_bias:**T**<br> *in* past_key:**T**<br> *in* past_value:**T**<br> *in* past_sequence_length:**M**<br> *in* cache_indirection:**M**<br> *out* output:**T**<br> *out* present_key:**T**<br> *out* present_value:**T**<br> *out* qk:**QK**|1+|**T** = tensor(float)|
|MurmurHash3|*in* X:**T1**<br> *out* Y:**T2**|1+|**T1** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(string), tensor(uint32), tensor(uint64)<br/> **T2** = tensor(int32), tensor(uint32)|
|NGramRepeatBlock|*in* input_ids:**Tid**<br> *in* scores:**T**<br> *out* scores_out:**T**|1+|**T** = tensor(float)<br/> **Tid** = tensor(int64)|
Expand Down
5 changes: 5 additions & 0 deletions docs/python/README.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,11 @@ For more information on ONNX Runtime, please see `aka.ms/onnxruntime <https://ak
Changes
-------

1.23.1
^^^^^^

Release Notes : https://github.com/Microsoft/onnxruntime/releases/tag/v1.23.1

1.23.0
^^^^^^

Expand Down
22 changes: 18 additions & 4 deletions include/onnxruntime/core/session/onnxruntime_cxx_api.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1000,11 +1000,25 @@ using UnownedAllocator = detail::AllocatorImpl<detail::Unowned<OrtAllocator>>;
/** \brief Wrapper around ::OrtSyncStream
*
*/
struct SyncStream : detail::Base<OrtSyncStream> {
explicit SyncStream(std::nullptr_t) {} ///< Create an empty SyncStream object, must be assigned a valid one to be used
explicit SyncStream(OrtSyncStream* p) : Base<OrtSyncStream>{p} {} ///< Take ownership of a pointer created by C API
void* GetHandle() const; ///< Wraps SyncStream_GetHandle

namespace detail {
template <typename T>
struct SyncStreamImpl : Base<T> {
using B = Base<T>;
using B::B;
// For some reason this is not a const method on the stream
void* GetHandle(); ///< Wraps SyncStream_GetHandle
};
} // namespace detail

struct SyncStream : detail::SyncStreamImpl<OrtSyncStream> {
///< Create an empty SyncStream object, must be assigned a valid one to be used
explicit SyncStream(std::nullptr_t) {}
///< Take ownership of a pointer created by C API
explicit SyncStream(OrtSyncStream* p) : SyncStreamImpl<OrtSyncStream>{p} {}
};

using UnownedSyncStream = detail::SyncStreamImpl<detail::Unowned<OrtSyncStream>>;

namespace detail {
template <typename T>
Expand Down
37 changes: 22 additions & 15 deletions include/onnxruntime/core/session/onnxruntime_cxx_inline.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -669,9 +669,12 @@ inline void KeyValuePairs::Remove(const char* key) {
GetApi().RemoveKeyValuePair(this->p_, key);
}

inline void* SyncStream::GetHandle() const {
namespace detail {
template <typename T>
inline void* SyncStreamImpl<T>::GetHandle() {
return GetApi().SyncStream_GetHandle(this->p_);
}
} // namespace detail

namespace detail {
template <typename T>
Expand Down Expand Up @@ -1582,11 +1585,13 @@ inline std::vector<ConstMemoryInfo> ConstSessionImpl<T>::GetMemoryInfoForInputs(

auto num_inputs = GetInputCount();
std::vector<ConstMemoryInfo> mem_infos;
mem_infos.resize(num_inputs);
if (num_inputs > 0) {
mem_infos.resize(num_inputs);

ThrowOnError(GetApi().SessionGetMemoryInfoForInputs(this->p_,
reinterpret_cast<const OrtMemoryInfo**>(mem_infos.data()),
num_inputs));
ThrowOnError(GetApi().SessionGetMemoryInfoForInputs(this->p_,
reinterpret_cast<const OrtMemoryInfo**>(mem_infos.data()),
num_inputs));
}

return mem_infos;
}
Expand All @@ -1598,11 +1603,13 @@ inline std::vector<ConstMemoryInfo> ConstSessionImpl<T>::GetMemoryInfoForOutputs

auto num_outputs = GetOutputCount();
std::vector<ConstMemoryInfo> mem_infos;
mem_infos.resize(num_outputs);
if (num_outputs > 0) {
mem_infos.resize(num_outputs);

ThrowOnError(GetApi().SessionGetMemoryInfoForOutputs(this->p_,
reinterpret_cast<const OrtMemoryInfo**>(mem_infos.data()),
num_outputs));
ThrowOnError(GetApi().SessionGetMemoryInfoForOutputs(this->p_,
reinterpret_cast<const OrtMemoryInfo**>(mem_infos.data()),
num_outputs));
}
return mem_infos;
}

Expand Down Expand Up @@ -1631,12 +1638,12 @@ template <typename T>
inline std::vector<ConstEpDevice> ConstSessionImpl<T>::GetEpDeviceForInputs() const {
auto num_inputs = GetInputCount();
std::vector<ConstEpDevice> input_devices;
input_devices.resize(num_inputs);

ThrowOnError(GetApi().SessionGetEpDeviceForInputs(this->p_,
reinterpret_cast<const OrtEpDevice**>(input_devices.data()),
num_inputs));

if (num_inputs > 0) {
input_devices.resize(num_inputs);
ThrowOnError(GetApi().SessionGetEpDeviceForInputs(this->p_,
reinterpret_cast<const OrtEpDevice**>(input_devices.data()),
num_inputs));
}
return input_devices;
}

Expand Down
2 changes: 1 addition & 1 deletion js/common/lib/version.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@
// This file is generated by /js/scripts/update-version.ts
// Do not modify file content manually.

export const version = '1.23.0';
export const version = '1.23.1';
4 changes: 2 additions & 2 deletions js/common/package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion js/common/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
"license": "MIT",
"type": "module",
"name": "onnxruntime-common",
"version": "1.23.0",
"version": "1.23.1",
"repository": {
"url": "https://github.com/Microsoft/onnxruntime.git",
"type": "git"
Expand Down
2 changes: 1 addition & 1 deletion js/node/lib/version.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@
// This file is generated by /js/scripts/update-version.ts
// Do not modify file content manually.

export const version = '1.23.0';
export const version = '1.23.1';
6 changes: 3 additions & 3 deletions js/node/package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion js/node/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
6
]
},
"version": "1.23.0",
"version": "1.23.1",
"dependencies": {
"adm-zip": "^0.5.16",
"global-agent": "^3.0.0",
Expand Down
2 changes: 1 addition & 1 deletion js/node/script/install-metadata-versions.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@
// This file is generated by /js/scripts/update-version.ts
// Do not modify file content manually.

module.exports = { nuget: [{ feed: 'nuget', version: '1.23.0' }] };
module.exports = { nuget: [{ feed: 'nuget', version: '1.23.1' }] };
2 changes: 1 addition & 1 deletion js/react_native/lib/version.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@
// This file is generated by /js/scripts/update-version.ts
// Do not modify file content manually.

export const version = '1.23.0';
export const version = '1.23.1';
6 changes: 3 additions & 3 deletions js/react_native/package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion js/react_native/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
"registry": "https://registry.npmjs.org/"
},
"source": "lib/index",
"version": "1.23.0",
"version": "1.23.1",
"main": "dist/commonjs/index",
"homepage": "https://github.com/microsoft/onnxruntime/blob/main/js/react_native/README.md",
"files": [
Expand Down
2 changes: 1 addition & 1 deletion js/web/lib/version.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,4 +4,4 @@
// This file is generated by /js/scripts/update-version.ts
// Do not modify file content manually.

export const version = '1.23.0';
export const version = '1.23.1';
6 changes: 3 additions & 3 deletions js/web/package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion js/web/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
"type": "git"
},
"author": "fs-eire",
"version": "1.23.0",
"version": "1.23.1",
"jsdelivr": "dist/ort.min.js",
"dependencies": {
"flatbuffers": "^25.1.24",
Expand Down
6 changes: 5 additions & 1 deletion onnxruntime/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
or the `Github project <https://github.com/microsoft/onnxruntime/>`_.
"""

__version__ = "1.23.0"
__version__ = "1.23.1"
__author__ = "Microsoft"

# we need to do device version validation (for example to check Cuda version for an onnxruntime-training package).
Expand All @@ -31,14 +31,17 @@
OrtAllocatorType, # noqa: F401
OrtArenaCfg, # noqa: F401
OrtCompileApiFlags, # noqa: F401
OrtDeviceMemoryType, # noqa: F401
OrtEpDevice, # noqa: F401
OrtExecutionProviderDevicePolicy, # noqa: F401
OrtExternalInitializerInfo, # noqa: F401
OrtHardwareDevice, # noqa: F401
OrtHardwareDeviceType, # noqa: F401
OrtMemoryInfo, # noqa: F401
OrtMemoryInfoDeviceType, # noqa: F401
OrtMemType, # noqa: F401
OrtSparseFormat, # noqa: F401
OrtSyncStream, # noqa: F401
RunOptions, # noqa: F401
SessionIOBinding, # noqa: F401
SessionOptions, # noqa: F401
Expand Down Expand Up @@ -78,6 +81,7 @@
OrtDevice, # noqa: F401
OrtValue, # noqa: F401
SparseTensor, # noqa: F401
copy_tensors, # noqa: F401
)

# TODO: thiagofc: Temporary experimental namespace for new PyTorch front-end
Expand Down
2 changes: 2 additions & 0 deletions onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,7 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1,
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, QGemm);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, MLFloat16, QMoE);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, QMoE);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, MoE);
// ******** End: Quantization ******************* //

#ifdef MLAS_F16VEC_INTRINSICS_SUPPORTED
Expand Down Expand Up @@ -275,6 +276,7 @@ Status RegisterQuantizationKernels(KernelRegistry& kernel_registry) {
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, QGemm)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, MLFloat16, QMoE)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, QMoE)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, MoE)>,
};

for (auto& function_table_entry : function_table) {
Expand Down
2 changes: 1 addition & 1 deletion onnxruntime/contrib_ops/cpu/moe/moe_base_cpu.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
#include "core/common/common.h"
#include "core/framework/tensor_shape.h"
#include "core/framework/op_kernel.h"
#include "contrib_ops/cpu/moe/moe_helper.h"
#include "moe_helper.h"

Check warning on line 9 in onnxruntime/contrib_ops/cpu/moe/moe_base_cpu.h

View workflow job for this annotation

GitHub Actions / Optional Lint C++ 

[cpplint] reported by reviewdog 🐶
 Include the directory when naming header files  [build/include_subdir] [4]

Raw Output:
onnxruntime/contrib_ops/cpu/moe/moe_base_cpu.h:9:  Include the directory when naming header files  [build/include_subdir] [4]
#include <limits>

namespace onnxruntime {
Expand Down
Loading
Loading