diff --git a/cmake/deps.txt b/cmake/deps.txt index 9d706bef41..faf47d240e 100644 --- a/cmake/deps.txt +++ b/cmake/deps.txt @@ -14,4 +14,4 @@ pybind11;https://github.com/pybind/pybind11/archive/refs/tags/v2.13.6.zip;f78029 googletest;https://github.com/google/googletest/archive/530d5c8c84abd2a46f38583ee817743c9b3a42b4.zip;5e3a61db2aa975cfd0f97ba92c818744e7fa7034 microsoft_wil;https://github.com/microsoft/wil/archive/refs/tags/v1.0.230629.1.zip;e4a542a323c070376f7c2d1973d0f7ddbc1d2fa5 directx_headers;https://github.com/microsoft/DirectX-Headers/archive/refs/tags/v1.613.1.zip;47653509a3371eabb156360f42faf582f314bf2e -onnxruntime_extensions;https://github.com/microsoft/onnxruntime-extensions.git;d1daadcb53a80645b3d96218e4713f24c12dfaf0 +onnxruntime_extensions;https://github.com/microsoft/onnxruntime-extensions.git;fc004859e82241e99d458a90d2a39d400050cc59 diff --git a/src/csharp/NativeMethods.cs b/src/csharp/NativeMethods.cs index 58f07939f7..b0e9008f74 100644 --- a/src/csharp/NativeMethods.cs +++ b/src/csharp/NativeMethods.cs @@ -186,7 +186,6 @@ public static extern UIntPtr OgaSequencesGetSequenceCount(IntPtr /* const OgaSeq byte[] /* const char* */ strings, IntPtr /* OgaSequences* */ sequences); - // This function is used to decode the given token into a string. The caller is responsible for freeing the // returned string using the OgaDestroyString function when it is no longer needed. [DllImport(NativeLib.DllName, CallingConvention = CallingConvention.Winapi)] diff --git a/src/models/model.cpp b/src/models/model.cpp index 82d78d82c7..7ae3579b1d 100644 --- a/src/models/model.cpp +++ b/src/models/model.cpp @@ -198,7 +198,7 @@ std::unique_ptr Tokenizer::CreateStream() const { std::vector Tokenizer::Encode(const char* text) const { OrtxPtr ids; - CheckResult(OrtxTokenize(tokenizer_, &text, 1, ids.Address())); + CheckResult(OrtxTokenizeWithOptions(tokenizer_, &text, 1, ids.Address(), false /* add_special_tokens */)); const extTokenId_t* tokens; size_t count;