Cherrypick 2

.pipelines/stages/jobs/py-validation-job.yml

@@ -208,7 +208,9 @@ jobs:
         displayName: 'Download CUDA $(cuda_version)'
         workingDirectory: '$(Build.Repository.LocalPath)'
     - powershell: |
-        python -m pip install -r test/python/requirements.txt
+        if ("$(arch)" -ne "arm64") {
+          python -m pip install -r test/python/requirements.txt
+        }
         if ("$(ep)" -eq "cuda") {
           $env:CUDA_PATH = '$(Build.Repository.LocalPath)\cuda_sdk\v$(cuda_version)'
           $env:PATH = "$env:CUDA_PATH\bin;$env:CUDA_PATH\extras\CUPTI\lib64;$env:PATH"
@@ -220,14 +222,18 @@ jobs:
           python -m pip install -r test/python/directml/torch/requirements.txt
           python -m pip install -r test/python/directml/ort/requirements.txt
         }
+        elseif ("$(arch)" -eq "arm64") {
+          python -m pip install numpy<2
+          python -m pip install onnxruntime-qnn==1.20.0
+        }
         else {
           python -m pip install -r test/python/cpu/torch/requirements.txt
           python -m pip install -r test/python/cpu/ort/requirements.txt
         }
         cd examples\python
         python -m pip install --no-index --find-links=$(Build.BinariesDirectory)/wheel $(pip_package_name)
 
-        python model-generate.py -m .\models\$(prebuild_phi3_mini_model_folder) --min_length 25 --max_length 50 --verbose
+        python model-generate.py -m .\models\$(prebuild_phi3_mini_model_folder) --min_length 25 --max_length 50 --batch_size_for_cuda_graph 3 --verbose
       displayName: 'Run Example With Artifact'
       workingDirectory: '$(Build.Repository.LocalPath)'
 

.pipelines/stages/jobs/steps/utils/download-huggingface-model.yml

@@ -28,7 +28,8 @@ steps:
     - powershell: |
         python -m pip install "huggingface_hub[cli]"
         huggingface-cli login --token $env:HF_TOKEN
-        huggingface-cli download ${{ parameters.HuggingFaceRepo }} --include ${{ parameters.RepoFolder }}/* --local-dir ${{ parameters.LocalFolder }} --local-dir-use-symlinks False
+        # Use maximum path length for Windows... otherwises hits the path character limit
+        huggingface-cli download ${{ parameters.HuggingFaceRepo }} --include ${{ parameters.RepoFolder }}/* --local-dir "\\?\${{ parameters.WorkingDirectory }}\\${{ parameters.LocalFolder }}" --local-dir-use-symlinks False
       displayName: ${{ parameters.StepName }}
       workingDirectory: ${{ parameters.WorkingDirectory }}
       env:

VERSION_INFO

@@ -1 +1 @@
-0.5.0-rc1
+0.5.0

documents/Runtime_option.md

@@ -0,0 +1,15 @@
+# Runtime Options
+
+This file will provide details on the usage of SetRuntimeOption API. It will list all the current key value pairs which can be used as an input for this API.
+
+## Set Terminate
+
+Set Terminate is a runtime option to terminate the current session or continue/restart an already terminated session. There are two valid ways to call Set Terminate.
+
+To enable terminate, the valid pair is: ("set_terminate", "1")
+
+To disable terminate, the valid pair is: ("set_terminate", "0")
+
+Key: "set_terminate"
+
+Accepted values: ("0", "1")

examples/csharp/HelloPhi/HelloPhi.csproj

@@ -10,9 +10,9 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.ML.OnnxRuntimeGenAI" Version="[0.5.0-dev]" Condition=" '$(Configuration)' == 'Debug' OR '$(Configuration)' == 'Release' " />
-    <PackageReference Include="Microsoft.ML.OnnxRuntimeGenAI.Cuda" Version="[0.5.0-dev]" Condition=" '$(Configuration)' == 'Debug_Cuda' OR '$(Configuration)' == 'Release_Cuda' " />
-    <PackageReference Include="Microsoft.ML.OnnxRuntimeGenAI.DirectML" Version="0.5.0-dev" Condition=" '$(Configuration)' == 'Debug_DirectML' OR '$(Configuration)' == 'Release_DirectML' " />
+    <PackageReference Include="Microsoft.ML.OnnxRuntimeGenAI" Version="[0.5.0]" Condition=" '$(Configuration)' == 'Debug' OR '$(Configuration)' == 'Release' " />
+    <PackageReference Include="Microsoft.ML.OnnxRuntimeGenAI.Cuda" Version="[0.5.0]" Condition=" '$(Configuration)' == 'Debug_Cuda' OR '$(Configuration)' == 'Release_Cuda' " />
+    <PackageReference Include="Microsoft.ML.OnnxRuntimeGenAI.DirectML" Version="0.5.0" Condition=" '$(Configuration)' == 'Debug_DirectML' OR '$(Configuration)' == 'Release_DirectML' " />
   </ItemGroup>
 
   <ItemGroup>

nuget/PACKAGE.md

@@ -121,11 +121,11 @@ This implementation checks if a number is prime by iterating only up to the squa
 ## Source code repository
 
 ONNX Runtime is an open source project. See:
-* (https://github.com/microsoft/onnxruntime)[https://github.com/microsoft/onnxruntime]
-* (https://github.com/microsoft/onnxruntime-genai)[https://github.com/microsoft/onnxruntime-genai]
+* (ONNX Runtime)[https://github.com/microsoft/onnxruntime]
+* (ONNX Runtime GenAI)[https://github.com/microsoft/onnxruntime-genai]
 
 ## Documentation
 
-See (https://onxxruntime.ai/docs/genai)[https://onxxruntime.ai/docs/genai]
+See (ONNX Runtime GenAI Documentation)[https://onxxruntime.ai/docs/genai]
 
 

src/generators.cpp

@@ -30,6 +30,11 @@ std::string CurrentModulePath() {
 }
 #endif
 
+void ThrowErrorIfSessionTerminated(bool is_session_terminated) {
+  if (is_session_terminated)
+    throw std::runtime_error("Session in Terminated state, exiting!");
+}
+
 namespace Generators {
 
 #if USE_CUDA
@@ -284,6 +289,7 @@ Generator::Generator(const Model& model, const GeneratorParams& params) : model_
 }
 
 void Generator::ComputeLogits() {
+  ThrowErrorIfSessionTerminated(state_->session_terminated_);
   if (computed_logits_)
     throw std::runtime_error("ComputeLogits called again without calling GenerateNextToken first");
 
@@ -301,7 +307,25 @@ void Generator::ComputeLogits() {
   search_->ApplyRepetitionPenalty(search.repetition_penalty);
 }
 
+void Generator::SetRuntimeOption(const char* key, const char* value) {
+  // TODO: Need a better way to handle different keys
+  // We can create a config manager to host all configurations and do comparison at that point
+  if (strcmp(key, "terminate_session") == 0) {
+    if (strcmp(value, "0") == 0) {
+      state_->UnsetTerminate();
+    } else if (strcmp(value, "1") == 0) {
+      state_->SetTerminate();
+    } else {
+      // Value not expected
+      throw std::runtime_error(std::string("terminate_session key value unexpected: ") + value);
+    }
+  } else {
+    throw std::runtime_error(std::string("SetRuntimeOption key is not expected: ") + key);
+  }
+}
+
 bool Generator::IsDone() const {
+  ThrowErrorIfSessionTerminated(state_->session_terminated_);
   if (computed_logits_)
     throw std::runtime_error("IsDone() can't be called in the middle of processing logits");
 
@@ -313,7 +337,12 @@ bool Generator::IsDone() const {
   return is_done;
 }
 
+bool Generator::IsSessionTerminated() const {
+  return state_->session_terminated_;
+}
+
 void Generator::GenerateNextToken() {
+  ThrowErrorIfSessionTerminated(state_->session_terminated_);
   if (!computed_logits_)
     throw std::runtime_error("Must call ComputeLogits before GenerateNextToken");
   computed_logits_ = false;

src/generators.h

@@ -40,6 +40,8 @@ using cudaStream_t = void*;
 #include "runtime_settings.h"
 #include "tensor.h"
 
+void ThrowErrorIfSessionTerminated(bool is_session_terminated);
+
 namespace Generators {
 struct Model;
 struct State;
@@ -108,7 +110,9 @@ struct Generator : LeakChecked<Generator> {
   Generator(const Model& model, const GeneratorParams& params);
 
   bool IsDone() const;
+  void SetRuntimeOption(const char* key, const char* value);
   void ComputeLogits();
+  bool IsSessionTerminated() const;
   void GenerateNextToken();
 
   DeviceMemorySpan<int32_t> GetSequence(size_t index) const;

src/models/model.cpp

@@ -65,7 +65,18 @@ void State::Run(OrtSession& session, int new_batch_size) {
   }
 }
 
+void State::SetTerminate() {
+  session_terminated_ = true;
+  run_options_->SetTerminate();
+}
+
+void State::UnsetTerminate() {
+  session_terminated_ = false;
+  run_options_->UnsetTerminate();
+}
+
 OrtValue* State::GetInput(const char* name) {
+  ThrowErrorIfSessionTerminated(session_terminated_);
   for (size_t i = 0; i < input_names_.size(); i++) {
     if (std::strcmp(input_names_[i], name) == 0) {
       return inputs_[i];
@@ -75,6 +86,7 @@ OrtValue* State::GetInput(const char* name) {
 }
 
 OrtValue* State::GetOutput(const char* name) {
+  ThrowErrorIfSessionTerminated(session_terminated_);
   for (size_t i = 0; i < output_names_.size(); i++) {
     if (std::strcmp(output_names_[i], name) == 0) {
       return outputs_[i];

src/models/model.h

@@ -34,6 +34,9 @@ struct State {
   virtual const CapturedGraphInfo* GetCapturedGraphInfo() const { return nullptr; }
   virtual void Finalize() {}
 
+  void SetTerminate();
+  void UnsetTerminate();
+  mutable bool session_terminated_{};
   OrtValue* GetInput(const char* name);
 
   virtual OrtValue* GetOutput(const char* name);

src/ort_genai.h

@@ -258,6 +258,10 @@ struct OgaGenerator : OgaAbstract {
     return OgaGenerator_IsDone(this);
   }
 
+  bool IsSessionTerminated() const {
+    return OgaGenerator_IsSessionTerminated(this);
+  }
+
   void ComputeLogits() {
     OgaCheckResult(OgaGenerator_ComputeLogits(this));
   }
@@ -266,6 +270,10 @@ struct OgaGenerator : OgaAbstract {
     OgaCheckResult(OgaGenerator_GenerateNextToken(this));
   }
 
+  void SetRuntimeOption(const char* key, const char* value) {
+    OgaCheckResult(OgaGenerator_SetRuntimeOption(this, key, value));
+  }
+
   size_t GetSequenceCount(size_t index) const {
     return OgaGenerator_GetSequenceCount(this, index);
   }

src/ort_genai_c.cpp

@@ -271,6 +271,10 @@ bool OGA_API_CALL OgaGenerator_IsDone(const OgaGenerator* generator) {
   return reinterpret_cast<const Generators::Generator*>(generator)->IsDone();
 }
 
+bool OGA_API_CALL OgaGenerator_IsSessionTerminated(const OgaGenerator* generator) {
+  return reinterpret_cast<const Generators::Generator*>(generator)->IsSessionTerminated();
+}
+
 OgaResult* OGA_API_CALL OgaGenerator_ComputeLogits(OgaGenerator* generator) {
   OGA_TRY
   reinterpret_cast<Generators::Generator*>(generator)->ComputeLogits();
@@ -285,6 +289,13 @@ OgaResult* OGA_API_CALL OgaGenerator_GenerateNextToken(OgaGenerator* generator)
   OGA_CATCH
 }
 
+OgaResult* OGA_API_CALL OgaGenerator_SetRuntimeOption(OgaGenerator* generator, const char* key, const char* value) {
+  OGA_TRY
+  reinterpret_cast<Generators::Generator*>(generator)->SetRuntimeOption(key, value);
+  return nullptr;
+  OGA_CATCH
+}
+
 OgaResult* OGA_API_CALL OgaGenerator_GetOutput(const OgaGenerator* oga_generator, const char* name, OgaTensor** out) {
   OGA_TRY
   auto& generator = *reinterpret_cast<const Generators::Generator*>(oga_generator);

src/ort_genai_c.h

@@ -278,6 +278,7 @@ OGA_EXPORT void OGA_API_CALL OgaDestroyGenerator(OgaGenerator* generator);
  * \return True if the generator has finished generating all the sequences, false otherwise.
  */
 OGA_EXPORT bool OGA_API_CALL OgaGenerator_IsDone(const OgaGenerator* generator);
+OGA_EXPORT bool OGA_API_CALL OgaGenerator_IsSessionTerminated(const OgaGenerator* generator);
 
 /*
  * \brief Computes the logits from the model based on the input ids and the past state. The computed logits are stored in the generator.
@@ -287,6 +288,8 @@ OGA_EXPORT bool OGA_API_CALL OgaGenerator_IsDone(const OgaGenerator* generator);
 OGA_EXPORT OgaResult* OGA_API_CALL OgaGenerator_ComputeLogits(OgaGenerator* generator);
 OGA_EXPORT OgaResult* OGA_API_CALL OgaGenerator_GenerateNextToken(OgaGenerator* generator);
 
+OGA_EXPORT OgaResult* OGA_API_CALL OgaGenerator_SetRuntimeOption(OgaGenerator* generator, const char* key, const char* value);
+
 /*
  * \brief Returns a copy of the model output identified by the given name as an OgaTensor on CPU. The buffer is owned by returned OgaTensor
  *       and will be released when the OgaTensor is destroyed