Update handling EOS token id detection

README.md

@@ -16,7 +16,7 @@ See documentation at the [ONNX Runtime website](https://onnxruntime.ai/docs/gena
 
 | Support matrix | Supported now | Under development | On the roadmap|
 | -------------- | ------------- | ----------------- | -------------- |
-| Model architectures | ChatGLM</br>DeepSeek</br>Ernie</br>Fara</br>Gemma</br>GPTOSS</br>Granite</br>Llama</br>Mistral</br>Nemotron</br>OLMo</br>Phi</br>Phi3V</br>Phi4MM</br>Qwen</br>Qwen-2.5VL</br>SmolLM3</br>Whisper</br>| Stable diffusion ||
+| Model architectures | AMD OLMo <br/> ChatGLM <br/> DeepSeek <br/> ERNIE 4.5 <br/> Fara <br/> Gemma <br/> gpt-oss <br/> Granite <br/> Llama <br/> Mistral <br/> Nemotron <br/> Phi (language + vision) <br/> Qwen (language + vision) <br/> SmolLM3 <br/> Whisper | Stable diffusion | Multi-modal models |
 | API| Python <br/>C# <br/>C/C++ <br/> Java ^ | Objective-C ||
 | O/S | Linux <br/> Windows <br/>Mac  <br/>Android   || iOS |||
 | Architecture | x86 <br/> x64 <br/> arm64 ||||
@@ -78,10 +78,8 @@ See [installation instructions](https://onnxruntime.ai/docs/genai/howto/install)
 
    try:
       generator.append_tokens(input_tokens)
-      while True:
+      while not generator.is_done():
          generator.generate_next_token()
-         if generator.is_done():
-            break
          new_token = generator.get_next_tokens()[0]
          print(tokenizer_stream.decode(new_token), end='', flush=True)
    except KeyboardInterrupt:
@@ -115,13 +113,13 @@ Windows:
 pip list | findstr "onnxruntime-genai"
 ```
 
-Checkout the version of the examples that correspond to that release.
+Then, check out the version of the examples that corresponds to that release.
 
 ```bash
 # Clone the repo
 git clone https://github.com/microsoft/onnxruntime-genai.git && cd onnxruntime-genai
 # Checkout the branch for the version you are using
-git checkout v0.11.4
+git checkout v0.11.5
 cd examples
 ```
 
@@ -145,30 +143,11 @@ Navigate to the examples folder in the main branch.
 cd examples
 ```
 
-## Breaking API changes
+To install the nightly Python build:
 
-### v0.11.0
-
-Between `v0.11.0` and `v0.10.1`, there is a breaking API usage change to improve model quality during multi-turn conversations.
-
-Previously, the decoding loop could be written as follows.
-
-```
-while not IsDone():
-    GenerateToken()
-    GetLastToken()
-    PrintLastToken()
-```
-
-In 0.11.0, the decoding loop should now be written as follows.
-
-```
-while True:
-    GenerateToken()
-    if IsDone():
-        break
-    GetLastToken()
-    PrintLastToken()
+```bash
+# Change onnxruntime-genai to the Python package you want to install
+pip install --index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/pypi/simple/ onnxruntime-genai
 ```
 
 ## Roadmap

examples/c/src/model_chat.cpp

@@ -91,20 +91,15 @@ void CXX_API(const char* model_path, const char* execution_provider) {
     const auto current_token_count = generator->GetSequenceCount(0);
 
     try {
-      while (true) {
+      while (!generator->IsDone()) {
         generator->GenerateNextToken();
 
         if (is_first_token) {
           timing.RecordFirstTokenTimestamp();
           is_first_token = false;
         }
 
-        if (generator->IsDone()) {
-          break;
-        }
-
-        const auto num_tokens = generator->GetSequenceCount(0);
-        const auto new_token = generator->GetSequenceData(0)[num_tokens - 1];
+        const auto new_token = generator->GetNextTokens()[0];
         std::cout << tokenizer_stream->Decode(new_token) << std::flush;
       }
     } catch (const std::exception& e) {

examples/c/src/model_qa.cpp

@@ -82,20 +82,15 @@ void CXX_API(const char* model_path, const char* execution_provider) {
     generator->AppendTokenSequences(*sequences);
 
     try {
-      while (true) {
+      while (!generator->IsDone()) {
         generator->GenerateNextToken();
 
         if (is_first_token) {
           timing.RecordFirstTokenTimestamp();
           is_first_token = false;
         }
 
-        if (generator->IsDone()) {
-          break;
-        }
-
-        const auto num_tokens = generator->GetSequenceCount(0);
-        const auto new_token = generator->GetSequenceData(0)[num_tokens - 1];
+        const auto new_token = generator->GetNextTokens()[0];
         std::cout << tokenizer_stream->Decode(new_token) << std::flush;
       }
     } catch (const std::exception& e) {

examples/c/src/model_vision.cpp

@@ -90,15 +90,9 @@ void CXX_API(const char* model_path, const char* execution_provider) {
     auto generator = OgaGenerator::Create(*model, *params);
     generator->SetInputs(*input_tensors);
 
-    while (true) {
+    while (!generator->IsDone()) {
       generator->GenerateNextToken();
-
-      if (generator->IsDone()) {
-        break;
-      }
-
-      const auto num_tokens = generator->GetSequenceCount(0);
-      const auto new_token = generator->GetSequenceData(0)[num_tokens - 1];
+      const auto new_token = generator->GetNextTokens()[0];
       std::cout << stream->Decode(new_token) << std::flush;
     }
 

examples/c/src/phi4-mm.cpp

@@ -101,15 +101,9 @@ void CXX_API(const char* model_path, const char* execution_provider) {
     auto generator = OgaGenerator::Create(*model, *params);
     generator->SetInputs(*input_tensors);
 
-    while (true) {
+    while (!generator->IsDone()) {
       generator->GenerateNextToken();
-
-      if (generator->IsDone()) {
-        break;
-      }
-
-      const auto num_tokens = generator->GetSequenceCount(0);
-      const auto new_token = generator->GetSequenceData(0)[num_tokens - 1];
+      const auto new_token = generator->GetNextTokens()[0];
       std::cout << stream->Decode(new_token) << std::flush;
     }
 

examples/c/src/whisper.cpp

@@ -57,11 +57,8 @@ void CXX_API(const char* model_path, int32_t num_beams) {
     auto generator = OgaGenerator::Create(*model, *params);
     generator->SetInputs(*inputs);
 
-    while (true) {
+    while (!generator->IsDone()) {
       generator->GenerateNextToken();
-      if (generator->IsDone()) {
-        break;
-      }
     }
 
     for (size_t i = 0; i < static_cast<size_t>(num_beams * batch_size); ++i) {

examples/csharp/HelloPhi/Program.cs

@@ -127,13 +127,9 @@ static string GetPrompt(bool interactive)
             using var generator = new Generator(model, generatorParams);
             generator.AppendTokenSequences(sequences);
             var watch = System.Diagnostics.Stopwatch.StartNew();
-            while (true)
+            while (!generator.IsDone())
             {
                 generator.GenerateNextToken();
-                if (generator.IsDone())
-                {
-                    break;
-                }
             }
 
             var outputSequence = generator.GetSequence(0);
@@ -155,13 +151,9 @@ static string GetPrompt(bool interactive)
             using var generator = new Generator(model, generatorParams);
             generator.AppendTokenSequences(sequences);
             var watch = System.Diagnostics.Stopwatch.StartNew();
-            while (true)
+            while (!generator.IsDone())
             {
                 generator.GenerateNextToken();
-                if (generator.IsDone())
-                {
-                    break;
-                }
                 Console.Write(tokenizerStream.Decode(generator.GetNextTokens()[0]));
             }
             Console.WriteLine();
@@ -196,13 +188,9 @@ static string GetPrompt(bool interactive)
         var sequences = tokenizer.Encode(tokenizer.ApplyChatTemplate("", messages, "", true));
         var watch = System.Diagnostics.Stopwatch.StartNew();
         generator.AppendTokenSequences(sequences);
-        while (true)
+        while (!generator.IsDone())
         {
             generator.GenerateNextToken();
-            if (generator.IsDone())
-            {
-                break;
-            }
             Console.Write(tokenizerStream.Decode(generator.GetNextTokens()[0]));
         }
         Console.WriteLine();

examples/csharp/HelloPhi3V/Program.cs

@@ -168,13 +168,9 @@ void PrintUsage()
     using var generator = new Generator(model, generatorParams);
     generator.SetInputs(inputTensors);
     var watch = System.Diagnostics.Stopwatch.StartNew();
-    while (true)
+    while (!generator.IsDone())
     {
         generator.GenerateNextToken();
-        if (generator.IsDone())
-        {
-            break;
-        }
         Console.Write(stream.Decode(generator.GetNextTokens()[0]));
     }
     watch.Stop();

examples/csharp/HelloPhi4MM/Program.cs

@@ -215,13 +215,9 @@ void PrintUsage()
     using var generator = new Generator(model, generatorParams);
     generator.SetInputs(inputTensors);
     var watch = System.Diagnostics.Stopwatch.StartNew();
-    while (true)
+    while (!generator.IsDone())
     {
         generator.GenerateNextToken();
-        if (generator.IsDone())
-        {
-            break;
-        }
         Console.Write(stream.Decode(generator.GetNextTokens()[0]));
     }
     watch.Stop();

examples/python/awq-quantized-model.py

@@ -108,11 +108,8 @@ def run_model(args):
         print("Output: ", end="", flush=True)
 
         try:
-            while True:
+            while not generator.is_done():
                 generator.generate_next_token()
-                if generator.is_done():
-                    break
-
                 new_token = generator.get_next_tokens()[0]
                 print(tokenizer_stream.decode(new_token), end="", flush=True)
         except KeyboardInterrupt:

examples/python/model-chat.py

@@ -216,16 +216,13 @@ def main(args):
         print("Output: ", end="", flush=True)
 
         try:
-            while True:
+            while not generator.is_done():
                 generator.generate_next_token()
                 if args.timings:
                     if first:
                         first_token_timestamp = time.time()
                         first = False
 
-                if generator.is_done():
-                    break
-
                 new_token = generator.get_next_tokens()[0]
                 print(tokenizer_stream.decode(new_token), end="", flush=True)
                 if args.timings:

examples/python/model-generate.py

@@ -99,10 +99,8 @@ def main(args):
     if args.verbose:
         print("Generating tokens ...\n")
     start_time = time.time()
-    while True:
+    while not generator.is_done():
         generator.generate_next_token()
-        if generator.is_done():
-            break
     run_time = time.time() - start_time
 
     for i in range(len(prompts)):

examples/python/model-qa.py

@@ -207,16 +207,13 @@ def main(args):
         print("Output: ", end="", flush=True)
 
         try:
-            while True:
+            while not generator.is_done():
                 generator.generate_next_token()
                 if args.timings:
                     if first:
                         first_token_timestamp = time.time()
                         first = False
 
-                if generator.is_done():
-                    break
-
                 new_token = generator.get_next_tokens()[0]
                 print(tokenizer_stream.decode(new_token), end="", flush=True)
                 if args.timings:

examples/python/model-vision.py

@@ -150,11 +150,8 @@ def run(args: argparse.Namespace):
         generator.set_inputs(inputs)
         start_time = time.time()
 
-        while True:
+        while not generator.is_done():
             generator.generate_next_token()
-            if generator.is_done():
-                break
-
             new_token = generator.get_next_tokens()[0]
             print(stream.decode(new_token), end="", flush=True)
 

examples/python/phi3-qa.py

@@ -75,16 +75,13 @@ def main(args):
         print("Output: ", end="", flush=True)
 
         try:
-            while True:
+            while not generator.is_done():
                 generator.generate_next_token()
                 if args.timings:
                     if first:
                         first_token_timestamp = time.time()
                         first = False
 
-                if generator.is_done():
-                    break
-
                 new_token = generator.get_next_tokens()[0]
                 print(tokenizer_stream.decode(new_token), end="", flush=True)
                 if args.timings:

examples/python/phi4-mm.py

@@ -145,11 +145,8 @@ def run(args: argparse.Namespace):
         generator.set_inputs(inputs)
         start_time = time.time()
 
-        while True:
+        while not generator.is_done():
             generator.generate_next_token()
-            if generator.is_done():
-                break
-
             new_token = generator.get_next_tokens()[0]
             print(tokenizer_stream.decode(new_token), end="", flush=True)
 

src/config.h

@@ -301,7 +301,7 @@ struct Config {
     int top_k{50};                     // Number of highest probability vocabulary tokens to keep for top-k-filtering that will be used by default in the generate method of the model.
     float top_p{};                     // If set to float >0 and <1, only the most probable tokens with probabilities that add up to top_p or higher are kept for generation.
     float temperature{1.0f};           // Temperature to control during generation. Default is 1.0.
-    bool early_stopping{true};         //  Whether to stop the beam search when at least num_beams sentences are finished per batch or not.
+    bool early_stopping{true};         // Whether to stop the beam search when at least num_beams sentences are finished per batch or not.
     int no_repeat_ngram_size{};        // Unused param
     float diversity_penalty{};         // Unused param
     float length_penalty{1.0f};        // Exponential penalty to the length that is used with beam-based generation. length_penalty > 0.0 promotes longer sequences, while length_penalty < 0.0 encourages shorter sequences.

src/csharp/Generator.cs

@@ -46,6 +46,15 @@ public void AppendTokenSequences(Sequences sequences)
             Result.VerifySuccess(NativeMethods.OgaGenerator_AppendTokenSequences(_generatorHandle, sequences.Handle));
         }
 
+        /// <summary>
+        /// Gets the number of tokens in the generator
+        /// </summary>
+        /// <returns>The token count</returns>
+        public ulong TokenCount()
+        {
+            return NativeMethods.OgaGenerator_TokenCount(_generatorHandle).ToUInt64();
+        }
+
         public void GenerateNextToken()
         {
             Result.VerifySuccess(NativeMethods.OgaGenerator_GenerateNextToken(_generatorHandle));

src/csharp/GeneratorParams.cs

@@ -29,14 +29,21 @@ public void SetSearchOption(string searchOption, bool value)
             Result.VerifySuccess(NativeMethods.OgaGeneratorParamsSetSearchBool(_generatorParamsHandle, StringUtils.ToUtf8(searchOption), value));
         }
 
-        public void TryGraphCaptureWithMaxBatchSize(int maxBatchSize)
+        public void SetGuidance(string type, string data, bool enableFFTokens = false)
         {
-            Console.WriteLine("TryGraphCaptureWithMaxBatchSize is deprecated and will be removed in a future release.");
+            Result.VerifySuccess(NativeMethods.OgaGeneratorParamsSetGuidance(_generatorParamsHandle, StringUtils.ToUtf8(type), StringUtils.ToUtf8(data), enableFFTokens));
         }
 
-        public void SetGuidance(string type, string data, bool enableFFTokens = false)
+        public double GetSearchNumber(string searchOption)
         {
-            Result.VerifySuccess(NativeMethods.OgaGeneratorParamsSetGuidance(_generatorParamsHandle, StringUtils.ToUtf8(type), StringUtils.ToUtf8(data), enableFFTokens));
+            Result.VerifySuccess(NativeMethods.OgaGeneratorParamsGetSearchNumber(_generatorParamsHandle, StringUtils.ToUtf8(searchOption), out double value));
+            return value;
+        }
+
+        public bool GetSearchBool(string searchOption)
+        {
+            Result.VerifySuccess(NativeMethods.OgaGeneratorParamsGetSearchBool(_generatorParamsHandle, StringUtils.ToUtf8(searchOption), out bool value));
+            return value;
         }
 
         ~GeneratorParams()