diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 96300bbaa1..9cced1adb7 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -149,7 +149,7 @@ The repo supports specific model architectures - check `src/models/model_type.h` ### Performance Considerations -- **KV caching** is automatically managed but can be configured via `runtime_settings.cpp` +- **KV caching** is automatically managed but can be configured via `src/runtime_settings.cpp` - **Continuous decoding** (session continuation) requires careful state management - **Multi-LoRA** adapters use separate weight loading in `src/models/adapters.cpp` diff --git a/benchmark/c/main.cpp b/benchmark/c/main.cpp index d2ded8cdcc..f7d2579e39 100644 --- a/benchmark/c/main.cpp +++ b/benchmark/c/main.cpp @@ -4,11 +4,14 @@ #include #include #include +#include +#include #include #include #include #include #include +#include #include #include "ort_genai.h" @@ -158,7 +161,10 @@ void RunBenchmark(const benchmark::Options& opts) { bool need_generate_prompt = false; std::string prompt; - if (const size_t* npt = std::get_if(&opts.prompt_num_tokens_or_content)) { + if (opts.use_random_tokens) { + num_prompt_tokens = std::get(opts.prompt_num_tokens_or_content); + need_generate_prompt = false; + } else if (const size_t* npt = std::get_if(&opts.prompt_num_tokens_or_content)) { num_prompt_tokens = *npt; need_generate_prompt = true; } else { @@ -203,8 +209,21 @@ void RunBenchmark(const benchmark::Options& opts) { } auto prompt_sequences = OgaSequences::Create(); - for (size_t i = 0; i < opts.batch_size; ++i) { - tokenizer->Encode(prompt.c_str(), *prompt_sequences); + if (opts.use_random_tokens) { + std::random_device rd; + std::mt19937 rng(rd()); + std::uniform_int_distribution dist(0, 99); + for (size_t i = 0; i < opts.batch_size; ++i) { + std::vector random_tokens(num_prompt_tokens); + std::generate(random_tokens.begin(), random_tokens.end(), [&]() { + return dist(rng); + }); + prompt_sequences->Append(random_tokens.data(), random_tokens.size()); + } + } else { + for (size_t i = 0; i < opts.batch_size; ++i) { + tokenizer->Encode(prompt.c_str(), *prompt_sequences); + } } // warmup @@ -226,7 +245,12 @@ void RunBenchmark(const benchmark::Options& opts) { if (opts.verbose && i == 0) { // show prompt and output on first iteration - std::cout << "[PROMPT BEGIN]" << prompt << "[PROMPT END]\n"; + if (opts.use_random_tokens) { + std::cout << "[PROMPT] random token IDs in [0, 99], batch_size=" << opts.batch_size + << ", tokens per sequence=" << num_prompt_tokens << "\n"; + } else { + std::cout << "[PROMPT BEGIN]" << prompt << "[PROMPT END]\n"; + } const auto output_sequence_length = gen->TokenCount(); const auto* output_sequence_data = gen->GetSequenceData(0); const auto output = tokenizer->Decode(output_sequence_data, output_sequence_length); diff --git a/benchmark/c/options.cpp b/benchmark/c/options.cpp index 63c66cc23e..752ea8716b 100644 --- a/benchmark/c/options.cpp +++ b/benchmark/c/options.cpp @@ -40,7 +40,12 @@ namespace { << " Prompt text to use. Default: See --prompt_length.\n" << " --prompt_file \n" << " Path to file containing prompt text to use. Default: See --prompt_length.\n" - << " Note: --prompt_length, --prompt, and --prompt_file are mutually exclusive.\n" + << " --use_random_tokens\n" + << " Use random token IDs in [0, 99] per position instead of generating or encoding\n" + << " a text prompt. Requires -l/--prompt_length (cannot be used with --prompt or\n" + << " --prompt_file).\n" + << " Note: --prompt, --prompt_file, and --use_random_tokens are mutually exclusive;\n" + << " --use_random_tokens requires --prompt_length.\n" << " -g,--generation_length \n" << " Number of tokens to generate. Default: " << defaults.num_tokens_to_generate << "\n" << " -r,--repetitions \n" @@ -125,10 +130,16 @@ Options ParseOptionsFromCommandLine(int argc, const char* const* argv) { } else if (arg == "-b" || arg == "--batch_size") { opts.batch_size = ParseNumber(next_arg(i)); } else if (arg == "-l" || arg == "--prompt_length") { + if (prompt_num_tokens_or_content.has_value()) + throw std::runtime_error("--prompt_length, --prompt, and --prompt_file are mutually exclusive."); prompt_num_tokens_or_content = ParseNumber(next_arg(i)); } else if (arg == "--prompt") { + if (prompt_num_tokens_or_content.has_value()) + throw std::runtime_error("--prompt_length, --prompt, and --prompt_file are mutually exclusive."); prompt_num_tokens_or_content = std::string{next_arg(i)}; } else if (arg == "--prompt_file") { + if (prompt_num_tokens_or_content.has_value()) + throw std::runtime_error("--prompt_length, --prompt, and --prompt_file are mutually exclusive."); prompt_num_tokens_or_content = ReadFileContent(next_arg(i)); } else if (arg == "-g" || arg == "--generation_length") { opts.num_tokens_to_generate = ParseNumber(next_arg(i)); @@ -140,6 +151,8 @@ Options ParseOptionsFromCommandLine(int argc, const char* const* argv) { opts.max_length = ParseNumber(next_arg(i)); } else if (arg == "--reuse_generator") { opts.reuse_generator = true; + } else if (arg == "--use_random_tokens") { + opts.use_random_tokens = true; } else if (arg == "-v" || arg == "--verbose") { opts.verbose = true; } else if (arg == "-h" || arg == "--help") { @@ -153,6 +166,15 @@ Options ParseOptionsFromCommandLine(int argc, const char* const* argv) { opts.prompt_num_tokens_or_content = std::move(*prompt_num_tokens_or_content); } + if (opts.use_random_tokens) { + if (!prompt_num_tokens_or_content.has_value() || + !std::holds_alternative(*prompt_num_tokens_or_content)) { + throw std::runtime_error(!prompt_num_tokens_or_content.has_value() + ? "--use_random_tokens requires -l/--prompt_length." + : "--use_random_tokens cannot be used with --prompt or --prompt_file."); + } + } + VerifyOptions(opts); return opts; diff --git a/benchmark/c/options.h b/benchmark/c/options.h index 8e774f0934..cd23b2ed1f 100644 --- a/benchmark/c/options.h +++ b/benchmark/c/options.h @@ -22,6 +22,7 @@ struct Options { int64_t max_length{0}; bool verbose{}; bool reuse_generator{}; + bool use_random_tokens{}; }; Options ParseOptionsFromCommandLine(int argc, const char* const* argv);