From cadf52baa44d1ffeb4864bf1b6dbdf8f9f5f0520 Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Thu, 26 Sep 2024 17:14:23 +0800
Subject: [PATCH 01/42] example scripts
---
examples/csharp/HelloPhi/HelloPhi.csproj | 14 +-
examples/csharp/HelloPhi/Program.cs | 10 +-
examples/csharp/HelloPhi3V/HelloPhi3V.csproj | 6 +-
examples/csharp/HelloPhi3V/Program.cs | 155 ++++++++++++-------
examples/python/phi3v.py | 49 ++++--
5 files changed, 151 insertions(+), 83 deletions(-)
diff --git a/examples/csharp/HelloPhi/HelloPhi.csproj b/examples/csharp/HelloPhi/HelloPhi.csproj
index 3cc34b8e90..482abb5270 100644
--- a/examples/csharp/HelloPhi/HelloPhi.csproj
+++ b/examples/csharp/HelloPhi/HelloPhi.csproj
@@ -10,17 +10,9 @@
-
-
-
-
-
-
-
- PreserveNewest
- false
- "phi-2\"
-
+
+
+
diff --git a/examples/csharp/HelloPhi/Program.cs b/examples/csharp/HelloPhi/Program.cs
index 26e20a3530..f5448d8b44 100644
--- a/examples/csharp/HelloPhi/Program.cs
+++ b/examples/csharp/HelloPhi/Program.cs
@@ -1,11 +1,15 @@
-// See https://aka.ms/new-console-template for more information
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
using Microsoft.ML.OnnxRuntimeGenAI;
void PrintUsage()
{
Console.WriteLine("Usage:");
Console.WriteLine(" -m model_path");
- Console.WriteLine(" -i (optional): Interactive mode");
+ Console.WriteLine("\t\t\t\tPath to the model");
+ Console.WriteLine(" --interactive (optional)");
+ Console.WriteLine("\t\t\t\tInteractive mode");
}
using OgaHandle ogaHandle = new OgaHandle();
@@ -23,7 +27,7 @@ void PrintUsage()
while (i < args.Length)
{
var arg = args[i];
- if (arg == "-i")
+ if (arg == "--interactive")
{
interactive = true;
}
diff --git a/examples/csharp/HelloPhi3V/HelloPhi3V.csproj b/examples/csharp/HelloPhi3V/HelloPhi3V.csproj
index ab7fc6c44d..2a85abc0e0 100644
--- a/examples/csharp/HelloPhi3V/HelloPhi3V.csproj
+++ b/examples/csharp/HelloPhi3V/HelloPhi3V.csproj
@@ -9,9 +9,9 @@
-
-
-
+
+
+
diff --git a/examples/csharp/HelloPhi3V/Program.cs b/examples/csharp/HelloPhi3V/Program.cs
index 1f03a45d1c..bcbad2b61b 100644
--- a/examples/csharp/HelloPhi3V/Program.cs
+++ b/examples/csharp/HelloPhi3V/Program.cs
@@ -4,79 +4,120 @@
using Microsoft.ML.OnnxRuntimeGenAI;
using System.Linq;
-class Program
+void PrintUsage()
{
- static void Run(string modelPath)
- {
- using Model model = new Model(modelPath);
- using MultiModalProcessor processor = new MultiModalProcessor(model);
- using var tokenizerStream = processor.CreateStream();
+ Console.WriteLine("Usage:");
+ Console.WriteLine(" -m model_path");
+ Console.WriteLine("\t\t\t\tPath to the model");
+ Console.WriteLine(" --image_paths");
+ Console.WriteLine("\t\t\t\tPath to the images");
+ Console.WriteLine(" --interactive (optional)");
+ Console.WriteLine("\t\t\t\tInteractive mode");
+}
+
+using OgaHandle ogaHandle = new OgaHandle();
+
+if (args.Length < 1)
+{
+ PrintUsage();
+ Environment.Exit(-1);
+}
+
+bool interactive = false;
+string modelPath = string.Empty;
+string[] imagePaths = new string[0];
- while (true)
+uint i_arg = 0;
+while (i_arg < args.Length)
+{
+ var arg = args[i_arg];
+ if (arg == "--interactive")
+ {
+ interactive = true;
+ }
+ else if (arg == "-m")
+ {
+ if (i_arg + 1 < args.Length)
{
- Console.WriteLine("Image Path (comma separated; leave empty if no image):");
- string[] imagePaths = Console.ReadLine().Split(',').ToList().Select(i => i.ToString().Trim()).ToArray();
+ modelPath = Path.Combine(args[i_arg+1]);
+ }
+ }
+ else if (arg == "--image_paths")
+ {
+ if (i_arg + 1 < args.Length)
+ {
+ imagePaths = args[i_arg+1].Split(',').ToList().Select(i => i.ToString().Trim()).ToArray();
+ }
+ }
+ i_arg++;
+}
- Images images = null;
- if (imagePaths.Length == 0)
- {
- Console.WriteLine("No image provided");
- }
- else
- {
- for (int i = 0; i < imagePaths.Length; i++)
- {
- string imagePath = imagePaths[i].Trim();
- if (!File.Exists(imagePath))
- {
- throw new Exception("Image file not found: " + imagePath);
- }
- }
- images = Images.Load(imagePaths);
- }
+Console.WriteLine("--------------------");
+Console.WriteLine("Hello, Phi-3-Vision!");
+Console.WriteLine("--------------------");
- Console.WriteLine("Prompt:");
- string text = Console.ReadLine();
- string prompt = "<|user|>\n";
- if (images != null)
- {
- for (int i = 0; i < imagePaths.Length; i++)
- {
- prompt += "<|image_" + (i + 1) + "|>\n";
- }
- }
- prompt += text + "<|end|>\n<|assistant|>\n";
+Console.WriteLine("Model path: " + modelPath);
+Console.WriteLine("Interactive: " + interactive);
- Console.WriteLine("Processing image and prompt...");
- var inputTensors = processor.ProcessImages(prompt, images);
+using Model model = new Model(modelPath);
+using MultiModalProcessor processor = new MultiModalProcessor(model);
+using var tokenizerStream = processor.CreateStream();
- Console.WriteLine("Generating response...");
- using GeneratorParams generatorParams = new GeneratorParams(model);
- generatorParams.SetSearchOption("max_length", 7680);
- generatorParams.SetInputs(inputTensors);
+do
+{
+ if (interactive)
+ {
+ Console.WriteLine("Image Path (comma separated; leave empty if no image):");
+ imagePaths = Console.ReadLine().Split(',').ToList().Select(i => i.ToString().Trim()).ToArray();
+ }
- using var generator = new Generator(model, generatorParams);
- while (!generator.IsDone())
+ Images images = null;
+ if (imagePaths.Length == 0)
+ {
+ Console.WriteLine("No image provided");
+ }
+ else
+ {
+ for (int i = 0; i < imagePaths.Length; i++)
+ {
+ string imagePath = imagePaths[i].Trim();
+ if (!File.Exists(imagePath))
{
- generator.ComputeLogits();
- generator.GenerateNextToken();
- Console.Write(tokenizerStream.Decode(generator.GetSequence(0)[^1]));
+ throw new Exception("Image file not found: " + imagePath);
}
}
+ images = Images.Load(imagePaths[0]);
+ }
+ string text = "What is shown in this image?";
+ if (interactive) {
+ Console.WriteLine("Prompt:");
+ text = Console.ReadLine();
}
- static void Main(string[] args)
+ string prompt = "<|user|>\n";
+ if (images != null)
{
- Console.WriteLine("--------------------");
- Console.WriteLine("Hello, Phi-3-Vision!");
- Console.WriteLine("--------------------");
-
- if (args.Length != 1)
+ for (int i = 0; i < imagePaths.Length; i++)
{
- throw new Exception("Usage: .\\HelloPhi3V ");
+ prompt += "<|image_" + (i + 1) + "|>\n";
}
+ }
+ prompt += text + "<|end|>\n<|assistant|>\n";
+
+ Console.WriteLine("Processing image and prompt...");
+ var inputTensors = processor.ProcessImages(prompt, images);
- Run(args[0]);
+ Console.WriteLine("Generating response...");
+ using GeneratorParams generatorParams = new GeneratorParams(model);
+ generatorParams.SetSearchOption("max_length", 7680);
+ generatorParams.SetInputs(inputTensors);
+
+ using var generator = new Generator(model, generatorParams);
+ while (!generator.IsDone())
+ {
+ generator.ComputeLogits();
+ generator.GenerateNextToken();
+ Console.Write(tokenizerStream.Decode(generator.GetSequence(0)[^1]));
}
-}
\ No newline at end of file
+} while (interactive);
\ No newline at end of file
diff --git a/examples/python/phi3v.py b/examples/python/phi3v.py
index fd92dbd93f..18e7739688 100644
--- a/examples/python/phi3v.py
+++ b/examples/python/phi3v.py
@@ -5,9 +5,12 @@
import os
import readline
import glob
+from pathlib import Path
import onnxruntime_genai as og
+REPO_ROOT = Path(__file__).parents[2]
+
def _complete(text, state):
return (glob.glob(text + "*") + [None])[state]
@@ -15,20 +18,30 @@ def _complete(text, state):
def run(args: argparse.Namespace):
print("Loading model...")
model = og.Model(args.model_path)
+ print("Model loaded")
processor = model.create_multimodal_processor()
tokenizer_stream = processor.create_stream()
+ interactive = args.interactive
+
while True:
readline.set_completer_delims(" \t\n;")
readline.parse_and_bind("tab: complete")
readline.set_completer(_complete)
- image_paths = [
- image_path.strip()
- for image_path in input(
- "Image Path (comma separated; leave empty if no image): "
- ).split(",")
- ]
- image_paths = [image_path for image_path in image_paths if len(image_path)]
+ if interactive:
+ image_paths = [
+ image_path.strip()
+ for image_path in input(
+ "Image Path (comma separated; leave empty if no image): "
+ ).split(",")
+ ]
+ else:
+ if args.image_paths:
+ image_paths = args.image_paths
+ else:
+ image_paths = [str(REPO_ROOT / "test" / "test_models" / "images" / "australia.jpg")]
+
+ image_paths = [image_path for image_path in image_paths]
print(image_paths)
images = None
@@ -36,7 +49,7 @@ def run(args: argparse.Namespace):
if len(image_paths) == 0:
print("No image provided")
else:
- print("Loading images...")
+ print(f"Loading images: {image_paths}")
for i, image_path in enumerate(image_paths):
if not os.path.exists(image_path):
raise FileNotFoundError(f"Image file not found: {image_path}")
@@ -44,7 +57,13 @@ def run(args: argparse.Namespace):
images = og.Images.open(*image_paths)
- text = input("Prompt: ")
+ if interactive:
+ text = input("Prompt: ")
+ else:
+ if args.prompt:
+ text = args.prompt
+ else:
+ text = "What is shown in this image?"
prompt += f"{text}<|end|>\n<|assistant|>\n"
print("Processing images and prompt...")
inputs = processor(prompt, images=images)
@@ -69,11 +88,23 @@ def run(args: argparse.Namespace):
# Delete the generator to free the captured graph before creating another one
del generator
+ if not interactive:
+ break
+
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument(
"-m", "--model_path", type=str, required=True, help="Path to the model"
)
+ parser.add_argument(
+ "--image_paths", nargs='*', type=str, required=False, help="Path to the images"
+ )
+ parser.add_argument(
+ '-pr', '--prompt', required=False, help='Input prompts to generate tokens from.'
+ )
+ parser.add_argument(
+ '--interactive', default=False, required=False, help='Interactive mode'
+ )
args = parser.parse_args()
run(args)
From 737e61f40fe546b4428e2ebfabe0529998a79603 Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Fri, 27 Sep 2024 11:38:14 +0800
Subject: [PATCH 02/42] phi3.5
---
examples/csharp/HelloPhi3V/Program.cs | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/examples/csharp/HelloPhi3V/Program.cs b/examples/csharp/HelloPhi3V/Program.cs
index bcbad2b61b..782d30f81f 100644
--- a/examples/csharp/HelloPhi3V/Program.cs
+++ b/examples/csharp/HelloPhi3V/Program.cs
@@ -86,7 +86,7 @@ void PrintUsage()
throw new Exception("Image file not found: " + imagePath);
}
}
- images = Images.Load(imagePaths[0]);
+ images = Images.Load(imagePaths);
}
string text = "What is shown in this image?";
From 20a18a823193967e9966bdd48363d0ebe081e1c4 Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Fri, 27 Sep 2024 12:27:30 +0800
Subject: [PATCH 03/42] nuget valiation
---
.../stages/jobs/nuget-validation-job.yml | 95 ++++---------------
.../perform-nuget-validation-with-model.yml | 89 +++++++++++++++++
2 files changed, 107 insertions(+), 77 deletions(-)
create mode 100644 .pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
diff --git a/.pipelines/stages/jobs/nuget-validation-job.yml b/.pipelines/stages/jobs/nuget-validation-job.yml
index c5498e12e0..927deb9366 100644
--- a/.pipelines/stages/jobs/nuget-validation-job.yml
+++ b/.pipelines/stages/jobs/nuget-validation-job.yml
@@ -120,9 +120,17 @@ jobs:
inputs:
version: '8.x'
+ - template: steps/utils//flex-download-pipeline-artifact.yml
+ parameters:
+ StepName: 'Download NuGet Artifacts'
+ ArtifactName: $(artifactName)-nuget
+ TargetPath: '$(Build.BinariesDirectory)/nuget'
+ SpecificArtifact: ${{ parameters.specificArtifact }}
+ BuildId: ${{ parameters.BuildId }}
+
- template: steps/utils/download-huggingface-model.yml
parameters:
- StepName: 'Download Model from HuggingFace'
+ StepName: 'Download Phi3-mini Model from HuggingFace'
HuggingFaceRepo: 'microsoft/Phi-3-mini-4k-instruct-onnx'
RepoFolder: $(prebuild_phi3_mini_model_folder)
LocalFolder: 'models'
@@ -130,83 +138,16 @@ jobs:
HuggingFaceToken: $(HF_TOKEN)
os: ${{ parameters.os }}
- - template: steps/utils//flex-download-pipeline-artifact.yml
+ - template: steps/utils/perform-nuget-validation-with-model.yml
parameters:
- StepName: 'Download NuGet Artifacts'
- ArtifactName: $(artifactName)-nuget
- TargetPath: '$(Build.BinariesDirectory)/nuget'
- SpecificArtifact: ${{ parameters.specificArtifact }}
- BuildId: ${{ parameters.BuildId }}
-
- - ${{ if eq(parameters.os, 'win') }}:
- - ${{ if eq(parameters.ep, 'cuda') }}:
- - powershell: |
- $env:AZCOPY_MSI_CLIENT_ID = "63b63039-6328-442f-954b-5a64d124e5b4";
- azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v$(cuda_version)" 'cuda_sdk'
- displayName: 'Download CUDA $(cuda_version)'
- workingDirectory: '$(Build.Repository.LocalPath)'
- - powershell: |
- if ("$(ep)" -eq "cuda") {
- $env:CUDA_PATH = '$(Build.Repository.LocalPath)\cuda_sdk\v$(cuda_version)'
- $env:PATH = "$env:CUDA_PATH\bin;$env:CUDA_PATH\extras\CUPTI\lib64;$env:PATH"
- Write-Host $env:PATH
- }
- dotnet --info
- Copy-Item -Force -Recurse -Verbose $(Build.BinariesDirectory)/nuget/* -Destination examples/csharp/HelloPhi/
- cd examples/csharp/HelloPhi
- Move-Item models\$(prebuild_phi3_mini_model_folder) models\phi-3
- dotnet restore -r $(os)-$(arch) /property:Configuration=$(csproj_configuration) --source https://api.nuget.org/v3/index.json --source https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/nuget/v3/index.json --source $PWD --disable-parallel --verbosity detailed
- dotnet run -r $(os)-$(arch) --configuration $(csproj_configuration) --no-restore --verbosity normal -- -m ./models/phi-3
- displayName: 'Run Example With Artifact'
- workingDirectory: '$(Build.Repository.LocalPath)'
- env:
- NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS: 180
- NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS: 180
- - ${{ elseif or(eq(parameters.os, 'linux'), eq(parameters.os, 'osx')) }}:
- - bash: |
- dotnet --info
- cp $(Build.BinariesDirectory)/nuget/* examples/csharp/HelloPhi/
- cd examples/csharp/HelloPhi
- mv models/$(prebuild_phi3_mini_model_folder) models/phi-3
- dotnet restore -r $(os)-$(arch) /property:Configuration=$(csproj_configuration) --source https://api.nuget.org/v3/index.json --source https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/nuget/v3/index.json --source $PWD --disable-parallel --verbosity detailed
- dotnet build ./HelloPhi.csproj -r $(os)-$(arch) /property:Configuration=$(csproj_configuration) --no-restore --self-contained
- ls -l ./bin/$(csproj_configuration)/net6.0/$(os)-$(arch)/
- displayName: 'Perform dotnet restore & build'
- workingDirectory: '$(Build.Repository.LocalPath)'
- env:
- NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS: 180
- NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS: 180
-
- - ${{ if eq(parameters.ep, 'cuda') }}:
- - bash: |
- set -e -x
- az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4
- az acr login --name onnxruntimebuildcache --subscription 00c06639-6ee4-454e-8058-8d8b1703bd87
- docker pull $(cuda_docker_image)
-
- docker run \
- --gpus all \
- --rm \
- --volume $(Build.Repository.LocalPath):/ort_genai_src \
- --volume $(Build.BinariesDirectory):/ort_genai_binary \
- -e HF_TOKEN=$HF_TOKEN \
- -w /ort_genai_src/ $(cuda_docker_image) \
- bash -c " \
- export ORTGENAI_LOG_ORT_LIB=1 && \
- cd /ort_genai_src/examples/csharp/HelloPhi && \
- chmod +x ./bin/Release_Cuda/net6.0/linux-x64/HelloPhi && \
- ./bin/Release_Cuda/net6.0/linux-x64/HelloPhi -m ./models/phi-3"
-
- displayName: 'Run Example With Artifact'
- workingDirectory: '$(Build.Repository.LocalPath)'
-
- - ${{ elseif eq(parameters.ep, 'cpu') }}:
- - bash: |
- export ORTGENAI_LOG_ORT_LIB=1
- cd examples/csharp/HelloPhi
- dotnet run -r $(os)-$(arch) --configuration $(csproj_configuration) --no-build --verbosity normal -- -m ./models/phi-3
- displayName: 'Run Example With Artifact'
- workingDirectory: '$(Build.Repository.LocalPath)'
+ CsprojFolder: "examples/csharp/HelloPhi"
+ CsprojName: "HelloPhi"
+ CsprojConfiguration: $(csproj_config)
+ CudaVersion: $(cuda_version)
+ ModelFolder: $(prebuild_phi3_mini_model_folder)
+ os: ${{ parameters.os }}
+ ep: ${{ parameters.ep }}
+ arch: ${{ parameters.arch }}
- template: steps/compliant-and-cleanup-step.yml
diff --git a/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml b/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
new file mode 100644
index 0000000000..4af22d4732
--- /dev/null
+++ b/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
@@ -0,0 +1,89 @@
+- name: CsprojFolder
+ type: string
+- name: CsprojName
+ type: string
+- name: CsprojConfiguration
+ type: string
+- name: CudaVersion
+ type: string
+- name: CudaDockerImage
+ type: string
+- name: ModelFolder
+ type: string
+- name: ep
+ type: string
+- name: os
+ type: string
+- name: arch
+ type: string
+
+steps:
+- ${{ if eq(parameters.os, 'win') }}:
+ - ${{ if eq(parameters.ep, 'cuda') }}:
+ - powershell: |
+ $env:AZCOPY_MSI_CLIENT_ID = "63b63039-6328-442f-954b-5a64d124e5b4";
+ azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v$(CudaVersion)" 'cuda_sdk'
+ displayName: 'Download CUDA $(CudaVersion)'
+ workingDirectory: '$(Build.Repository.LocalPath)'
+ - powershell: |
+ if ("$(ep)" -eq "cuda") {
+ $env:CUDA_PATH = '$(Build.Repository.LocalPath)\cuda_sdk\v$(CudaVersion)'
+ $env:PATH = "$env:CUDA_PATH\bin;$env:CUDA_PATH\extras\CUPTI\lib64;$env:PATH"
+ Write-Host $env:PATH
+ }
+ dotnet --info
+ Copy-Item -Force -Recurse -Verbose $(Build.BinariesDirectory)/nuget/* -Destination $(CsprojFolder)
+ cd $(CsprojFolder)
+ Move-Item models\$(ModelFolder) models\targeted_model
+ dotnet restore -r $(os)-$(arch) /property:Configuration=$(CsprojConfiguration) --source https://api.nuget.org/v3/index.json --source https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/nuget/v3/index.json --source $PWD --disable-parallel --verbosity detailed
+ dotnet run -r $(os)-$(arch) --configuration $(CsprojConfiguration) --no-restore --verbosity normal -- -m ./models/targeted_model
+ displayName: 'Run $(CsprojName) With Artifact'
+ workingDirectory: '$(Build.Repository.LocalPath)'
+ env:
+ NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS: 180
+ NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS: 180
+- ${{ elseif or(eq(parameters.os, 'linux'), eq(parameters.os, 'osx')) }}:
+ - bash: |
+ dotnet --info
+ cp $(Build.BinariesDirectory)/nuget/* $(CsprojFolder)
+ cd $(CsprojFolder)
+ mv models/$(ModelFolder) models/targeted_model
+ dotnet restore -r $(os)-$(arch) /property:Configuration=$(CsprojConfiguration) --source https://api.nuget.org/v3/index.json --source https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/nuget/v3/index.json --source $PWD --disable-parallel --verbosity detailed
+ dotnet build ./$(CsprojName).csproj -r $(os)-$(arch) /property:Configuration=$(CsprojConfiguration) --no-restore --self-contained
+ ls -l ./bin/$(CsprojConfiguration)/net6.0/$(os)-$(arch)/
+ displayName: 'Perform dotnet restore & build'
+ workingDirectory: '$(Build.Repository.LocalPath)'
+ env:
+ NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS: 180
+ NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS: 180
+
+ - ${{ if eq(parameters.ep, 'cuda') }}:
+ - bash: |
+ set -e -x
+ az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4
+ az acr login --name onnxruntimebuildcache --subscription 00c06639-6ee4-454e-8058-8d8b1703bd87
+ docker pull $(CudaDockerImage)
+
+ docker run \
+ --gpus all \
+ --rm \
+ --volume $(Build.Repository.LocalPath):/ort_genai_src \
+ --volume $(Build.BinariesDirectory):/ort_genai_binary \
+ -e HF_TOKEN=$HF_TOKEN \
+ -w /ort_genai_src/ $(CudaDockerImage) \
+ bash -c " \
+ export ORTGENAI_LOG_ORT_LIB=1 && \
+ cd /ort_genai_src/$(CsprojFolder) && \
+ chmod +x ./bin/Release_Cuda/net6.0/linux-x64/$(CsprojName) && \
+ ./bin/Release_Cuda/net6.0/linux-x64/$(CsprojName) -m ./models/targeted_model"
+
+ displayName: 'Run $(CsprojName) With Artifact'
+ workingDirectory: '$(Build.Repository.LocalPath)'
+
+ - ${{ elseif eq(parameters.ep, 'cpu') }}:
+ - bash: |
+ export ORTGENAI_LOG_ORT_LIB=1
+ cd $(CsprojFolder)
+ dotnet run -r $(os)-$(arch) --configuration $(CsprojConfiguration) --no-build --verbosity normal -- -m ./models/targeted_model
+ displayName: 'Run $(CsprojName) With Artifact'
+ workingDirectory: '$(Build.Repository.LocalPath)'
From 2513826f0331fd93922b9dd638f2f617440ed773 Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Fri, 27 Sep 2024 12:29:32 +0800
Subject: [PATCH 04/42] nuget valiation
---
.../perform-nuget-validation-with-model.yml | 126 +++++++++---------
1 file changed, 63 insertions(+), 63 deletions(-)
diff --git a/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml b/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
index 4af22d4732..e607ac15f6 100644
--- a/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
+++ b/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
@@ -18,72 +18,72 @@
type: string
steps:
-- ${{ if eq(parameters.os, 'win') }}:
- - ${{ if eq(parameters.ep, 'cuda') }}:
+ - ${{ if eq(parameters.os, 'win') }}:
+ - ${{ if eq(parameters.ep, 'cuda') }}:
+ - powershell: |
+ $env:AZCOPY_MSI_CLIENT_ID = "63b63039-6328-442f-954b-5a64d124e5b4";
+ azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v$(CudaVersion)" 'cuda_sdk'
+ displayName: 'Download CUDA $(CudaVersion)'
+ workingDirectory: '$(Build.Repository.LocalPath)'
- powershell: |
- $env:AZCOPY_MSI_CLIENT_ID = "63b63039-6328-442f-954b-5a64d124e5b4";
- azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v$(CudaVersion)" 'cuda_sdk'
- displayName: 'Download CUDA $(CudaVersion)'
- workingDirectory: '$(Build.Repository.LocalPath)'
- - powershell: |
- if ("$(ep)" -eq "cuda") {
- $env:CUDA_PATH = '$(Build.Repository.LocalPath)\cuda_sdk\v$(CudaVersion)'
- $env:PATH = "$env:CUDA_PATH\bin;$env:CUDA_PATH\extras\CUPTI\lib64;$env:PATH"
- Write-Host $env:PATH
- }
- dotnet --info
- Copy-Item -Force -Recurse -Verbose $(Build.BinariesDirectory)/nuget/* -Destination $(CsprojFolder)
- cd $(CsprojFolder)
- Move-Item models\$(ModelFolder) models\targeted_model
- dotnet restore -r $(os)-$(arch) /property:Configuration=$(CsprojConfiguration) --source https://api.nuget.org/v3/index.json --source https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/nuget/v3/index.json --source $PWD --disable-parallel --verbosity detailed
- dotnet run -r $(os)-$(arch) --configuration $(CsprojConfiguration) --no-restore --verbosity normal -- -m ./models/targeted_model
- displayName: 'Run $(CsprojName) With Artifact'
- workingDirectory: '$(Build.Repository.LocalPath)'
- env:
- NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS: 180
- NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS: 180
-- ${{ elseif or(eq(parameters.os, 'linux'), eq(parameters.os, 'osx')) }}:
- - bash: |
- dotnet --info
- cp $(Build.BinariesDirectory)/nuget/* $(CsprojFolder)
- cd $(CsprojFolder)
- mv models/$(ModelFolder) models/targeted_model
- dotnet restore -r $(os)-$(arch) /property:Configuration=$(CsprojConfiguration) --source https://api.nuget.org/v3/index.json --source https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/nuget/v3/index.json --source $PWD --disable-parallel --verbosity detailed
- dotnet build ./$(CsprojName).csproj -r $(os)-$(arch) /property:Configuration=$(CsprojConfiguration) --no-restore --self-contained
- ls -l ./bin/$(CsprojConfiguration)/net6.0/$(os)-$(arch)/
- displayName: 'Perform dotnet restore & build'
- workingDirectory: '$(Build.Repository.LocalPath)'
- env:
- NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS: 180
- NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS: 180
-
- - ${{ if eq(parameters.ep, 'cuda') }}:
- - bash: |
- set -e -x
- az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4
- az acr login --name onnxruntimebuildcache --subscription 00c06639-6ee4-454e-8058-8d8b1703bd87
- docker pull $(CudaDockerImage)
-
- docker run \
- --gpus all \
- --rm \
- --volume $(Build.Repository.LocalPath):/ort_genai_src \
- --volume $(Build.BinariesDirectory):/ort_genai_binary \
- -e HF_TOKEN=$HF_TOKEN \
- -w /ort_genai_src/ $(CudaDockerImage) \
- bash -c " \
- export ORTGENAI_LOG_ORT_LIB=1 && \
- cd /ort_genai_src/$(CsprojFolder) && \
- chmod +x ./bin/Release_Cuda/net6.0/linux-x64/$(CsprojName) && \
- ./bin/Release_Cuda/net6.0/linux-x64/$(CsprojName) -m ./models/targeted_model"
-
+ if ("$(ep)" -eq "cuda") {
+ $env:CUDA_PATH = '$(Build.Repository.LocalPath)\cuda_sdk\v$(CudaVersion)'
+ $env:PATH = "$env:CUDA_PATH\bin;$env:CUDA_PATH\extras\CUPTI\lib64;$env:PATH"
+ Write-Host $env:PATH
+ }
+ dotnet --info
+ Copy-Item -Force -Recurse -Verbose $(Build.BinariesDirectory)/nuget/* -Destination $(CsprojFolder)
+ cd $(CsprojFolder)
+ Move-Item models\$(ModelFolder) models\targeted_model
+ dotnet restore -r $(os)-$(arch) /property:Configuration=$(CsprojConfiguration) --source https://api.nuget.org/v3/index.json --source https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/nuget/v3/index.json --source $PWD --disable-parallel --verbosity detailed
+ dotnet run -r $(os)-$(arch) --configuration $(CsprojConfiguration) --no-restore --verbosity normal -- -m ./models/targeted_model
displayName: 'Run $(CsprojName) With Artifact'
workingDirectory: '$(Build.Repository.LocalPath)'
-
- - ${{ elseif eq(parameters.ep, 'cpu') }}:
+ env:
+ NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS: 180
+ NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS: 180
+ - ${{ elseif or(eq(parameters.os, 'linux'), eq(parameters.os, 'osx')) }}:
- bash: |
- export ORTGENAI_LOG_ORT_LIB=1
+ dotnet --info
+ cp $(Build.BinariesDirectory)/nuget/* $(CsprojFolder)
cd $(CsprojFolder)
- dotnet run -r $(os)-$(arch) --configuration $(CsprojConfiguration) --no-build --verbosity normal -- -m ./models/targeted_model
- displayName: 'Run $(CsprojName) With Artifact'
+ mv models/$(ModelFolder) models/targeted_model
+ dotnet restore -r $(os)-$(arch) /property:Configuration=$(CsprojConfiguration) --source https://api.nuget.org/v3/index.json --source https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/nuget/v3/index.json --source $PWD --disable-parallel --verbosity detailed
+ dotnet build ./$(CsprojName).csproj -r $(os)-$(arch) /property:Configuration=$(CsprojConfiguration) --no-restore --self-contained
+ ls -l ./bin/$(CsprojConfiguration)/net6.0/$(os)-$(arch)/
+ displayName: 'Perform dotnet restore & build'
workingDirectory: '$(Build.Repository.LocalPath)'
+ env:
+ NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS: 180
+ NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS: 180
+
+ - ${{ if eq(parameters.ep, 'cuda') }}:
+ - bash: |
+ set -e -x
+ az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4
+ az acr login --name onnxruntimebuildcache --subscription 00c06639-6ee4-454e-8058-8d8b1703bd87
+ docker pull $(CudaDockerImage)
+
+ docker run \
+ --gpus all \
+ --rm \
+ --volume $(Build.Repository.LocalPath):/ort_genai_src \
+ --volume $(Build.BinariesDirectory):/ort_genai_binary \
+ -e HF_TOKEN=$HF_TOKEN \
+ -w /ort_genai_src/ $(CudaDockerImage) \
+ bash -c " \
+ export ORTGENAI_LOG_ORT_LIB=1 && \
+ cd /ort_genai_src/$(CsprojFolder) && \
+ chmod +x ./bin/Release_Cuda/net6.0/linux-x64/$(CsprojName) && \
+ ./bin/Release_Cuda/net6.0/linux-x64/$(CsprojName) -m ./models/targeted_model"
+
+ displayName: 'Run $(CsprojName) With Artifact'
+ workingDirectory: '$(Build.Repository.LocalPath)'
+
+ - ${{ elseif eq(parameters.ep, 'cpu') }}:
+ - bash: |
+ export ORTGENAI_LOG_ORT_LIB=1
+ cd $(CsprojFolder)
+ dotnet run -r $(os)-$(arch) --configuration $(CsprojConfiguration) --no-build --verbosity normal -- -m ./models/targeted_model
+ displayName: 'Run $(CsprojName) With Artifact'
+ workingDirectory: '$(Build.Repository.LocalPath)'
From 31cd57b8acf928aead1c7e1594eca0dba1417a17 Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Fri, 27 Sep 2024 12:30:45 +0800
Subject: [PATCH 05/42] LF
---
.../perform-nuget-validation-with-model.yml | 178 +++++++++---------
1 file changed, 89 insertions(+), 89 deletions(-)
diff --git a/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml b/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
index e607ac15f6..38583e74b0 100644
--- a/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
+++ b/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
@@ -1,89 +1,89 @@
-- name: CsprojFolder
- type: string
-- name: CsprojName
- type: string
-- name: CsprojConfiguration
- type: string
-- name: CudaVersion
- type: string
-- name: CudaDockerImage
- type: string
-- name: ModelFolder
- type: string
-- name: ep
- type: string
-- name: os
- type: string
-- name: arch
- type: string
-
-steps:
- - ${{ if eq(parameters.os, 'win') }}:
- - ${{ if eq(parameters.ep, 'cuda') }}:
- - powershell: |
- $env:AZCOPY_MSI_CLIENT_ID = "63b63039-6328-442f-954b-5a64d124e5b4";
- azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v$(CudaVersion)" 'cuda_sdk'
- displayName: 'Download CUDA $(CudaVersion)'
- workingDirectory: '$(Build.Repository.LocalPath)'
- - powershell: |
- if ("$(ep)" -eq "cuda") {
- $env:CUDA_PATH = '$(Build.Repository.LocalPath)\cuda_sdk\v$(CudaVersion)'
- $env:PATH = "$env:CUDA_PATH\bin;$env:CUDA_PATH\extras\CUPTI\lib64;$env:PATH"
- Write-Host $env:PATH
- }
- dotnet --info
- Copy-Item -Force -Recurse -Verbose $(Build.BinariesDirectory)/nuget/* -Destination $(CsprojFolder)
- cd $(CsprojFolder)
- Move-Item models\$(ModelFolder) models\targeted_model
- dotnet restore -r $(os)-$(arch) /property:Configuration=$(CsprojConfiguration) --source https://api.nuget.org/v3/index.json --source https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/nuget/v3/index.json --source $PWD --disable-parallel --verbosity detailed
- dotnet run -r $(os)-$(arch) --configuration $(CsprojConfiguration) --no-restore --verbosity normal -- -m ./models/targeted_model
- displayName: 'Run $(CsprojName) With Artifact'
- workingDirectory: '$(Build.Repository.LocalPath)'
- env:
- NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS: 180
- NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS: 180
- - ${{ elseif or(eq(parameters.os, 'linux'), eq(parameters.os, 'osx')) }}:
- - bash: |
- dotnet --info
- cp $(Build.BinariesDirectory)/nuget/* $(CsprojFolder)
- cd $(CsprojFolder)
- mv models/$(ModelFolder) models/targeted_model
- dotnet restore -r $(os)-$(arch) /property:Configuration=$(CsprojConfiguration) --source https://api.nuget.org/v3/index.json --source https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/nuget/v3/index.json --source $PWD --disable-parallel --verbosity detailed
- dotnet build ./$(CsprojName).csproj -r $(os)-$(arch) /property:Configuration=$(CsprojConfiguration) --no-restore --self-contained
- ls -l ./bin/$(CsprojConfiguration)/net6.0/$(os)-$(arch)/
- displayName: 'Perform dotnet restore & build'
- workingDirectory: '$(Build.Repository.LocalPath)'
- env:
- NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS: 180
- NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS: 180
-
- - ${{ if eq(parameters.ep, 'cuda') }}:
- - bash: |
- set -e -x
- az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4
- az acr login --name onnxruntimebuildcache --subscription 00c06639-6ee4-454e-8058-8d8b1703bd87
- docker pull $(CudaDockerImage)
-
- docker run \
- --gpus all \
- --rm \
- --volume $(Build.Repository.LocalPath):/ort_genai_src \
- --volume $(Build.BinariesDirectory):/ort_genai_binary \
- -e HF_TOKEN=$HF_TOKEN \
- -w /ort_genai_src/ $(CudaDockerImage) \
- bash -c " \
- export ORTGENAI_LOG_ORT_LIB=1 && \
- cd /ort_genai_src/$(CsprojFolder) && \
- chmod +x ./bin/Release_Cuda/net6.0/linux-x64/$(CsprojName) && \
- ./bin/Release_Cuda/net6.0/linux-x64/$(CsprojName) -m ./models/targeted_model"
-
- displayName: 'Run $(CsprojName) With Artifact'
- workingDirectory: '$(Build.Repository.LocalPath)'
-
- - ${{ elseif eq(parameters.ep, 'cpu') }}:
- - bash: |
- export ORTGENAI_LOG_ORT_LIB=1
- cd $(CsprojFolder)
- dotnet run -r $(os)-$(arch) --configuration $(CsprojConfiguration) --no-build --verbosity normal -- -m ./models/targeted_model
- displayName: 'Run $(CsprojName) With Artifact'
- workingDirectory: '$(Build.Repository.LocalPath)'
+- name: CsprojFolder
+ type: string
+- name: CsprojName
+ type: string
+- name: CsprojConfiguration
+ type: string
+- name: CudaVersion
+ type: string
+- name: CudaDockerImage
+ type: string
+- name: ModelFolder
+ type: string
+- name: ep
+ type: string
+- name: os
+ type: string
+- name: arch
+ type: string
+
+steps:
+ - ${{ if eq(parameters.os, 'win') }}:
+ - ${{ if eq(parameters.ep, 'cuda') }}:
+ - powershell: |
+ $env:AZCOPY_MSI_CLIENT_ID = "63b63039-6328-442f-954b-5a64d124e5b4";
+ azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v$(CudaVersion)" 'cuda_sdk'
+ displayName: 'Download CUDA $(CudaVersion)'
+ workingDirectory: '$(Build.Repository.LocalPath)'
+ - powershell: |
+ if ("$(ep)" -eq "cuda") {
+ $env:CUDA_PATH = '$(Build.Repository.LocalPath)\cuda_sdk\v$(CudaVersion)'
+ $env:PATH = "$env:CUDA_PATH\bin;$env:CUDA_PATH\extras\CUPTI\lib64;$env:PATH"
+ Write-Host $env:PATH
+ }
+ dotnet --info
+ Copy-Item -Force -Recurse -Verbose $(Build.BinariesDirectory)/nuget/* -Destination $(CsprojFolder)
+ cd $(CsprojFolder)
+ Move-Item models\$(ModelFolder) models\targeted_model
+ dotnet restore -r $(os)-$(arch) /property:Configuration=$(CsprojConfiguration) --source https://api.nuget.org/v3/index.json --source https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/nuget/v3/index.json --source $PWD --disable-parallel --verbosity detailed
+ dotnet run -r $(os)-$(arch) --configuration $(CsprojConfiguration) --no-restore --verbosity normal -- -m ./models/targeted_model
+ displayName: 'Run $(CsprojName) With Artifact'
+ workingDirectory: '$(Build.Repository.LocalPath)'
+ env:
+ NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS: 180
+ NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS: 180
+ - ${{ elseif or(eq(parameters.os, 'linux'), eq(parameters.os, 'osx')) }}:
+ - bash: |
+ dotnet --info
+ cp $(Build.BinariesDirectory)/nuget/* $(CsprojFolder)
+ cd $(CsprojFolder)
+ mv models/$(ModelFolder) models/targeted_model
+ dotnet restore -r $(os)-$(arch) /property:Configuration=$(CsprojConfiguration) --source https://api.nuget.org/v3/index.json --source https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/nuget/v3/index.json --source $PWD --disable-parallel --verbosity detailed
+ dotnet build ./$(CsprojName).csproj -r $(os)-$(arch) /property:Configuration=$(CsprojConfiguration) --no-restore --self-contained
+ ls -l ./bin/$(CsprojConfiguration)/net6.0/$(os)-$(arch)/
+ displayName: 'Perform dotnet restore & build'
+ workingDirectory: '$(Build.Repository.LocalPath)'
+ env:
+ NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS: 180
+ NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS: 180
+
+ - ${{ if eq(parameters.ep, 'cuda') }}:
+ - bash: |
+ set -e -x
+ az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4
+ az acr login --name onnxruntimebuildcache --subscription 00c06639-6ee4-454e-8058-8d8b1703bd87
+ docker pull $(CudaDockerImage)
+
+ docker run \
+ --gpus all \
+ --rm \
+ --volume $(Build.Repository.LocalPath):/ort_genai_src \
+ --volume $(Build.BinariesDirectory):/ort_genai_binary \
+ -e HF_TOKEN=$HF_TOKEN \
+ -w /ort_genai_src/ $(CudaDockerImage) \
+ bash -c " \
+ export ORTGENAI_LOG_ORT_LIB=1 && \
+ cd /ort_genai_src/$(CsprojFolder) && \
+ chmod +x ./bin/Release_Cuda/net6.0/linux-x64/$(CsprojName) && \
+ ./bin/Release_Cuda/net6.0/linux-x64/$(CsprojName) -m ./models/targeted_model"
+
+ displayName: 'Run $(CsprojName) With Artifact'
+ workingDirectory: '$(Build.Repository.LocalPath)'
+
+ - ${{ elseif eq(parameters.ep, 'cpu') }}:
+ - bash: |
+ export ORTGENAI_LOG_ORT_LIB=1
+ cd $(CsprojFolder)
+ dotnet run -r $(os)-$(arch) --configuration $(CsprojConfiguration) --no-build --verbosity normal -- -m ./models/targeted_model
+ displayName: 'Run $(CsprojName) With Artifact'
+ workingDirectory: '$(Build.Repository.LocalPath)'
From b65c13cee25671d08432c1f44d7e87d3dfa52e2f Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Fri, 27 Sep 2024 12:31:35 +0800
Subject: [PATCH 06/42] nuget valiation
---
.../perform-nuget-validation-with-model.yml | 127 +++++++++---------
1 file changed, 64 insertions(+), 63 deletions(-)
diff --git a/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml b/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
index 38583e74b0..8a3a2b9e12 100644
--- a/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
+++ b/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
@@ -1,3 +1,4 @@
+parameters:
- name: CsprojFolder
type: string
- name: CsprojName
@@ -18,72 +19,72 @@
type: string
steps:
- - ${{ if eq(parameters.os, 'win') }}:
- - ${{ if eq(parameters.ep, 'cuda') }}:
- - powershell: |
- $env:AZCOPY_MSI_CLIENT_ID = "63b63039-6328-442f-954b-5a64d124e5b4";
- azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v$(CudaVersion)" 'cuda_sdk'
- displayName: 'Download CUDA $(CudaVersion)'
- workingDirectory: '$(Build.Repository.LocalPath)'
+- ${{ if eq(parameters.os, 'win') }}:
+ - ${{ if eq(parameters.ep, 'cuda') }}:
- powershell: |
- if ("$(ep)" -eq "cuda") {
- $env:CUDA_PATH = '$(Build.Repository.LocalPath)\cuda_sdk\v$(CudaVersion)'
- $env:PATH = "$env:CUDA_PATH\bin;$env:CUDA_PATH\extras\CUPTI\lib64;$env:PATH"
- Write-Host $env:PATH
- }
- dotnet --info
- Copy-Item -Force -Recurse -Verbose $(Build.BinariesDirectory)/nuget/* -Destination $(CsprojFolder)
- cd $(CsprojFolder)
- Move-Item models\$(ModelFolder) models\targeted_model
- dotnet restore -r $(os)-$(arch) /property:Configuration=$(CsprojConfiguration) --source https://api.nuget.org/v3/index.json --source https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/nuget/v3/index.json --source $PWD --disable-parallel --verbosity detailed
- dotnet run -r $(os)-$(arch) --configuration $(CsprojConfiguration) --no-restore --verbosity normal -- -m ./models/targeted_model
- displayName: 'Run $(CsprojName) With Artifact'
- workingDirectory: '$(Build.Repository.LocalPath)'
- env:
- NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS: 180
- NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS: 180
- - ${{ elseif or(eq(parameters.os, 'linux'), eq(parameters.os, 'osx')) }}:
- - bash: |
- dotnet --info
- cp $(Build.BinariesDirectory)/nuget/* $(CsprojFolder)
- cd $(CsprojFolder)
- mv models/$(ModelFolder) models/targeted_model
- dotnet restore -r $(os)-$(arch) /property:Configuration=$(CsprojConfiguration) --source https://api.nuget.org/v3/index.json --source https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/nuget/v3/index.json --source $PWD --disable-parallel --verbosity detailed
- dotnet build ./$(CsprojName).csproj -r $(os)-$(arch) /property:Configuration=$(CsprojConfiguration) --no-restore --self-contained
- ls -l ./bin/$(CsprojConfiguration)/net6.0/$(os)-$(arch)/
- displayName: 'Perform dotnet restore & build'
+ $env:AZCOPY_MSI_CLIENT_ID = "63b63039-6328-442f-954b-5a64d124e5b4";
+ azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v$(CudaVersion)" 'cuda_sdk'
+ displayName: 'Download CUDA $(CudaVersion)'
workingDirectory: '$(Build.Repository.LocalPath)'
- env:
- NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS: 180
- NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS: 180
+ - powershell: |
+ if ("$(ep)" -eq "cuda") {
+ $env:CUDA_PATH = '$(Build.Repository.LocalPath)\cuda_sdk\v$(CudaVersion)'
+ $env:PATH = "$env:CUDA_PATH\bin;$env:CUDA_PATH\extras\CUPTI\lib64;$env:PATH"
+ Write-Host $env:PATH
+ }
+ dotnet --info
+ Copy-Item -Force -Recurse -Verbose $(Build.BinariesDirectory)/nuget/* -Destination $(CsprojFolder)
+ cd $(CsprojFolder)
+ Move-Item models\$(ModelFolder) models\targeted_model
+ dotnet restore -r $(os)-$(arch) /property:Configuration=$(CsprojConfiguration) --source https://api.nuget.org/v3/index.json --source https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/nuget/v3/index.json --source $PWD --disable-parallel --verbosity detailed
+ dotnet run -r $(os)-$(arch) --configuration $(CsprojConfiguration) --no-restore --verbosity normal -- -m ./models/targeted_model
+ displayName: 'Run $(CsprojName) With Artifact'
+ workingDirectory: '$(Build.Repository.LocalPath)'
+ env:
+ NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS: 180
+ NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS: 180
+- ${{ elseif or(eq(parameters.os, 'linux'), eq(parameters.os, 'osx')) }}:
+ - bash: |
+ dotnet --info
+ cp $(Build.BinariesDirectory)/nuget/* $(CsprojFolder)
+ cd $(CsprojFolder)
+ mv models/$(ModelFolder) models/targeted_model
+ dotnet restore -r $(os)-$(arch) /property:Configuration=$(CsprojConfiguration) --source https://api.nuget.org/v3/index.json --source https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/nuget/v3/index.json --source $PWD --disable-parallel --verbosity detailed
+ dotnet build ./$(CsprojName).csproj -r $(os)-$(arch) /property:Configuration=$(CsprojConfiguration) --no-restore --self-contained
+ ls -l ./bin/$(CsprojConfiguration)/net6.0/$(os)-$(arch)/
+ displayName: 'Perform dotnet restore & build'
+ workingDirectory: '$(Build.Repository.LocalPath)'
+ env:
+ NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS: 180
+ NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS: 180
- - ${{ if eq(parameters.ep, 'cuda') }}:
- - bash: |
- set -e -x
- az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4
- az acr login --name onnxruntimebuildcache --subscription 00c06639-6ee4-454e-8058-8d8b1703bd87
- docker pull $(CudaDockerImage)
+ - ${{ if eq(parameters.ep, 'cuda') }}:
+ - bash: |
+ set -e -x
+ az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4
+ az acr login --name onnxruntimebuildcache --subscription 00c06639-6ee4-454e-8058-8d8b1703bd87
+ docker pull $(CudaDockerImage)
- docker run \
- --gpus all \
- --rm \
- --volume $(Build.Repository.LocalPath):/ort_genai_src \
- --volume $(Build.BinariesDirectory):/ort_genai_binary \
- -e HF_TOKEN=$HF_TOKEN \
- -w /ort_genai_src/ $(CudaDockerImage) \
- bash -c " \
- export ORTGENAI_LOG_ORT_LIB=1 && \
- cd /ort_genai_src/$(CsprojFolder) && \
- chmod +x ./bin/Release_Cuda/net6.0/linux-x64/$(CsprojName) && \
- ./bin/Release_Cuda/net6.0/linux-x64/$(CsprojName) -m ./models/targeted_model"
+ docker run \
+ --gpus all \
+ --rm \
+ --volume $(Build.Repository.LocalPath):/ort_genai_src \
+ --volume $(Build.BinariesDirectory):/ort_genai_binary \
+ -e HF_TOKEN=$HF_TOKEN \
+ -w /ort_genai_src/ $(CudaDockerImage) \
+ bash -c " \
+ export ORTGENAI_LOG_ORT_LIB=1 && \
+ cd /ort_genai_src/$(CsprojFolder) && \
+ chmod +x ./bin/Release_Cuda/net6.0/linux-x64/$(CsprojName) && \
+ ./bin/Release_Cuda/net6.0/linux-x64/$(CsprojName) -m ./models/targeted_model"
- displayName: 'Run $(CsprojName) With Artifact'
- workingDirectory: '$(Build.Repository.LocalPath)'
+ displayName: 'Run $(CsprojName) With Artifact'
+ workingDirectory: '$(Build.Repository.LocalPath)'
- - ${{ elseif eq(parameters.ep, 'cpu') }}:
- - bash: |
- export ORTGENAI_LOG_ORT_LIB=1
- cd $(CsprojFolder)
- dotnet run -r $(os)-$(arch) --configuration $(CsprojConfiguration) --no-build --verbosity normal -- -m ./models/targeted_model
- displayName: 'Run $(CsprojName) With Artifact'
- workingDirectory: '$(Build.Repository.LocalPath)'
+ - ${{ elseif eq(parameters.ep, 'cpu') }}:
+ - bash: |
+ export ORTGENAI_LOG_ORT_LIB=1
+ cd $(CsprojFolder)
+ dotnet run -r $(os)-$(arch) --configuration $(CsprojConfiguration) --no-build --verbosity normal -- -m ./models/targeted_model
+ displayName: 'Run $(CsprojName) With Artifact'
+ workingDirectory: '$(Build.Repository.LocalPath)'
From 1816cb6ad95ff7b981a03ba0234eb66b2f2f635e Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Fri, 27 Sep 2024 12:32:28 +0800
Subject: [PATCH 07/42] cuda docker image
---
.pipelines/stages/jobs/nuget-validation-job.yml | 1 +
1 file changed, 1 insertion(+)
diff --git a/.pipelines/stages/jobs/nuget-validation-job.yml b/.pipelines/stages/jobs/nuget-validation-job.yml
index 927deb9366..48968d5fa2 100644
--- a/.pipelines/stages/jobs/nuget-validation-job.yml
+++ b/.pipelines/stages/jobs/nuget-validation-job.yml
@@ -144,6 +144,7 @@ jobs:
CsprojName: "HelloPhi"
CsprojConfiguration: $(csproj_config)
CudaVersion: $(cuda_version)
+ CudaDockerImage: $(cuda_docker_image)
ModelFolder: $(prebuild_phi3_mini_model_folder)
os: ${{ parameters.os }}
ep: ${{ parameters.ep }}
From d86cfcc9c5fed87d2a9b6d5a73335d9fa16d70fc Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Fri, 27 Sep 2024 13:19:03 +0800
Subject: [PATCH 08/42] update
---
.../stages/jobs/nuget-validation-job.yml | 3 -
.pipelines/stages/jobs/py-validation-job.yml | 76 ++----------------
.../stages/jobs/steps/capi-win-step.yml | 18 -----
.../perform-nuget-validation-with-model.yml | 50 +++++-------
.../perform-python-validation-with-model.yml | 78 +++++++++++++++++++
5 files changed, 103 insertions(+), 122 deletions(-)
create mode 100644 .pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
diff --git a/.pipelines/stages/jobs/nuget-validation-job.yml b/.pipelines/stages/jobs/nuget-validation-job.yml
index 48968d5fa2..63af39e89c 100644
--- a/.pipelines/stages/jobs/nuget-validation-job.yml
+++ b/.pipelines/stages/jobs/nuget-validation-job.yml
@@ -146,9 +146,6 @@ jobs:
CudaVersion: $(cuda_version)
CudaDockerImage: $(cuda_docker_image)
ModelFolder: $(prebuild_phi3_mini_model_folder)
- os: ${{ parameters.os }}
- ep: ${{ parameters.ep }}
- arch: ${{ parameters.arch }}
- template: steps/compliant-and-cleanup-step.yml
diff --git a/.pipelines/stages/jobs/py-validation-job.yml b/.pipelines/stages/jobs/py-validation-job.yml
index 2282f8f775..8cc79af7d5 100644
--- a/.pipelines/stages/jobs/py-validation-job.yml
+++ b/.pipelines/stages/jobs/py-validation-job.yml
@@ -161,7 +161,7 @@ jobs:
- template: steps/utils/download-huggingface-model.yml
parameters:
- StepName: 'Download Model from HuggingFace'
+ StepName: 'Download Phi3-mini from HuggingFace'
HuggingFaceRepo: 'microsoft/Phi-3-mini-4k-instruct-onnx'
RepoFolder: $(prebuild_phi3_mini_model_folder)
LocalFolder: 'models'
@@ -169,74 +169,10 @@ jobs:
HuggingFaceToken: $(HF_TOKEN)
os: ${{ parameters.os }}
- - ${{ if or(eq(parameters.os, 'linux'), eq(parameters.os, 'osx')) }}:
- - ${{ if eq(parameters.ep, 'cuda') }}:
- - bash: |
- set -e -x
- az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4
- az acr login --name onnxruntimebuildcache --subscription 00c06639-6ee4-454e-8058-8d8b1703bd87
- docker pull $(cuda_docker_image)
- python_exe=/opt/python/cp310-cp310/bin/python3.10
-
- docker run \
- --gpus all \
- --rm \
- --volume $(Build.Repository.LocalPath):/ort_genai_src \
- --volume $(Build.BinariesDirectory):/ort_genai_binary \
- -e HF_TOKEN=$HF_TOKEN \
- -w /ort_genai_src/ $(cuda_docker_image) \
- bash -c " \
- export ORTGENAI_LOG_ORT_LIB=1 && \
- $python_exe -m pip install -r /ort_genai_src/test/python/requirements.txt && \
- $python_exe -m pip install -r /ort_genai_src/test/python/requirements-cuda.txt && \
- cd /ort_genai_src/examples/python && \
- $python_exe -m pip install --no-index --find-links=/ort_genai_binary/wheel $(pip_package_name) && \
- $python_exe model-generate.py -m ./models/$(prebuild_phi3_mini_model_folder) --min_length 25 --max_length 50 --verbose"
-
- displayName: 'Run Example With Artifact'
- workingDirectory: '$(Build.Repository.LocalPath)'
- - ${{ elseif eq(parameters.ep, 'cpu') }}:
- - bash: |
- export ORTGENAI_LOG_ORT_LIB=1
- python -m pip install -r test/python/requirements.txt
- if [[ "$(os)" == "linux" ]]; then
- python -m pip install -r test/python/requirements-cpu.txt
- fi
- if [[ "$(os)" == "osx" ]]; then
- python -m pip install -r test/python/requirements-macos.txt
- fi
- cd examples/python
- python -m pip install --no-index --find-links=$(Build.BinariesDirectory)/wheel $(pip_package_name)
- python model-generate.py -m ./models/$(prebuild_phi3_mini_model_folder) --min_length 25 --max_length 50 --verbose
- displayName: 'Run Example With Artifact'
- workingDirectory: '$(Build.Repository.LocalPath)'
-
- - ${{ if eq(parameters.os, 'win') }}:
- - ${{ if eq(parameters.ep, 'cuda') }}:
- - powershell: |
- $env:AZCOPY_MSI_CLIENT_ID = "63b63039-6328-442f-954b-5a64d124e5b4";
- azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v$(cuda_version)" 'cuda_sdk'
- displayName: 'Download CUDA $(cuda_version)'
- workingDirectory: '$(Build.Repository.LocalPath)'
- - powershell: |
- python -m pip install -r test/python/requirements.txt
- if ("$(ep)" -eq "cuda") {
- $env:CUDA_PATH = '$(Build.Repository.LocalPath)\cuda_sdk\v$(cuda_version)'
- $env:PATH = "$env:CUDA_PATH\bin;$env:CUDA_PATH\extras\CUPTI\lib64;$env:PATH"
- Write-Host $env:PATH
- python -m pip install -r test/python/requirements-cuda.txt
- }
- elseif ("$(ep)" -eq "directml") {
- python -m pip install -r test/python/requirements-directml.txt
- }
- else {
- python -m pip install -r test/python/requirements-cpu.txt
- }
- cd examples\python
- python -m pip install --no-index --find-links=$(Build.BinariesDirectory)/wheel $(pip_package_name)
-
- python model-generate.py -m .\models\$(prebuild_phi3_mini_model_folder) --min_length 25 --max_length 50 --verbose
- displayName: 'Run Example With Artifact'
- workingDirectory: '$(Build.Repository.LocalPath)'
+ - template: steps/utils/perform-python-validation-with-model.yml
+ parameters:
+ PythonScriptFolder: "examples/python"
+ PythonScriptName: "model-generate.py"
+ ModelFolder: $(prebuild_phi3_mini_model_folder)
- template: steps/compliant-and-cleanup-step.yml
diff --git a/.pipelines/stages/jobs/steps/capi-win-step.yml b/.pipelines/stages/jobs/steps/capi-win-step.yml
index 863b4ff4a0..07a494bfde 100644
--- a/.pipelines/stages/jobs/steps/capi-win-step.yml
+++ b/.pipelines/stages/jobs/steps/capi-win-step.yml
@@ -41,24 +41,6 @@ steps:
echo "build_config=${{ parameters.build_config }}"
displayName: 'Print Parameters'
-- ${{ if eq(parameters.ep, 'directml') }}:
- - powershell: |
- Invoke-WebRequest -Uri $(dml_url) -OutFile $(dml_zip)
- Expand-Archive $(dml_zip) -DestinationPath $(dml_dir)
- Remove-Item -Path $(dml_zip)
- Get-ChildItem -Recurse $(dml_dir)
- mv $(dml_dir)\bin\$(arch)-win\DirectML.dll ort\lib
- mv $(dml_dir)\include\DirectML.h ort\include
-
- Invoke-WebRequest -Uri $(d3d12_url) -OutFile $(d3d12_zip)
- Expand-Archive $(d3d12_zip) -DestinationPath $(d3d12_dir)
- Remove-Item -Path $(d3d12_zip)
- Get-ChildItem -Recurse $(d3d12_dir)
- mv $(d3d12_dir)\build\native\bin\$(arch)\D3D12Core.dll ort\lib
- workingDirectory: '$(Build.Repository.LocalPath)'
- displayName: 'Download DirectML & Direct3D DLLs'
- continueOnError: true
-
- powershell: |
azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v$(cuda_version)" 'cuda_sdk'
displayName: 'Download CUDA $(cuda_version)'
diff --git a/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml b/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
index 8a3a2b9e12..0a39c35578 100644
--- a/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
+++ b/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
@@ -5,40 +5,29 @@ parameters:
type: string
- name: CsprojConfiguration
type: string
-- name: CudaVersion
- type: string
-- name: CudaDockerImage
- type: string
- name: ModelFolder
type: string
-- name: ep
- type: string
-- name: os
- type: string
-- name: arch
- type: string
steps:
- ${{ if eq(parameters.os, 'win') }}:
- ${{ if eq(parameters.ep, 'cuda') }}:
- powershell: |
$env:AZCOPY_MSI_CLIENT_ID = "63b63039-6328-442f-954b-5a64d124e5b4";
- azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v$(CudaVersion)" 'cuda_sdk'
- displayName: 'Download CUDA $(CudaVersion)'
+ azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v$(cuda_version)" 'cuda_sdk'
+ displayName: 'Download CUDA $(cuda_version)'
workingDirectory: '$(Build.Repository.LocalPath)'
- powershell: |
if ("$(ep)" -eq "cuda") {
- $env:CUDA_PATH = '$(Build.Repository.LocalPath)\cuda_sdk\v$(CudaVersion)'
+ $env:CUDA_PATH = '$(Build.Repository.LocalPath)\cuda_sdk\v$(cuda_version)'
$env:PATH = "$env:CUDA_PATH\bin;$env:CUDA_PATH\extras\CUPTI\lib64;$env:PATH"
Write-Host $env:PATH
}
dotnet --info
- Copy-Item -Force -Recurse -Verbose $(Build.BinariesDirectory)/nuget/* -Destination $(CsprojFolder)
- cd $(CsprojFolder)
- Move-Item models\$(ModelFolder) models\targeted_model
- dotnet restore -r $(os)-$(arch) /property:Configuration=$(CsprojConfiguration) --source https://api.nuget.org/v3/index.json --source https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/nuget/v3/index.json --source $PWD --disable-parallel --verbosity detailed
- dotnet run -r $(os)-$(arch) --configuration $(CsprojConfiguration) --no-restore --verbosity normal -- -m ./models/targeted_model
- displayName: 'Run $(CsprojName) With Artifact'
+ Copy-Item -Force -Recurse -Verbose $(Build.BinariesDirectory)/nuget/* -Destination ${{ parameters.CsprojFolder }}
+ cd ${{ parameters.CsprojFolder }}
+ dotnet restore -r $(os)-$(arch) /property:Configuration=${{ parameters.CsprojConfiguration }} --source https://api.nuget.org/v3/index.json --source https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/nuget/v3/index.json --source $PWD --disable-parallel --verbosity detailed
+ dotnet run -r $(os)-$(arch) --configuration ${{ parameters.CsprojConfiguration }} --no-restore --verbosity normal -- -m ./models/${{ parameters.ModelFolder }}
+ displayName: 'Run ${{ parameters.CsprojName }} With Artifact'
workingDirectory: '$(Build.Repository.LocalPath)'
env:
NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS: 180
@@ -46,12 +35,11 @@ steps:
- ${{ elseif or(eq(parameters.os, 'linux'), eq(parameters.os, 'osx')) }}:
- bash: |
dotnet --info
- cp $(Build.BinariesDirectory)/nuget/* $(CsprojFolder)
- cd $(CsprojFolder)
- mv models/$(ModelFolder) models/targeted_model
- dotnet restore -r $(os)-$(arch) /property:Configuration=$(CsprojConfiguration) --source https://api.nuget.org/v3/index.json --source https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/nuget/v3/index.json --source $PWD --disable-parallel --verbosity detailed
- dotnet build ./$(CsprojName).csproj -r $(os)-$(arch) /property:Configuration=$(CsprojConfiguration) --no-restore --self-contained
- ls -l ./bin/$(CsprojConfiguration)/net6.0/$(os)-$(arch)/
+ cp $(Build.BinariesDirectory)/nuget/* ${{ parameters.CsprojFolder }}
+ cd ${{ parameters.CsprojFolder }}
+ dotnet restore -r $(os)-$(arch) /property:Configuration=${{ parameters.CsprojConfiguration }} --source https://api.nuget.org/v3/index.json --source https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/nuget/v3/index.json --source $PWD --disable-parallel --verbosity detailed
+ dotnet build ./${{ parameters.CsprojName }}.csproj -r $(os)-$(arch) /property:Configuration=${{ parameters.CsprojConfiguration }} --no-restore --self-contained
+ ls -l ./bin/${{ parameters.CsprojConfiguration }}/net6.0/$(os)-$(arch)/
displayName: 'Perform dotnet restore & build'
workingDirectory: '$(Build.Repository.LocalPath)'
env:
@@ -63,7 +51,7 @@ steps:
set -e -x
az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4
az acr login --name onnxruntimebuildcache --subscription 00c06639-6ee4-454e-8058-8d8b1703bd87
- docker pull $(CudaDockerImage)
+ docker pull $(cuda_docker_image)
docker run \
--gpus all \
@@ -71,12 +59,12 @@ steps:
--volume $(Build.Repository.LocalPath):/ort_genai_src \
--volume $(Build.BinariesDirectory):/ort_genai_binary \
-e HF_TOKEN=$HF_TOKEN \
- -w /ort_genai_src/ $(CudaDockerImage) \
+ -w /ort_genai_src/ $(cuda_docker_image) \
bash -c " \
export ORTGENAI_LOG_ORT_LIB=1 && \
- cd /ort_genai_src/$(CsprojFolder) && \
+ cd /ort_genai_src/${{ parameters.CsprojFolder }} && \
chmod +x ./bin/Release_Cuda/net6.0/linux-x64/$(CsprojName) && \
- ./bin/Release_Cuda/net6.0/linux-x64/$(CsprojName) -m ./models/targeted_model"
+ ./bin/Release_Cuda/net6.0/linux-x64/$(CsprojName) -m ./models/${{ parameters.ModelFolder }}"
displayName: 'Run $(CsprojName) With Artifact'
workingDirectory: '$(Build.Repository.LocalPath)'
@@ -84,7 +72,7 @@ steps:
- ${{ elseif eq(parameters.ep, 'cpu') }}:
- bash: |
export ORTGENAI_LOG_ORT_LIB=1
- cd $(CsprojFolder)
- dotnet run -r $(os)-$(arch) --configuration $(CsprojConfiguration) --no-build --verbosity normal -- -m ./models/targeted_model
+ cd ${{ parameters.CsprojFolder }}
+ dotnet run -r $(os)-$(arch) --configuration ${{ parameters.CsprojConfiguration }} --no-build --verbosity normal -- -m ./models/${{ parameters.ModelFolder }}
displayName: 'Run $(CsprojName) With Artifact'
workingDirectory: '$(Build.Repository.LocalPath)'
diff --git a/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml b/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
new file mode 100644
index 0000000000..7e1115890d
--- /dev/null
+++ b/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
@@ -0,0 +1,78 @@
+parameters:
+- name: PythonScriptFolder
+ type: string
+- name: PythonScriptName
+ type: string
+- name: ModelFolder
+ type: string
+
+steps:
+ - ${{ if or(eq(parameters.os, 'linux'), eq(parameters.os, 'osx')) }}:
+ - ${{ if eq(parameters.ep, 'cuda') }}:
+ - bash: |
+ set -e -x
+ az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4
+ az acr login --name onnxruntimebuildcache --subscription 00c06639-6ee4-454e-8058-8d8b1703bd87
+ docker pull $(CudaDockerImage)
+ python_exe=/opt/python/cp310-cp310/bin/python3.10
+
+ docker run \
+ --gpus all \
+ --rm \
+ --volume $(Build.Repository.LocalPath):/ort_genai_src \
+ --volume $(Build.BinariesDirectory):/ort_genai_binary \
+ -e HF_TOKEN=$HF_TOKEN \
+ -w /ort_genai_src/ $(CudaDockerImage) \
+ bash -c " \
+ export ORTGENAI_LOG_ORT_LIB=1 && \
+ $python_exe -m pip install -r /ort_genai_src/test/python/requirements.txt && \
+ $python_exe -m pip install -r /ort_genai_src/test/python/requirements-cuda.txt && \
+ cd /ort_genai_src/$PythonScriptFolder) && \
+ $python_exe -m pip install --no-index --find-links=/ort_genai_binary/wheel $(pip_package_name) && \
+ $python_exe model-generate.py -m ./models/${{ parameters.ModelFolder }} --min_length 25 --max_length 50 --verbose"
+
+ displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact'
+ workingDirectory: '$(Build.Repository.LocalPath)'
+ - ${{ elseif eq(parameters.ep, 'cpu') }}:
+ - bash: |
+ export ORTGENAI_LOG_ORT_LIB=1
+ python -m pip install -r test/python/requirements.txt
+ if [[ "$(os)" == "linux" ]]; then
+ python -m pip install -r test/python/requirements-cpu.txt
+ fi
+ if [[ "$(os)" == "osx" ]]; then
+ python -m pip install -r test/python/requirements-macos.txt
+ fi
+ cd ${{ parameters.PythonScriptFolder }}
+ python -m pip install --no-index --find-links=$(Build.BinariesDirectory)/wheel $(pip_package_name)
+ python ${{ parameters.PythonScriptName }} -m ./models/${{ parameters.ModelFolder }} --min_length 25 --max_length 50 --verbose
+ displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact'
+ workingDirectory: '$(Build.Repository.LocalPath)'
+
+ - ${{ if eq(parameters.os, 'win') }}:
+ - ${{ if eq(parameters.ep, 'cuda') }}:
+ - powershell: |
+ $env:AZCOPY_MSI_CLIENT_ID = "63b63039-6328-442f-954b-5a64d124e5b4";
+ azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v$(cuda_version)" 'cuda_sdk'
+ displayName: 'Download CUDA $(cuda_version)'
+ workingDirectory: '$(Build.Repository.LocalPath)'
+ - powershell: |
+ python -m pip install -r test/python/requirements.txt
+ if ("$(ep)" -eq "cuda") {
+ $env:CUDA_PATH = '$(Build.Repository.LocalPath)\cuda_sdk\v$(cuda_version)'
+ $env:PATH = "$env:CUDA_PATH\bin;$env:CUDA_PATH\extras\CUPTI\lib64;$env:PATH"
+ Write-Host $env:PATH
+ python -m pip install -r test/python/requirements-cuda.txt
+ }
+ elseif ("$(ep)" -eq "directml") {
+ python -m pip install -r test/python/requirements-directml.txt
+ }
+ else {
+ python -m pip install -r test/python/requirements-cpu.txt
+ }
+ cd ${{ parameters.PythonScriptFolder }}
+ python -m pip install --no-index --find-links=$(Build.BinariesDirectory)/wheel $(pip_package_name)
+
+ python ${{ parameters.PythonScriptName }} -m .\models\${{ parameters.ModelFolder }} --min_length 25 --max_length 50 --verbose
+ displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact'
+ workingDirectory: '$(Build.Repository.LocalPath)'
\ No newline at end of file
From e75d4003e77c48da27246090b3545d6677b8b86f Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Fri, 27 Sep 2024 13:20:28 +0800
Subject: [PATCH 09/42] fix
---
.pipelines/stages/jobs/nuget-validation-job.yml | 1 -
.../jobs/steps/utils/perform-python-validation-with-model.yml | 4 ++--
2 files changed, 2 insertions(+), 3 deletions(-)
diff --git a/.pipelines/stages/jobs/nuget-validation-job.yml b/.pipelines/stages/jobs/nuget-validation-job.yml
index 63af39e89c..46bb2e0983 100644
--- a/.pipelines/stages/jobs/nuget-validation-job.yml
+++ b/.pipelines/stages/jobs/nuget-validation-job.yml
@@ -144,7 +144,6 @@ jobs:
CsprojName: "HelloPhi"
CsprojConfiguration: $(csproj_config)
CudaVersion: $(cuda_version)
- CudaDockerImage: $(cuda_docker_image)
ModelFolder: $(prebuild_phi3_mini_model_folder)
- template: steps/compliant-and-cleanup-step.yml
diff --git a/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml b/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
index 7e1115890d..986a464ece 100644
--- a/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
+++ b/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
@@ -13,7 +13,7 @@ steps:
set -e -x
az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4
az acr login --name onnxruntimebuildcache --subscription 00c06639-6ee4-454e-8058-8d8b1703bd87
- docker pull $(CudaDockerImage)
+ docker pull $(cuda_docker_image)
python_exe=/opt/python/cp310-cp310/bin/python3.10
docker run \
@@ -22,7 +22,7 @@ steps:
--volume $(Build.Repository.LocalPath):/ort_genai_src \
--volume $(Build.BinariesDirectory):/ort_genai_binary \
-e HF_TOKEN=$HF_TOKEN \
- -w /ort_genai_src/ $(CudaDockerImage) \
+ -w /ort_genai_src/ $(cuda_docker_image) \
bash -c " \
export ORTGENAI_LOG_ORT_LIB=1 && \
$python_exe -m pip install -r /ort_genai_src/test/python/requirements.txt && \
From b81d0cfa6cc87cde881cbe4890287befde770f40 Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Fri, 27 Sep 2024 13:21:04 +0800
Subject: [PATCH 10/42] fix
---
.pipelines/stages/jobs/nuget-validation-job.yml | 1 -
1 file changed, 1 deletion(-)
diff --git a/.pipelines/stages/jobs/nuget-validation-job.yml b/.pipelines/stages/jobs/nuget-validation-job.yml
index 46bb2e0983..2387d2a41c 100644
--- a/.pipelines/stages/jobs/nuget-validation-job.yml
+++ b/.pipelines/stages/jobs/nuget-validation-job.yml
@@ -143,7 +143,6 @@ jobs:
CsprojFolder: "examples/csharp/HelloPhi"
CsprojName: "HelloPhi"
CsprojConfiguration: $(csproj_config)
- CudaVersion: $(cuda_version)
ModelFolder: $(prebuild_phi3_mini_model_folder)
- template: steps/compliant-and-cleanup-step.yml
From 5d8e3fd401ee816954e715b81513011da1a6a769 Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Fri, 27 Sep 2024 13:24:51 +0800
Subject: [PATCH 11/42] variables
---
.../utils/perform-nuget-validation-with-model.yml | 10 +++++-----
.../utils/perform-python-validation-with-model.yml | 10 +++++-----
2 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml b/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
index 0a39c35578..99307d9b72 100644
--- a/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
+++ b/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
@@ -9,8 +9,8 @@ parameters:
type: string
steps:
-- ${{ if eq(parameters.os, 'win') }}:
- - ${{ if eq(parameters.ep, 'cuda') }}:
+- ${{ if eq(variables['os'], 'win') }}:
+ - ${{ if eq(variables['ep'], 'cuda') }}:
- powershell: |
$env:AZCOPY_MSI_CLIENT_ID = "63b63039-6328-442f-954b-5a64d124e5b4";
azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v$(cuda_version)" 'cuda_sdk'
@@ -32,7 +32,7 @@ steps:
env:
NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS: 180
NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS: 180
-- ${{ elseif or(eq(parameters.os, 'linux'), eq(parameters.os, 'osx')) }}:
+- ${{ elseif or(eq(variables['os'], 'linux'), eq(variables['os'], 'osx')) }}:
- bash: |
dotnet --info
cp $(Build.BinariesDirectory)/nuget/* ${{ parameters.CsprojFolder }}
@@ -46,7 +46,7 @@ steps:
NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS: 180
NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS: 180
- - ${{ if eq(parameters.ep, 'cuda') }}:
+ - ${{ if eq(variables['ep'], 'cuda') }}:
- bash: |
set -e -x
az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4
@@ -69,7 +69,7 @@ steps:
displayName: 'Run $(CsprojName) With Artifact'
workingDirectory: '$(Build.Repository.LocalPath)'
- - ${{ elseif eq(parameters.ep, 'cpu') }}:
+ - ${{ elseif eq(variables['ep'], 'cpu') }}:
- bash: |
export ORTGENAI_LOG_ORT_LIB=1
cd ${{ parameters.CsprojFolder }}
diff --git a/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml b/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
index 986a464ece..947ed4d2d6 100644
--- a/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
+++ b/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
@@ -7,8 +7,8 @@ parameters:
type: string
steps:
- - ${{ if or(eq(parameters.os, 'linux'), eq(parameters.os, 'osx')) }}:
- - ${{ if eq(parameters.ep, 'cuda') }}:
+ - ${{ if or(eq(variables['os'], 'linux'), eq(variables['os'], 'osx')) }}:
+ - ${{ if eq(variables['ep'], 'cuda') }}:
- bash: |
set -e -x
az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4
@@ -33,7 +33,7 @@ steps:
displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact'
workingDirectory: '$(Build.Repository.LocalPath)'
- - ${{ elseif eq(parameters.ep, 'cpu') }}:
+ - ${{ elseif eq(variables['ep'], 'cpu') }}:
- bash: |
export ORTGENAI_LOG_ORT_LIB=1
python -m pip install -r test/python/requirements.txt
@@ -49,8 +49,8 @@ steps:
displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact'
workingDirectory: '$(Build.Repository.LocalPath)'
- - ${{ if eq(parameters.os, 'win') }}:
- - ${{ if eq(parameters.ep, 'cuda') }}:
+ - ${{ if eq(variables['os'], 'win') }}:
+ - ${{ if eq(variables['ep'], 'cuda') }}:
- powershell: |
$env:AZCOPY_MSI_CLIENT_ID = "63b63039-6328-442f-954b-5a64d124e5b4";
azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v$(cuda_version)" 'cuda_sdk'
From 3588769630fc5c264047fd1b3ac95ba34dee37c4 Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Fri, 27 Sep 2024 14:06:35 +0800
Subject: [PATCH 12/42] condition
---
.../stages/jobs/nuget-validation-job.yml | 3 +-
.../perform-nuget-validation-with-model.yml | 73 +++++++++----------
2 files changed, 37 insertions(+), 39 deletions(-)
diff --git a/.pipelines/stages/jobs/nuget-validation-job.yml b/.pipelines/stages/jobs/nuget-validation-job.yml
index 2387d2a41c..8232d3541e 100644
--- a/.pipelines/stages/jobs/nuget-validation-job.yml
+++ b/.pipelines/stages/jobs/nuget-validation-job.yml
@@ -120,7 +120,7 @@ jobs:
inputs:
version: '8.x'
- - template: steps/utils//flex-download-pipeline-artifact.yml
+ - template: steps/utils/flex-download-pipeline-artifact.yml
parameters:
StepName: 'Download NuGet Artifacts'
ArtifactName: $(artifactName)-nuget
@@ -146,4 +146,3 @@ jobs:
ModelFolder: $(prebuild_phi3_mini_model_folder)
- template: steps/compliant-and-cleanup-step.yml
-
diff --git a/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml b/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
index 99307d9b72..4d6cba34d5 100644
--- a/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
+++ b/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
@@ -9,13 +9,12 @@ parameters:
type: string
steps:
-- ${{ if eq(variables['os'], 'win') }}:
- - ${{ if eq(variables['ep'], 'cuda') }}:
- - powershell: |
- $env:AZCOPY_MSI_CLIENT_ID = "63b63039-6328-442f-954b-5a64d124e5b4";
- azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v$(cuda_version)" 'cuda_sdk'
- displayName: 'Download CUDA $(cuda_version)'
- workingDirectory: '$(Build.Repository.LocalPath)'
+ - powershell: |
+ $env:AZCOPY_MSI_CLIENT_ID = "63b63039-6328-442f-954b-5a64d124e5b4";
+ azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v$(cuda_version)" 'cuda_sdk'
+ displayName: 'Download CUDA $(cuda_version)'
+ workingDirectory: '$(Build.Repository.LocalPath)'
+ condition: and(eq(variables['os'], 'win'), eq(variables['ep'], 'cuda'))
- powershell: |
if ("$(ep)" -eq "cuda") {
$env:CUDA_PATH = '$(Build.Repository.LocalPath)\cuda_sdk\v$(cuda_version)'
@@ -27,12 +26,12 @@ steps:
cd ${{ parameters.CsprojFolder }}
dotnet restore -r $(os)-$(arch) /property:Configuration=${{ parameters.CsprojConfiguration }} --source https://api.nuget.org/v3/index.json --source https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/nuget/v3/index.json --source $PWD --disable-parallel --verbosity detailed
dotnet run -r $(os)-$(arch) --configuration ${{ parameters.CsprojConfiguration }} --no-restore --verbosity normal -- -m ./models/${{ parameters.ModelFolder }}
- displayName: 'Run ${{ parameters.CsprojName }} With Artifact'
+ displayName: 'Run ${{ parameters.CsprojName }} With Artifact'
workingDirectory: '$(Build.Repository.LocalPath)'
+ condition: eq(variables['os'], 'win'))
env:
NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS: 180
NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS: 180
-- ${{ elseif or(eq(variables['os'], 'linux'), eq(variables['os'], 'osx')) }}:
- bash: |
dotnet --info
cp $(Build.BinariesDirectory)/nuget/* ${{ parameters.CsprojFolder }}
@@ -42,37 +41,37 @@ steps:
ls -l ./bin/${{ parameters.CsprojConfiguration }}/net6.0/$(os)-$(arch)/
displayName: 'Perform dotnet restore & build'
workingDirectory: '$(Build.Repository.LocalPath)'
+ condition: or(eq(variables['os'], 'linux'), eq(variables['os'], 'osx'))
env:
NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS: 180
NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS: 180
+ - bash: |
+ set -e -x
+ az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4
+ az acr login --name onnxruntimebuildcache --subscription 00c06639-6ee4-454e-8058-8d8b1703bd87
+ docker pull $(cuda_docker_image)
- - ${{ if eq(variables['ep'], 'cuda') }}:
- - bash: |
- set -e -x
- az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4
- az acr login --name onnxruntimebuildcache --subscription 00c06639-6ee4-454e-8058-8d8b1703bd87
- docker pull $(cuda_docker_image)
-
- docker run \
- --gpus all \
- --rm \
- --volume $(Build.Repository.LocalPath):/ort_genai_src \
- --volume $(Build.BinariesDirectory):/ort_genai_binary \
- -e HF_TOKEN=$HF_TOKEN \
- -w /ort_genai_src/ $(cuda_docker_image) \
- bash -c " \
- export ORTGENAI_LOG_ORT_LIB=1 && \
- cd /ort_genai_src/${{ parameters.CsprojFolder }} && \
- chmod +x ./bin/Release_Cuda/net6.0/linux-x64/$(CsprojName) && \
- ./bin/Release_Cuda/net6.0/linux-x64/$(CsprojName) -m ./models/${{ parameters.ModelFolder }}"
+ docker run \
+ --gpus all \
+ --rm \
+ --volume $(Build.Repository.LocalPath):/ort_genai_src \
+ --volume $(Build.BinariesDirectory):/ort_genai_binary \
+ -e HF_TOKEN=$HF_TOKEN \
+ -w /ort_genai_src/ $(cuda_docker_image) \
+ bash -c " \
+ export ORTGENAI_LOG_ORT_LIB=1 && \
+ cd /ort_genai_src/${{ parameters.CsprojFolder }} && \
+ chmod +x ./bin/Release_Cuda/net6.0/linux-x64/$(CsprojName) && \
+ ./bin/Release_Cuda/net6.0/linux-x64/$(CsprojName) -m ./models/${{ parameters.ModelFolder }}"
- displayName: 'Run $(CsprojName) With Artifact'
- workingDirectory: '$(Build.Repository.LocalPath)'
+ displayName: 'Run $(CsprojName) With Artifact'
+ workingDirectory: '$(Build.Repository.LocalPath)'
+ condition: and(eq(variables['os'], 'linux'), eq(variables['ep'], 'cuda'))
- - ${{ elseif eq(variables['ep'], 'cpu') }}:
- - bash: |
- export ORTGENAI_LOG_ORT_LIB=1
- cd ${{ parameters.CsprojFolder }}
- dotnet run -r $(os)-$(arch) --configuration ${{ parameters.CsprojConfiguration }} --no-build --verbosity normal -- -m ./models/${{ parameters.ModelFolder }}
- displayName: 'Run $(CsprojName) With Artifact'
- workingDirectory: '$(Build.Repository.LocalPath)'
+ - bash: |
+ export ORTGENAI_LOG_ORT_LIB=1
+ cd ${{ parameters.CsprojFolder }}
+ dotnet run -r $(os)-$(arch) --configuration ${{ parameters.CsprojConfiguration }} --no-build --verbosity normal -- -m ./models/${{ parameters.ModelFolder }}
+ displayName: 'Run $(CsprojName) With Artifact'
+ workingDirectory: '$(Build.Repository.LocalPath)'
+ condition: and(or(eq(variables['os'], 'linux'), eq(variables['os'], 'osx'), eq(variables['ep'], 'cpu'))
From b409c5665c9fdc3e9135b5c169af3b631b6476a6 Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Fri, 27 Sep 2024 14:30:06 +0800
Subject: [PATCH 13/42] condition
---
.../jobs/steps/utils/perform-nuget-validation-with-model.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml b/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
index 4d6cba34d5..9cd18d5f1d 100644
--- a/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
+++ b/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
@@ -28,7 +28,7 @@ steps:
dotnet run -r $(os)-$(arch) --configuration ${{ parameters.CsprojConfiguration }} --no-restore --verbosity normal -- -m ./models/${{ parameters.ModelFolder }}
displayName: 'Run ${{ parameters.CsprojName }} With Artifact'
workingDirectory: '$(Build.Repository.LocalPath)'
- condition: eq(variables['os'], 'win'))
+ condition: eq(variables['os'], 'win')
env:
NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS: 180
NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS: 180
From 9463b986129dfcf26e071f7adf5dc0fe6191157f Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Fri, 27 Sep 2024 14:31:44 +0800
Subject: [PATCH 14/42] condition
---
.../jobs/steps/utils/perform-nuget-validation-with-model.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml b/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
index 9cd18d5f1d..9d9d54d18e 100644
--- a/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
+++ b/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
@@ -74,4 +74,4 @@ steps:
dotnet run -r $(os)-$(arch) --configuration ${{ parameters.CsprojConfiguration }} --no-build --verbosity normal -- -m ./models/${{ parameters.ModelFolder }}
displayName: 'Run $(CsprojName) With Artifact'
workingDirectory: '$(Build.Repository.LocalPath)'
- condition: and(or(eq(variables['os'], 'linux'), eq(variables['os'], 'osx'), eq(variables['ep'], 'cpu'))
+ condition: and(or(eq(variables['os'], 'linux'), eq(variables['os'], 'osx')), eq(variables['ep'], 'cpu'))
From 5e0bc64a7c14e74b2fd60ba046502a8e76633d87 Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Fri, 27 Sep 2024 14:58:43 +0800
Subject: [PATCH 15/42] update
---
.../perform-nuget-validation-with-model.yml | 9 +-
.../perform-python-validation-with-model.yml | 130 +++++++++---------
2 files changed, 70 insertions(+), 69 deletions(-)
diff --git a/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml b/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
index 9d9d54d18e..f49ee2b9a8 100644
--- a/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
+++ b/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
@@ -32,6 +32,7 @@ steps:
env:
NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS: 180
NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS: 180
+
- bash: |
dotnet --info
cp $(Build.BinariesDirectory)/nuget/* ${{ parameters.CsprojFolder }}
@@ -61,10 +62,10 @@ steps:
bash -c " \
export ORTGENAI_LOG_ORT_LIB=1 && \
cd /ort_genai_src/${{ parameters.CsprojFolder }} && \
- chmod +x ./bin/Release_Cuda/net6.0/linux-x64/$(CsprojName) && \
- ./bin/Release_Cuda/net6.0/linux-x64/$(CsprojName) -m ./models/${{ parameters.ModelFolder }}"
+ chmod +x ./bin/Release_Cuda/net6.0/linux-x64/${{ parameters.CsprojName }} && \
+ ./bin/Release_Cuda/net6.0/linux-x64/${{ parameters.CsprojName }} -m ./models/${{ parameters.ModelFolder }}"
- displayName: 'Run $(CsprojName) With Artifact'
+ displayName: 'Run ${{ parameters.CsprojName }} With Artifact'
workingDirectory: '$(Build.Repository.LocalPath)'
condition: and(eq(variables['os'], 'linux'), eq(variables['ep'], 'cuda'))
@@ -72,6 +73,6 @@ steps:
export ORTGENAI_LOG_ORT_LIB=1
cd ${{ parameters.CsprojFolder }}
dotnet run -r $(os)-$(arch) --configuration ${{ parameters.CsprojConfiguration }} --no-build --verbosity normal -- -m ./models/${{ parameters.ModelFolder }}
- displayName: 'Run $(CsprojName) With Artifact'
+ displayName: 'Run ${{ parameters.CsprojName }} With Artifact'
workingDirectory: '$(Build.Repository.LocalPath)'
condition: and(or(eq(variables['os'], 'linux'), eq(variables['os'], 'osx')), eq(variables['ep'], 'cpu'))
diff --git a/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml b/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
index 947ed4d2d6..ea5cb50c40 100644
--- a/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
+++ b/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
@@ -7,72 +7,72 @@ parameters:
type: string
steps:
- - ${{ if or(eq(variables['os'], 'linux'), eq(variables['os'], 'osx')) }}:
- - ${{ if eq(variables['ep'], 'cuda') }}:
- - bash: |
- set -e -x
- az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4
- az acr login --name onnxruntimebuildcache --subscription 00c06639-6ee4-454e-8058-8d8b1703bd87
- docker pull $(cuda_docker_image)
- python_exe=/opt/python/cp310-cp310/bin/python3.10
+ - powershell: |
+ $env:AZCOPY_MSI_CLIENT_ID = "63b63039-6328-442f-954b-5a64d124e5b4";
+ azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v$(cuda_version)" 'cuda_sdk'
+ displayName: 'Download CUDA $(cuda_version)'
+ workingDirectory: '$(Build.Repository.LocalPath)'
+ condition: and(eq(variables['os'], 'win'), eq(variables['ep'], 'cuda'))
+ - powershell: |
+ python -m pip install -r test/python/requirements.txt
+ if ("$(ep)" -eq "cuda") {
+ $env:CUDA_PATH = '$(Build.Repository.LocalPath)\cuda_sdk\v$(cuda_version)'
+ $env:PATH = "$env:CUDA_PATH\bin;$env:CUDA_PATH\extras\CUPTI\lib64;$env:PATH"
+ Write-Host $env:PATH
+ python -m pip install -r test/python/requirements-cuda.txt
+ }
+ elseif ("$(ep)" -eq "directml") {
+ python -m pip install -r test/python/requirements-directml.txt
+ }
+ else {
+ python -m pip install -r test/python/requirements-cpu.txt
+ }
+ cd ${{ parameters.PythonScriptFolder }}
+ python -m pip install --no-index --find-links=$(Build.BinariesDirectory)/wheel $(pip_package_name)
- docker run \
- --gpus all \
- --rm \
- --volume $(Build.Repository.LocalPath):/ort_genai_src \
- --volume $(Build.BinariesDirectory):/ort_genai_binary \
- -e HF_TOKEN=$HF_TOKEN \
- -w /ort_genai_src/ $(cuda_docker_image) \
- bash -c " \
- export ORTGENAI_LOG_ORT_LIB=1 && \
- $python_exe -m pip install -r /ort_genai_src/test/python/requirements.txt && \
- $python_exe -m pip install -r /ort_genai_src/test/python/requirements-cuda.txt && \
- cd /ort_genai_src/$PythonScriptFolder) && \
- $python_exe -m pip install --no-index --find-links=/ort_genai_binary/wheel $(pip_package_name) && \
- $python_exe model-generate.py -m ./models/${{ parameters.ModelFolder }} --min_length 25 --max_length 50 --verbose"
+ python ${{ parameters.PythonScriptName }} -m .\models\${{ parameters.ModelFolder }} --min_length 25 --max_length 50 --verbose
+ displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact'
+ workingDirectory: '$(Build.Repository.LocalPath)'
+ condition: eq(variables['os'], 'win')
- displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact'
- workingDirectory: '$(Build.Repository.LocalPath)'
- - ${{ elseif eq(variables['ep'], 'cpu') }}:
- - bash: |
- export ORTGENAI_LOG_ORT_LIB=1
- python -m pip install -r test/python/requirements.txt
- if [[ "$(os)" == "linux" ]]; then
- python -m pip install -r test/python/requirements-cpu.txt
- fi
- if [[ "$(os)" == "osx" ]]; then
- python -m pip install -r test/python/requirements-macos.txt
- fi
- cd ${{ parameters.PythonScriptFolder }}
- python -m pip install --no-index --find-links=$(Build.BinariesDirectory)/wheel $(pip_package_name)
- python ${{ parameters.PythonScriptName }} -m ./models/${{ parameters.ModelFolder }} --min_length 25 --max_length 50 --verbose
- displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact'
- workingDirectory: '$(Build.Repository.LocalPath)'
+ - bash: |
+ set -e -x
+ az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4
+ az acr login --name onnxruntimebuildcache --subscription 00c06639-6ee4-454e-8058-8d8b1703bd87
+ docker pull $(cuda_docker_image)
+ python_exe=/opt/python/cp310-cp310/bin/python3.10
- - ${{ if eq(variables['os'], 'win') }}:
- - ${{ if eq(variables['ep'], 'cuda') }}:
- - powershell: |
- $env:AZCOPY_MSI_CLIENT_ID = "63b63039-6328-442f-954b-5a64d124e5b4";
- azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v$(cuda_version)" 'cuda_sdk'
- displayName: 'Download CUDA $(cuda_version)'
- workingDirectory: '$(Build.Repository.LocalPath)'
- - powershell: |
- python -m pip install -r test/python/requirements.txt
- if ("$(ep)" -eq "cuda") {
- $env:CUDA_PATH = '$(Build.Repository.LocalPath)\cuda_sdk\v$(cuda_version)'
- $env:PATH = "$env:CUDA_PATH\bin;$env:CUDA_PATH\extras\CUPTI\lib64;$env:PATH"
- Write-Host $env:PATH
- python -m pip install -r test/python/requirements-cuda.txt
- }
- elseif ("$(ep)" -eq "directml") {
- python -m pip install -r test/python/requirements-directml.txt
- }
- else {
- python -m pip install -r test/python/requirements-cpu.txt
- }
- cd ${{ parameters.PythonScriptFolder }}
- python -m pip install --no-index --find-links=$(Build.BinariesDirectory)/wheel $(pip_package_name)
+ docker run \
+ --gpus all \
+ --rm \
+ --volume $(Build.Repository.LocalPath):/ort_genai_src \
+ --volume $(Build.BinariesDirectory):/ort_genai_binary \
+ -e HF_TOKEN=$HF_TOKEN \
+ -w /ort_genai_src/ $(cuda_docker_image) \
+ bash -c " \
+ export ORTGENAI_LOG_ORT_LIB=1 && \
+ $python_exe -m pip install -r /ort_genai_src/test/python/requirements.txt && \
+ $python_exe -m pip install -r /ort_genai_src/test/python/requirements-cuda.txt && \
+ cd /ort_genai_src/$PythonScriptFolder) && \
+ $python_exe -m pip install --no-index --find-links=/ort_genai_binary/wheel $(pip_package_name) && \
+ $python_exe model-generate.py -m ./models/${{ parameters.ModelFolder }} --min_length 25 --max_length 50 --verbose"
- python ${{ parameters.PythonScriptName }} -m .\models\${{ parameters.ModelFolder }} --min_length 25 --max_length 50 --verbose
- displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact'
- workingDirectory: '$(Build.Repository.LocalPath)'
\ No newline at end of file
+ displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact'
+ workingDirectory: '$(Build.Repository.LocalPath)'
+ condition: and(eq(variables['os'], 'linux'), eq(variables['ep'], 'cuda'))
+
+ - bash: |
+ export ORTGENAI_LOG_ORT_LIB=1
+ python -m pip install -r test/python/requirements.txt
+ if [[ "$(os)" == "linux" ]]; then
+ python -m pip install -r test/python/requirements-cpu.txt
+ fi
+ if [[ "$(os)" == "osx" ]]; then
+ python -m pip install -r test/python/requirements-macos.txt
+ fi
+ cd ${{ parameters.PythonScriptFolder }}
+ python -m pip install --no-index --find-links=$(Build.BinariesDirectory)/wheel $(pip_package_name)
+ python ${{ parameters.PythonScriptName }} -m ./models/${{ parameters.ModelFolder }} --min_length 25 --max_length 50 --verbose
+ displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact'
+ workingDirectory: '$(Build.Repository.LocalPath)'
+ condition: and(or(eq(variables['os'], 'linux'), eq(variables['os'], 'osx')), eq(variables['ep'], 'cpu'))
\ No newline at end of file
From 07b2483f5df72a5afbbeaf0b75f2ad2400e47ca1 Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Fri, 27 Sep 2024 15:28:37 +0800
Subject: [PATCH 16/42] csproj_configuration
---
.pipelines/stages/jobs/nuget-validation-job.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.pipelines/stages/jobs/nuget-validation-job.yml b/.pipelines/stages/jobs/nuget-validation-job.yml
index 8232d3541e..0547555797 100644
--- a/.pipelines/stages/jobs/nuget-validation-job.yml
+++ b/.pipelines/stages/jobs/nuget-validation-job.yml
@@ -142,7 +142,7 @@ jobs:
parameters:
CsprojFolder: "examples/csharp/HelloPhi"
CsprojName: "HelloPhi"
- CsprojConfiguration: $(csproj_config)
+ CsprojConfiguration: $(csproj_configuration)
ModelFolder: $(prebuild_phi3_mini_model_folder)
- template: steps/compliant-and-cleanup-step.yml
From 61e1472ee304e6f1b1bb731f434e32e549c65969 Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Wed, 30 Oct 2024 11:32:13 +0800
Subject: [PATCH 17/42] catch up the latest change
---
.../utils/perform-python-validation-with-model.yml | 14 +++++++++-----
1 file changed, 9 insertions(+), 5 deletions(-)
diff --git a/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml b/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
index ea5cb50c40..2f03c28d77 100644
--- a/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
+++ b/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
@@ -19,13 +19,16 @@ steps:
$env:CUDA_PATH = '$(Build.Repository.LocalPath)\cuda_sdk\v$(cuda_version)'
$env:PATH = "$env:CUDA_PATH\bin;$env:CUDA_PATH\extras\CUPTI\lib64;$env:PATH"
Write-Host $env:PATH
- python -m pip install -r test/python/requirements-cuda.txt
+ python -m pip install -r test/python/cuda/torch/requirements.txt
+ python -m pip install -r test/python/cuda/ort/requirements.txt
}
elseif ("$(ep)" -eq "directml") {
- python -m pip install -r test/python/requirements-directml.txt
+ python -m pip install -r test/python/directml/torch/requirements.txt
+ python -m pip install -r test/python/directml/ort/requirements.txt
}
else {
- python -m pip install -r test/python/requirements-cpu.txt
+ python -m pip install -r test/python/cpu/torch/requirements.txt
+ python -m pip install -r test/python/cpu/ort/requirements.txt
}
cd ${{ parameters.PythonScriptFolder }}
python -m pip install --no-index --find-links=$(Build.BinariesDirectory)/wheel $(pip_package_name)
@@ -52,10 +55,11 @@ steps:
bash -c " \
export ORTGENAI_LOG_ORT_LIB=1 && \
$python_exe -m pip install -r /ort_genai_src/test/python/requirements.txt && \
- $python_exe -m pip install -r /ort_genai_src/test/python/requirements-cuda.txt && \
+ $python_exe -m pip install -r /ort_genai_src/test/python/cuda/torch/requirements.txt && \
+ $python_exe -m pip install -r /ort_genai_src/test/python/cuda/ort/requirements.txt && \
cd /ort_genai_src/$PythonScriptFolder) && \
$python_exe -m pip install --no-index --find-links=/ort_genai_binary/wheel $(pip_package_name) && \
- $python_exe model-generate.py -m ./models/${{ parameters.ModelFolder }} --min_length 25 --max_length 50 --verbose"
+ $python_exe ${{ parameters.PythonScriptName }} -m ./models/${{ parameters.ModelFolder }} --min_length 25 --max_length 50 --verbose"
displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact'
workingDirectory: '$(Build.Repository.LocalPath)'
From f5a5d703143f1553309fdf18254550f248da0935 Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Wed, 30 Oct 2024 15:03:47 +0800
Subject: [PATCH 18/42] Fix
---
.../utils/perform-nuget-validation-with-model.yml | 6 +++---
.../utils/perform-python-validation-with-model.yml | 14 ++++++++------
2 files changed, 11 insertions(+), 9 deletions(-)
diff --git a/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml b/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
index f49ee2b9a8..1d37b4a685 100644
--- a/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
+++ b/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
@@ -26,7 +26,7 @@ steps:
cd ${{ parameters.CsprojFolder }}
dotnet restore -r $(os)-$(arch) /property:Configuration=${{ parameters.CsprojConfiguration }} --source https://api.nuget.org/v3/index.json --source https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/nuget/v3/index.json --source $PWD --disable-parallel --verbosity detailed
dotnet run -r $(os)-$(arch) --configuration ${{ parameters.CsprojConfiguration }} --no-restore --verbosity normal -- -m ./models/${{ parameters.ModelFolder }}
- displayName: 'Run ${{ parameters.CsprojName }} With Artifact'
+ displayName: 'Run ${{ parameters.CsprojName }} With Artifact on Windows'
workingDirectory: '$(Build.Repository.LocalPath)'
condition: eq(variables['os'], 'win')
env:
@@ -65,7 +65,7 @@ steps:
chmod +x ./bin/Release_Cuda/net6.0/linux-x64/${{ parameters.CsprojName }} && \
./bin/Release_Cuda/net6.0/linux-x64/${{ parameters.CsprojName }} -m ./models/${{ parameters.ModelFolder }}"
- displayName: 'Run ${{ parameters.CsprojName }} With Artifact'
+ displayName: 'Run ${{ parameters.CsprojName }} With Artifact on Linux CUDA'
workingDirectory: '$(Build.Repository.LocalPath)'
condition: and(eq(variables['os'], 'linux'), eq(variables['ep'], 'cuda'))
@@ -73,6 +73,6 @@ steps:
export ORTGENAI_LOG_ORT_LIB=1
cd ${{ parameters.CsprojFolder }}
dotnet run -r $(os)-$(arch) --configuration ${{ parameters.CsprojConfiguration }} --no-build --verbosity normal -- -m ./models/${{ parameters.ModelFolder }}
- displayName: 'Run ${{ parameters.CsprojName }} With Artifact'
+ displayName: 'Run ${{ parameters.CsprojName }} With Artifact on Linux/macOS CPU'
workingDirectory: '$(Build.Repository.LocalPath)'
condition: and(or(eq(variables['os'], 'linux'), eq(variables['os'], 'osx')), eq(variables['ep'], 'cpu'))
diff --git a/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml b/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
index 2f03c28d77..163e1ecf8e 100644
--- a/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
+++ b/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
@@ -34,7 +34,7 @@ steps:
python -m pip install --no-index --find-links=$(Build.BinariesDirectory)/wheel $(pip_package_name)
python ${{ parameters.PythonScriptName }} -m .\models\${{ parameters.ModelFolder }} --min_length 25 --max_length 50 --verbose
- displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact'
+ displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact on Windows'
workingDirectory: '$(Build.Repository.LocalPath)'
condition: eq(variables['os'], 'win')
@@ -57,11 +57,11 @@ steps:
$python_exe -m pip install -r /ort_genai_src/test/python/requirements.txt && \
$python_exe -m pip install -r /ort_genai_src/test/python/cuda/torch/requirements.txt && \
$python_exe -m pip install -r /ort_genai_src/test/python/cuda/ort/requirements.txt && \
- cd /ort_genai_src/$PythonScriptFolder) && \
+ cd /ort_genai_src/${{ parameters.PythonScriptFolder }} && \
$python_exe -m pip install --no-index --find-links=/ort_genai_binary/wheel $(pip_package_name) && \
$python_exe ${{ parameters.PythonScriptName }} -m ./models/${{ parameters.ModelFolder }} --min_length 25 --max_length 50 --verbose"
- displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact'
+ displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact on Linux CUDA'
workingDirectory: '$(Build.Repository.LocalPath)'
condition: and(eq(variables['os'], 'linux'), eq(variables['ep'], 'cuda'))
@@ -69,14 +69,16 @@ steps:
export ORTGENAI_LOG_ORT_LIB=1
python -m pip install -r test/python/requirements.txt
if [[ "$(os)" == "linux" ]]; then
- python -m pip install -r test/python/requirements-cpu.txt
+ python -m pip install -r test/python/cpu/torch/requirements.txt
+ python -m pip install -r test/python/cpu/ort/requirements.txt
fi
if [[ "$(os)" == "osx" ]]; then
- python -m pip install -r test/python/requirements-macos.txt
+ python -m pip install -r test/python/macos/torch/requirements.txt
+ python -m pip install -r test/python/macos/ort/requirements.txt
fi
cd ${{ parameters.PythonScriptFolder }}
python -m pip install --no-index --find-links=$(Build.BinariesDirectory)/wheel $(pip_package_name)
python ${{ parameters.PythonScriptName }} -m ./models/${{ parameters.ModelFolder }} --min_length 25 --max_length 50 --verbose
- displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact'
+ displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact on Linux/macOS CPU'
workingDirectory: '$(Build.Repository.LocalPath)'
condition: and(or(eq(variables['os'], 'linux'), eq(variables['os'], 'osx')), eq(variables['ep'], 'cpu'))
\ No newline at end of file
From 0a23bde57bafbe3b0d14544b3df61586eb43e85e Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Fri, 15 Nov 2024 12:12:37 +0800
Subject: [PATCH 19/42] phi3.5 vision validation
---
.../stages/jobs/nuget-validation-job.yml | 31 ++++++++++++++++++-
.pipelines/stages/jobs/py-validation-job.yml | 30 +++++++++++++++++-
.../perform-nuget-validation-with-model.yml | 8 +++--
.../perform-python-validation-with-model.yml | 6 ++--
4 files changed, 68 insertions(+), 7 deletions(-)
diff --git a/.pipelines/stages/jobs/nuget-validation-job.yml b/.pipelines/stages/jobs/nuget-validation-job.yml
index 0547555797..9217f2a909 100644
--- a/.pipelines/stages/jobs/nuget-validation-job.yml
+++ b/.pipelines/stages/jobs/nuget-validation-job.yml
@@ -88,6 +88,16 @@ jobs:
${{ else }}:
value: 'cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4'
+ - name: prebuild_phi3_5_vision_model_folder
+ ${{ if eq(parameters.ep, 'cpu') }}:
+ value: 'cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4'
+ ${{ elseif eq(parameters.ep, 'cuda') }}:
+ value: 'gpu/gpu-int4-rtn-block-32'
+ ${{ elseif eq(parameters.ep, 'directml')}}:
+ value: 'gpu/gpu-int4-rtn-block-32'
+ ${{ else }}:
+ value: 'cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4'
+
- name: cuda_docker_image
${{ if eq(parameters.cuda_version, '11.8') }}:
value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20240531.1
@@ -133,7 +143,7 @@ jobs:
StepName: 'Download Phi3-mini Model from HuggingFace'
HuggingFaceRepo: 'microsoft/Phi-3-mini-4k-instruct-onnx'
RepoFolder: $(prebuild_phi3_mini_model_folder)
- LocalFolder: 'models'
+ LocalFolder: 'phi3-mini'
WorkingDirectory: '$(Build.Repository.LocalPath)/examples/csharp/HelloPhi'
HuggingFaceToken: $(HF_TOKEN)
os: ${{ parameters.os }}
@@ -143,6 +153,25 @@ jobs:
CsprojFolder: "examples/csharp/HelloPhi"
CsprojName: "HelloPhi"
CsprojConfiguration: $(csproj_configuration)
+ LocalFolder: 'phi3-mini'
ModelFolder: $(prebuild_phi3_mini_model_folder)
+ - template: steps/utils/download-huggingface-model.yml
+ parameters:
+ StepName: 'Download Phi-3.5-vision-instruct-onnx Model from HuggingFace'
+ HuggingFaceRepo: 'microsoft/Phi-3.5-vision-instruct-onnx'
+ RepoFolder: $(prebuild_phi3_5_vision_model_folder)
+ LocalFolder: 'phi3.5-vision'
+ WorkingDirectory: '$(Build.Repository.LocalPath)/examples/csharp/HelloPhi'
+ HuggingFaceToken: $(HF_TOKEN)
+ os: ${{ parameters.os }}
+
+ - template: steps/utils/perform-nuget-validation-with-model.yml
+ parameters:
+ CsprojFolder: "examples/csharp/HelloPhi3V"
+ CsprojName: "HelloPhi3V"
+ CsprojConfiguration: $(csproj_configuration)
+ LocalFolder: 'phi3.5-vision'
+ ModelFolder: $(prebuild_phi3_5_vision_model_folder)
+
- template: steps/compliant-and-cleanup-step.yml
diff --git a/.pipelines/stages/jobs/py-validation-job.yml b/.pipelines/stages/jobs/py-validation-job.yml
index cc13fa1850..6bb287d9e0 100644
--- a/.pipelines/stages/jobs/py-validation-job.yml
+++ b/.pipelines/stages/jobs/py-validation-job.yml
@@ -97,6 +97,16 @@ jobs:
${{ else }}:
value: 'cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4'
+ - name: prebuild_phi3_5_vision_model_folder
+ ${{ if eq(parameters.ep, 'cpu') }}:
+ value: 'cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4'
+ ${{ elseif eq(parameters.ep, 'cuda') }}:
+ value: 'gpu/gpu-int4-rtn-block-32'
+ ${{ elseif eq(parameters.ep, 'directml')}}:
+ value: 'gpu/gpu-int4-rtn-block-32'
+ ${{ else }}:
+ value: 'cpu_and_mobile/cpu-int4-rtn-block-32-acc-level-4'
+
- name: cuda_docker_image
${{ if eq(parameters.cuda_version, '11.8') }}:
value: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20240531.1
@@ -150,7 +160,7 @@ jobs:
StepName: 'Download Phi3-mini from HuggingFace'
HuggingFaceRepo: 'microsoft/Phi-3-mini-4k-instruct-onnx'
RepoFolder: $(prebuild_phi3_mini_model_folder)
- LocalFolder: 'models'
+ LocalFolder: 'phi3-mini'
WorkingDirectory: '$(Build.Repository.LocalPath)/examples/python'
HuggingFaceToken: $(HF_TOKEN)
os: ${{ parameters.os }}
@@ -159,6 +169,24 @@ jobs:
parameters:
PythonScriptFolder: "examples/python"
PythonScriptName: "model-generate.py"
+ LocalFolder: 'phi3-mini'
ModelFolder: $(prebuild_phi3_mini_model_folder)
+ - template: steps/utils/download-huggingface-model.yml
+ parameters:
+ StepName: 'Download Phi-3.5-vision-instruct-onnx Model from HuggingFace'
+ HuggingFaceRepo: 'microsoft/Phi-3.5-vision-instruct-onnx'
+ RepoFolder: $(prebuild_phi3_5_vision_model_folder)
+ LocalFolder: 'phi3.5-vision'
+ WorkingDirectory: '$(Build.Repository.LocalPath)/examples/csharp/HelloPhi'
+ HuggingFaceToken: $(HF_TOKEN)
+ os: ${{ parameters.os }}
+
+ - template: steps/utils/perform-python-validation-with-model.yml
+ parameters:
+ PythonScriptFolder: "examples/python"
+ PythonScriptName: "phi3v.py"
+ LocalFolder: 'phi3.5-vision'
+ ModelFolder: $(prebuild_phi3_5_vision_model_folder)
+
- template: steps/compliant-and-cleanup-step.yml
diff --git a/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml b/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
index 8a449ff999..a2f48383f1 100644
--- a/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
+++ b/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
@@ -5,6 +5,8 @@ parameters:
type: string
- name: CsprojConfiguration
type: string
+- name: LocalFolder
+ type: string
- name: ModelFolder
type: string
@@ -32,7 +34,7 @@ steps:
Copy-Item -Force -Recurse -Verbose $(Build.BinariesDirectory)/nuget/* -Destination ${{ parameters.CsprojFolder }}
cd ${{ parameters.CsprojFolder }}
dotnet restore -r $(os)-$(arch) /property:Configuration=${{ parameters.CsprojConfiguration }} --source https://api.nuget.org/v3/index.json --source https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/nuget/v3/index.json --source $PWD --disable-parallel --verbosity detailed
- dotnet run -r $(os)-$(arch) --configuration ${{ parameters.CsprojConfiguration }} --no-restore --verbosity normal -- -m ./models/${{ parameters.ModelFolder }}
+ dotnet run -r $(os)-$(arch) --configuration ${{ parameters.CsprojConfiguration }} --no-restore --verbosity normal -- -m ./{{ $parameters.LocalFolder }}/${{ parameters.ModelFolder }}
displayName: 'Run ${{ parameters.CsprojName }} With Artifact on Windows'
workingDirectory: '$(Build.Repository.LocalPath)'
condition: eq(variables['os'], 'win')
@@ -70,7 +72,7 @@ steps:
export ORTGENAI_LOG_ORT_LIB=1 && \
cd /ort_genai_src/${{ parameters.CsprojFolder }} && \
chmod +x ./bin/Release_Cuda/net6.0/linux-x64/${{ parameters.CsprojName }} && \
- ./bin/Release_Cuda/net6.0/linux-x64/${{ parameters.CsprojName }} -m ./models/${{ parameters.ModelFolder }}"
+ ./bin/Release_Cuda/net6.0/linux-x64/${{ parameters.CsprojName }} -m ./{{ $parameters.LocalFolder }}/${{ parameters.ModelFolder }}"
displayName: 'Run ${{ parameters.CsprojName }} With Artifact on Linux CUDA'
workingDirectory: '$(Build.Repository.LocalPath)'
@@ -79,7 +81,7 @@ steps:
- bash: |
export ORTGENAI_LOG_ORT_LIB=1
cd ${{ parameters.CsprojFolder }}
- dotnet run -r $(os)-$(arch) --configuration ${{ parameters.CsprojConfiguration }} --no-build --verbosity normal -- -m ./models/${{ parameters.ModelFolder }}
+ dotnet run -r $(os)-$(arch) --configuration ${{ parameters.CsprojConfiguration }} --no-build --verbosity normal -- -m ./{{ $parameters.LocalFolder }}/${{ parameters.ModelFolder }}
displayName: 'Run ${{ parameters.CsprojName }} With Artifact on Linux/macOS CPU'
workingDirectory: '$(Build.Repository.LocalPath)'
condition: and(or(eq(variables['os'], 'linux'), eq(variables['os'], 'osx')), eq(variables['ep'], 'cpu'))
diff --git a/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml b/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
index 5cb8f742f5..5e23f9d25d 100644
--- a/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
+++ b/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
@@ -3,6 +3,8 @@ parameters:
type: string
- name: PythonScriptName
type: string
+- name: LocalFolder
+ type: string
- name: ModelFolder
type: string
@@ -66,7 +68,7 @@ steps:
$python_exe -m pip install -r /ort_genai_src/test/python/cuda/ort/requirements.txt && \
cd /ort_genai_src/${{ parameters.PythonScriptFolder }} && \
$python_exe -m pip install --no-index --find-links=/ort_genai_binary/wheel $(pip_package_name) && \
- $python_exe ${{ parameters.PythonScriptName }} -m ./models/${{ parameters.ModelFolder }} --min_length 25 --max_length 50 --verbose"
+ $python_exe ${{ parameters.PythonScriptName }} -m ./{{ $parameters.LocalFolder }}/${{ parameters.ModelFolder }} --min_length 25 --max_length 50 --verbose"
displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact on Linux CUDA'
workingDirectory: '$(Build.Repository.LocalPath)'
@@ -85,7 +87,7 @@ steps:
fi
cd ${{ parameters.PythonScriptFolder }}
python -m pip install --no-index --find-links=$(Build.BinariesDirectory)/wheel $(pip_package_name)
- python ${{ parameters.PythonScriptName }} -m ./models/${{ parameters.ModelFolder }} --min_length 25 --max_length 50 --verbose
+ python ${{ parameters.PythonScriptName }} -m ./{{ $parameters.LocalFolder }}/${{ parameters.ModelFolder }} --min_length 25 --max_length 50 --verbose
displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact on Linux/macOS CPU'
workingDirectory: '$(Build.Repository.LocalPath)'
condition: and(or(eq(variables['os'], 'linux'), eq(variables['os'], 'osx')), eq(variables['ep'], 'cpu'))
\ No newline at end of file
From ff339782f59b42f235f356d5a3916bca472b1ef2 Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Fri, 15 Nov 2024 12:20:49 +0800
Subject: [PATCH 20/42] Fix
---
.pipelines/stages/jobs/nuget-validation-job.yml | 8 +++-----
.pipelines/stages/jobs/py-validation-job.yml | 8 +++-----
.../jobs/steps/utils/download-huggingface-model.yml | 6 ++----
3 files changed, 8 insertions(+), 14 deletions(-)
diff --git a/.pipelines/stages/jobs/nuget-validation-job.yml b/.pipelines/stages/jobs/nuget-validation-job.yml
index 9217f2a909..1b22a702b6 100644
--- a/.pipelines/stages/jobs/nuget-validation-job.yml
+++ b/.pipelines/stages/jobs/nuget-validation-job.yml
@@ -140,10 +140,9 @@ jobs:
- template: steps/utils/download-huggingface-model.yml
parameters:
- StepName: 'Download Phi3-mini Model from HuggingFace'
HuggingFaceRepo: 'microsoft/Phi-3-mini-4k-instruct-onnx'
- RepoFolder: $(prebuild_phi3_mini_model_folder)
LocalFolder: 'phi3-mini'
+ RepoFolder: $(prebuild_phi3_mini_model_folder)
WorkingDirectory: '$(Build.Repository.LocalPath)/examples/csharp/HelloPhi'
HuggingFaceToken: $(HF_TOKEN)
os: ${{ parameters.os }}
@@ -158,11 +157,10 @@ jobs:
- template: steps/utils/download-huggingface-model.yml
parameters:
- StepName: 'Download Phi-3.5-vision-instruct-onnx Model from HuggingFace'
HuggingFaceRepo: 'microsoft/Phi-3.5-vision-instruct-onnx'
- RepoFolder: $(prebuild_phi3_5_vision_model_folder)
LocalFolder: 'phi3.5-vision'
- WorkingDirectory: '$(Build.Repository.LocalPath)/examples/csharp/HelloPhi'
+ RepoFolder: $(prebuild_phi3_5_vision_model_folder)
+ WorkingDirectory: '$(Build.Repository.LocalPath)/examples/csharp/HelloPhi3V'
HuggingFaceToken: $(HF_TOKEN)
os: ${{ parameters.os }}
diff --git a/.pipelines/stages/jobs/py-validation-job.yml b/.pipelines/stages/jobs/py-validation-job.yml
index 6bb287d9e0..ab2d2a983e 100644
--- a/.pipelines/stages/jobs/py-validation-job.yml
+++ b/.pipelines/stages/jobs/py-validation-job.yml
@@ -157,10 +157,9 @@ jobs:
- template: steps/utils/download-huggingface-model.yml
parameters:
- StepName: 'Download Phi3-mini from HuggingFace'
HuggingFaceRepo: 'microsoft/Phi-3-mini-4k-instruct-onnx'
- RepoFolder: $(prebuild_phi3_mini_model_folder)
LocalFolder: 'phi3-mini'
+ RepoFolder: $(prebuild_phi3_mini_model_folder)
WorkingDirectory: '$(Build.Repository.LocalPath)/examples/python'
HuggingFaceToken: $(HF_TOKEN)
os: ${{ parameters.os }}
@@ -174,11 +173,10 @@ jobs:
- template: steps/utils/download-huggingface-model.yml
parameters:
- StepName: 'Download Phi-3.5-vision-instruct-onnx Model from HuggingFace'
HuggingFaceRepo: 'microsoft/Phi-3.5-vision-instruct-onnx'
- RepoFolder: $(prebuild_phi3_5_vision_model_folder)
LocalFolder: 'phi3.5-vision'
- WorkingDirectory: '$(Build.Repository.LocalPath)/examples/csharp/HelloPhi'
+ RepoFolder: $(prebuild_phi3_5_vision_model_folder)
+ WorkingDirectory: '$(Build.Repository.LocalPath)/examples/python'
HuggingFaceToken: $(HF_TOKEN)
os: ${{ parameters.os }}
diff --git a/.pipelines/stages/jobs/steps/utils/download-huggingface-model.yml b/.pipelines/stages/jobs/steps/utils/download-huggingface-model.yml
index 3bb0caa222..7f3d4dc14f 100644
--- a/.pipelines/stages/jobs/steps/utils/download-huggingface-model.yml
+++ b/.pipelines/stages/jobs/steps/utils/download-huggingface-model.yml
@@ -1,6 +1,4 @@
parameters:
- - name: StepName
- type: string
- name: WorkingDirectory
type: string
- name: HuggingFaceRepo
@@ -20,7 +18,7 @@ steps:
python -m pip install "huggingface_hub[cli]"
huggingface-cli login --token $HF_TOKEN
huggingface-cli download ${{ parameters.HuggingFaceRepo }} --include ${{ parameters.RepoFolder }}/* --local-dir ${{ parameters.LocalFolder }} --local-dir-use-symlinks False
- displayName: ${{ parameters.StepName }}
+ displayName: Download ${{ parameters.HuggingFaceRepo }} from HuggingFace
workingDirectory: ${{ parameters.WorkingDirectory }}
env:
HF_TOKEN: ${{ parameters.HuggingFaceToken }}
@@ -30,7 +28,7 @@ steps:
huggingface-cli login --token $env:HF_TOKEN
# Use maximum path length for Windows... otherwises hits the path character limit
huggingface-cli download ${{ parameters.HuggingFaceRepo }} --include ${{ parameters.RepoFolder }}/* --local-dir "\\?\${{ parameters.WorkingDirectory }}\\${{ parameters.LocalFolder }}" --local-dir-use-symlinks False
- displayName: ${{ parameters.StepName }}
+ displayName: Download ${{ parameters.HuggingFaceRepo }} from HuggingFace
workingDirectory: ${{ parameters.WorkingDirectory }}
env:
HF_TOKEN: ${{ parameters.HuggingFaceToken }}
From f40d8d6b5046ef4a333dc4345748d51135d3ecb0 Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Fri, 15 Nov 2024 12:54:51 +0800
Subject: [PATCH 21/42] Fix
---
.../steps/utils/perform-nuget-validation-with-model.yml | 6 +++---
.../steps/utils/perform-python-validation-with-model.yml | 4 ++--
2 files changed, 5 insertions(+), 5 deletions(-)
diff --git a/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml b/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
index a2f48383f1..e035a1c106 100644
--- a/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
+++ b/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
@@ -34,7 +34,7 @@ steps:
Copy-Item -Force -Recurse -Verbose $(Build.BinariesDirectory)/nuget/* -Destination ${{ parameters.CsprojFolder }}
cd ${{ parameters.CsprojFolder }}
dotnet restore -r $(os)-$(arch) /property:Configuration=${{ parameters.CsprojConfiguration }} --source https://api.nuget.org/v3/index.json --source https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/nuget/v3/index.json --source $PWD --disable-parallel --verbosity detailed
- dotnet run -r $(os)-$(arch) --configuration ${{ parameters.CsprojConfiguration }} --no-restore --verbosity normal -- -m ./{{ $parameters.LocalFolder }}/${{ parameters.ModelFolder }}
+ dotnet run -r $(os)-$(arch) --configuration ${{ parameters.CsprojConfiguration }} --no-restore --verbosity normal -- -m ./${{ parameters.LocalFolder }}/${{ parameters.ModelFolder }}
displayName: 'Run ${{ parameters.CsprojName }} With Artifact on Windows'
workingDirectory: '$(Build.Repository.LocalPath)'
condition: eq(variables['os'], 'win')
@@ -72,7 +72,7 @@ steps:
export ORTGENAI_LOG_ORT_LIB=1 && \
cd /ort_genai_src/${{ parameters.CsprojFolder }} && \
chmod +x ./bin/Release_Cuda/net6.0/linux-x64/${{ parameters.CsprojName }} && \
- ./bin/Release_Cuda/net6.0/linux-x64/${{ parameters.CsprojName }} -m ./{{ $parameters.LocalFolder }}/${{ parameters.ModelFolder }}"
+ ./bin/Release_Cuda/net6.0/linux-x64/${{ parameters.CsprojName }} -m ./${{ parameters.LocalFolder }}/${{ parameters.ModelFolder }}"
displayName: 'Run ${{ parameters.CsprojName }} With Artifact on Linux CUDA'
workingDirectory: '$(Build.Repository.LocalPath)'
@@ -81,7 +81,7 @@ steps:
- bash: |
export ORTGENAI_LOG_ORT_LIB=1
cd ${{ parameters.CsprojFolder }}
- dotnet run -r $(os)-$(arch) --configuration ${{ parameters.CsprojConfiguration }} --no-build --verbosity normal -- -m ./{{ $parameters.LocalFolder }}/${{ parameters.ModelFolder }}
+ dotnet run -r $(os)-$(arch) --configuration ${{ parameters.CsprojConfiguration }} --no-build --verbosity normal -- -m ./${{ parameters.LocalFolder }}/${{ parameters.ModelFolder }}
displayName: 'Run ${{ parameters.CsprojName }} With Artifact on Linux/macOS CPU'
workingDirectory: '$(Build.Repository.LocalPath)'
condition: and(or(eq(variables['os'], 'linux'), eq(variables['os'], 'osx')), eq(variables['ep'], 'cpu'))
diff --git a/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml b/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
index 5e23f9d25d..1c5e7b2aa8 100644
--- a/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
+++ b/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
@@ -68,7 +68,7 @@ steps:
$python_exe -m pip install -r /ort_genai_src/test/python/cuda/ort/requirements.txt && \
cd /ort_genai_src/${{ parameters.PythonScriptFolder }} && \
$python_exe -m pip install --no-index --find-links=/ort_genai_binary/wheel $(pip_package_name) && \
- $python_exe ${{ parameters.PythonScriptName }} -m ./{{ $parameters.LocalFolder }}/${{ parameters.ModelFolder }} --min_length 25 --max_length 50 --verbose"
+ $python_exe ${{ parameters.PythonScriptName }} -m ./${{ parameters.LocalFolder }}/${{ parameters.ModelFolder }} --min_length 25 --max_length 50 --verbose"
displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact on Linux CUDA'
workingDirectory: '$(Build.Repository.LocalPath)'
@@ -87,7 +87,7 @@ steps:
fi
cd ${{ parameters.PythonScriptFolder }}
python -m pip install --no-index --find-links=$(Build.BinariesDirectory)/wheel $(pip_package_name)
- python ${{ parameters.PythonScriptName }} -m ./{{ $parameters.LocalFolder }}/${{ parameters.ModelFolder }} --min_length 25 --max_length 50 --verbose
+ python ${{ parameters.PythonScriptName }} -m ./${{ parameters.LocalFolder }}/${{ parameters.ModelFolder }} --min_length 25 --max_length 50 --verbose
displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact on Linux/macOS CPU'
workingDirectory: '$(Build.Repository.LocalPath)'
condition: and(or(eq(variables['os'], 'linux'), eq(variables['os'], 'osx')), eq(variables['ep'], 'cpu'))
\ No newline at end of file
From b63e052fa73f17a069179cf529d15297d83493f7 Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Fri, 15 Nov 2024 14:00:20 +0800
Subject: [PATCH 22/42] Release resources
---
examples/csharp/HelloPhi3V/Program.cs | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/examples/csharp/HelloPhi3V/Program.cs b/examples/csharp/HelloPhi3V/Program.cs
index 782d30f81f..e63aad2058 100644
--- a/examples/csharp/HelloPhi3V/Program.cs
+++ b/examples/csharp/HelloPhi3V/Program.cs
@@ -106,7 +106,7 @@ void PrintUsage()
prompt += text + "<|end|>\n<|assistant|>\n";
Console.WriteLine("Processing image and prompt...");
- var inputTensors = processor.ProcessImages(prompt, images);
+ using var inputTensors = processor.ProcessImages(prompt, images);
Console.WriteLine("Generating response...");
using GeneratorParams generatorParams = new GeneratorParams(model);
@@ -120,4 +120,6 @@ void PrintUsage()
generator.GenerateNextToken();
Console.Write(tokenizerStream.Decode(generator.GetSequence(0)[^1]));
}
+
+ images.Dispose();
} while (interactive);
\ No newline at end of file
From 0776f00896fd3ac692b2120f5828e7fdcd0ecbb8 Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Fri, 15 Nov 2024 14:42:37 +0800
Subject: [PATCH 23/42] Provider
---
.../perform-python-validation-with-model.yml | 10 +++++++---
examples/csharp/HelloPhi3V/Program.cs | 17 +++++++++++++++--
2 files changed, 22 insertions(+), 5 deletions(-)
diff --git a/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml b/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
index 1c5e7b2aa8..8be0a8035e 100644
--- a/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
+++ b/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
@@ -42,7 +42,11 @@ steps:
cd ${{ parameters.PythonScriptFolder }}
python -m pip install --no-index --find-links=$(Build.BinariesDirectory)/wheel $(pip_package_name)
- python ${{ parameters.PythonScriptName }} -m .\models\${{ parameters.ModelFolder }} --min_length 25 --max_length 50 --verbose
+ if ("$(ep)" -eq "cuda") {
+ python ${{ parameters.PythonScriptName }} -m .\models\${{ parameters.ModelFolder }} --provider dml --min_length 25 --max_length 50 --verbose
+ } else {
+ python ${{ parameters.PythonScriptName }} -m .\models\${{ parameters.ModelFolder }} --provider $(ep) --min_length 25 --max_length 50 --verbose
+ }
displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact on Windows'
workingDirectory: '$(Build.Repository.LocalPath)'
condition: eq(variables['os'], 'win')
@@ -68,7 +72,7 @@ steps:
$python_exe -m pip install -r /ort_genai_src/test/python/cuda/ort/requirements.txt && \
cd /ort_genai_src/${{ parameters.PythonScriptFolder }} && \
$python_exe -m pip install --no-index --find-links=/ort_genai_binary/wheel $(pip_package_name) && \
- $python_exe ${{ parameters.PythonScriptName }} -m ./${{ parameters.LocalFolder }}/${{ parameters.ModelFolder }} --min_length 25 --max_length 50 --verbose"
+ $python_exe ${{ parameters.PythonScriptName }} -m ./${{ parameters.LocalFolder }}/${{ parameters.ModelFolder }} --provider $(ep) --min_length 25 --max_length 50 --verbose"
displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact on Linux CUDA'
workingDirectory: '$(Build.Repository.LocalPath)'
@@ -87,7 +91,7 @@ steps:
fi
cd ${{ parameters.PythonScriptFolder }}
python -m pip install --no-index --find-links=$(Build.BinariesDirectory)/wheel $(pip_package_name)
- python ${{ parameters.PythonScriptName }} -m ./${{ parameters.LocalFolder }}/${{ parameters.ModelFolder }} --min_length 25 --max_length 50 --verbose
+ python ${{ parameters.PythonScriptName }} -m ./${{ parameters.LocalFolder }}/${{ parameters.ModelFolder }} --provider $(ep) --min_length 25 --max_length 50 --verbose
displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact on Linux/macOS CPU'
workingDirectory: '$(Build.Repository.LocalPath)'
condition: and(or(eq(variables['os'], 'linux'), eq(variables['os'], 'osx')), eq(variables['ep'], 'cpu'))
\ No newline at end of file
diff --git a/examples/csharp/HelloPhi3V/Program.cs b/examples/csharp/HelloPhi3V/Program.cs
index e63aad2058..8c6cdd7cde 100644
--- a/examples/csharp/HelloPhi3V/Program.cs
+++ b/examples/csharp/HelloPhi3V/Program.cs
@@ -3,6 +3,7 @@
using Microsoft.ML.OnnxRuntimeGenAI;
using System.Linq;
+using System.Runtime.CompilerServices;
void PrintUsage()
{
@@ -50,6 +51,12 @@ void PrintUsage()
}
}
i_arg++;
+}
+
+// From https://stackoverflow.com/a/47841442
+static string GetThisFilePath([CallerFilePath] string path = null)
+{
+ return path;
}
Console.WriteLine("--------------------");
@@ -74,7 +81,9 @@ void PrintUsage()
Images images = null;
if (imagePaths.Length == 0)
{
- Console.WriteLine("No image provided");
+ Console.WriteLine("No image provided. Using default image.");
+ imagePaths.Append(Path.GetFullPath(Path.Combine(
+ GetThisFilePath(), "../../..", "test_models", "images", "australia.jpg")));
}
else
{
@@ -85,6 +94,7 @@ void PrintUsage()
{
throw new Exception("Image file not found: " + imagePath);
}
+ Console.WriteLine("Using image: " + imagePath);
}
images = Images.Load(imagePaths);
}
@@ -121,5 +131,8 @@ void PrintUsage()
Console.Write(tokenizerStream.Decode(generator.GetSequence(0)[^1]));
}
- images.Dispose();
+ if (images != null)
+ {
+ images.Dispose();
+ }
} while (interactive);
\ No newline at end of file
From 61fcadadc73f247a68e6210b229cdb9f23882866 Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Fri, 15 Nov 2024 15:17:17 +0800
Subject: [PATCH 24/42] Fix
---
.../perform-python-validation-with-model.yml | 8 ++---
examples/csharp/HelloPhi3V/Program.cs | 29 +++++++++----------
examples/python/model-generate.py | 22 +++++++++-----
3 files changed, 32 insertions(+), 27 deletions(-)
diff --git a/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml b/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
index 8be0a8035e..cc8a443a6a 100644
--- a/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
+++ b/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
@@ -43,9 +43,9 @@ steps:
python -m pip install --no-index --find-links=$(Build.BinariesDirectory)/wheel $(pip_package_name)
if ("$(ep)" -eq "cuda") {
- python ${{ parameters.PythonScriptName }} -m .\models\${{ parameters.ModelFolder }} --provider dml --min_length 25 --max_length 50 --verbose
+ python ${{ parameters.PythonScriptName }} -m .\models\${{ parameters.ModelFolder }} --provider dml
} else {
- python ${{ parameters.PythonScriptName }} -m .\models\${{ parameters.ModelFolder }} --provider $(ep) --min_length 25 --max_length 50 --verbose
+ python ${{ parameters.PythonScriptName }} -m .\models\${{ parameters.ModelFolder }} --provider $(ep)
}
displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact on Windows'
workingDirectory: '$(Build.Repository.LocalPath)'
@@ -72,7 +72,7 @@ steps:
$python_exe -m pip install -r /ort_genai_src/test/python/cuda/ort/requirements.txt && \
cd /ort_genai_src/${{ parameters.PythonScriptFolder }} && \
$python_exe -m pip install --no-index --find-links=/ort_genai_binary/wheel $(pip_package_name) && \
- $python_exe ${{ parameters.PythonScriptName }} -m ./${{ parameters.LocalFolder }}/${{ parameters.ModelFolder }} --provider $(ep) --min_length 25 --max_length 50 --verbose"
+ $python_exe ${{ parameters.PythonScriptName }} -m ./${{ parameters.LocalFolder }}/${{ parameters.ModelFolder }} --provider $(ep)"
displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact on Linux CUDA'
workingDirectory: '$(Build.Repository.LocalPath)'
@@ -91,7 +91,7 @@ steps:
fi
cd ${{ parameters.PythonScriptFolder }}
python -m pip install --no-index --find-links=$(Build.BinariesDirectory)/wheel $(pip_package_name)
- python ${{ parameters.PythonScriptName }} -m ./${{ parameters.LocalFolder }}/${{ parameters.ModelFolder }} --provider $(ep) --min_length 25 --max_length 50 --verbose
+ python ${{ parameters.PythonScriptName }} -m ./${{ parameters.LocalFolder }}/${{ parameters.ModelFolder }} --provider $(ep)
displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact on Linux/macOS CPU'
workingDirectory: '$(Build.Repository.LocalPath)'
condition: and(or(eq(variables['os'], 'linux'), eq(variables['os'], 'osx')), eq(variables['ep'], 'cpu'))
\ No newline at end of file
diff --git a/examples/csharp/HelloPhi3V/Program.cs b/examples/csharp/HelloPhi3V/Program.cs
index 8c6cdd7cde..5f9c3ff820 100644
--- a/examples/csharp/HelloPhi3V/Program.cs
+++ b/examples/csharp/HelloPhi3V/Program.cs
@@ -51,12 +51,12 @@ void PrintUsage()
}
}
i_arg++;
-}
-
-// From https://stackoverflow.com/a/47841442
-static string GetThisFilePath([CallerFilePath] string path = null)
-{
- return path;
+}
+
+// From https://stackoverflow.com/a/47841442
+static string GetThisFilePath([CallerFilePath] string path = null)
+{
+ return path;
}
Console.WriteLine("--------------------");
@@ -78,27 +78,24 @@ static string GetThisFilePath([CallerFilePath] string path = null)
imagePaths = Console.ReadLine().Split(',').ToList().Select(i => i.ToString().Trim()).ToArray();
}
- Images images = null;
if (imagePaths.Length == 0)
{
Console.WriteLine("No image provided. Using default image.");
imagePaths.Append(Path.GetFullPath(Path.Combine(
GetThisFilePath(), "../../..", "test_models", "images", "australia.jpg")));
}
- else
+ for (int i = 0; i < imagePaths.Length; i++)
{
- for (int i = 0; i < imagePaths.Length; i++)
+ string imagePath = Path.GetFullPath(imagePaths[i].Trim());
+ if (!File.Exists(imagePath))
{
- string imagePath = imagePaths[i].Trim();
- if (!File.Exists(imagePath))
- {
- throw new Exception("Image file not found: " + imagePath);
- }
- Console.WriteLine("Using image: " + imagePath);
+ throw new Exception("Image file not found: " + imagePath);
}
- images = Images.Load(imagePaths);
+ Console.WriteLine("Using image: " + imagePath);
}
+ Images images = imagePaths.Length > 0 ? Images.Load(imagePaths) : null;
+
string text = "What is shown in this image?";
if (interactive) {
Console.WriteLine("Prompt:");
diff --git a/examples/python/model-generate.py b/examples/python/model-generate.py
index 0a97f25b4d..a74848a2b5 100644
--- a/examples/python/model-generate.py
+++ b/examples/python/model-generate.py
@@ -4,7 +4,14 @@
def main(args):
if args.verbose: print("Loading model...")
- model = og.Model(f'{args.model}')
+ config = og.Config(args.model_path)
+ config.clear_providers()
+ if args.provider != "cpu":
+ if args.verbose:
+ print(f"Setting model to {args.provider}...")
+ config.append_provider(args.provider)
+ model = og.Model(config)
+
if args.verbose: print("Model loaded")
tokenizer = og.Tokenizer(model)
if args.verbose: print("Tokenizer created")
@@ -15,19 +22,19 @@ def main(args):
prompts = ["I like walking my cute dog",
"What is the best restaurant in town?",
"Hello, how are you today?"]
-
+
if args.chat_template:
if args.chat_template.count('{') != 1 or args.chat_template.count('}') != 1:
print("Error, chat template must have exactly one pair of curly braces, e.g. '<|user|>\n{input} <|end|>\n<|assistant|>'")
exit(1)
prompts[:] = [f'{args.chat_template.format(input=text)}' for text in prompts]
-
+
input_tokens = tokenizer.encode_batch(prompts)
if args.verbose: print(f'Prompt(s) encoded: {prompts}')
params = og.GeneratorParams(model)
- search_options = {name:getattr(args, name) for name in ['do_sample', 'max_length', 'min_length', 'top_p', 'top_k', 'temperature', 'repetition_penalty'] if name in args}
+ search_options = {name:getattr(args, name) for name in ['do_sample', 'max_length', 'min_length', 'top_p', 'top_k', 'temperature', 'repetition_penalty'] if name in args}
if (args.verbose): print(f'Args: {args}')
if (args.verbose): print(f'Search options: {search_options}')
@@ -59,11 +66,12 @@ def main(args):
if __name__ == "__main__":
parser = argparse.ArgumentParser(argument_default=argparse.SUPPRESS, description="End-to-end token generation loop example for gen-ai")
parser.add_argument('-m', '--model', type=str, required=True, help='Onnx model folder path (must contain config.json and model.onnx)')
+ parser.add_argument("-p", "--provider", type=str, required=True, help="Provider to run model")
parser.add_argument('-pr', '--prompts', nargs='*', required=False, help='Input prompts to generate tokens from. Provide this parameter multiple times to batch multiple prompts')
- parser.add_argument('-i', '--min_length', type=int, help='Min number of tokens to generate including the prompt')
- parser.add_argument('-l', '--max_length', type=int, help='Max number of tokens to generate including the prompt')
+ parser.add_argument('-i', '--min_length', type=int, default=25, help='Min number of tokens to generate including the prompt')
+ parser.add_argument('-l', '--max_length', type=int, default=50, help='Max number of tokens to generate including the prompt')
parser.add_argument('-ds', '--do_random_sampling', action='store_true', help='Do random sampling. When false, greedy or beam search are used to generate the output. Defaults to false')
- parser.add_argument('-p', '--top_p', type=float, help='Top p probability to sample with')
+ parser.add_argument('--top_p', type=float, help='Top p probability to sample with')
parser.add_argument('-k', '--top_k', type=int, help='Top k tokens to sample from')
parser.add_argument('-t', '--temperature', type=float, help='Temperature to sample with')
parser.add_argument('-r', '--repetition_penalty', type=float, help='Repetition penalty to sample with')
From 9f411159342ee5edfa8c9518868f11727f8d2f78 Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Fri, 15 Nov 2024 15:50:07 +0800
Subject: [PATCH 25/42] More fixes
---
.../steps/utils/perform-python-validation-with-model.yml | 4 ++--
examples/csharp/HelloPhi3V/Program.cs | 8 ++++----
examples/python/model-generate.py | 2 +-
examples/python/phi3v.py | 2 +-
4 files changed, 8 insertions(+), 8 deletions(-)
diff --git a/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml b/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
index cc8a443a6a..103743398f 100644
--- a/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
+++ b/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
@@ -43,9 +43,9 @@ steps:
python -m pip install --no-index --find-links=$(Build.BinariesDirectory)/wheel $(pip_package_name)
if ("$(ep)" -eq "cuda") {
- python ${{ parameters.PythonScriptName }} -m .\models\${{ parameters.ModelFolder }} --provider dml
+ python ${{ parameters.PythonScriptName }} -m .\${{ parameters.LocalFolder }}\${{ parameters.ModelFolder }} --provider dml
} else {
- python ${{ parameters.PythonScriptName }} -m .\models\${{ parameters.ModelFolder }} --provider $(ep)
+ python ${{ parameters.PythonScriptName }} -m .\${{ parameters.LocalFolder }}\${{ parameters.ModelFolder }} --provider $(ep)
}
displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact on Windows'
workingDirectory: '$(Build.Repository.LocalPath)'
diff --git a/examples/csharp/HelloPhi3V/Program.cs b/examples/csharp/HelloPhi3V/Program.cs
index 5f9c3ff820..6c77bceecc 100644
--- a/examples/csharp/HelloPhi3V/Program.cs
+++ b/examples/csharp/HelloPhi3V/Program.cs
@@ -26,7 +26,7 @@ void PrintUsage()
bool interactive = false;
string modelPath = string.Empty;
-string[] imagePaths = new string[0];
+List imagePaths = new List();
uint i_arg = 0;
while (i_arg < args.Length)
@@ -78,13 +78,13 @@ static string GetThisFilePath([CallerFilePath] string path = null)
imagePaths = Console.ReadLine().Split(',').ToList().Select(i => i.ToString().Trim()).ToArray();
}
- if (imagePaths.Length == 0)
+ if (imagePaths.Count == 0)
{
Console.WriteLine("No image provided. Using default image.");
- imagePaths.Append(Path.GetFullPath(Path.Combine(
+ imagePaths.Add(Path.GetFullPath(Path.Combine(
GetThisFilePath(), "../../..", "test_models", "images", "australia.jpg")));
}
- for (int i = 0; i < imagePaths.Length; i++)
+ for (int i = 0; i < imagePaths.Count; i++)
{
string imagePath = Path.GetFullPath(imagePaths[i].Trim());
if (!File.Exists(imagePath))
diff --git a/examples/python/model-generate.py b/examples/python/model-generate.py
index a74848a2b5..e98d84dd9a 100644
--- a/examples/python/model-generate.py
+++ b/examples/python/model-generate.py
@@ -65,7 +65,7 @@ def main(args):
if __name__ == "__main__":
parser = argparse.ArgumentParser(argument_default=argparse.SUPPRESS, description="End-to-end token generation loop example for gen-ai")
- parser.add_argument('-m', '--model', type=str, required=True, help='Onnx model folder path (must contain config.json and model.onnx)')
+ parser.add_argument('-m', '--model_path', type=str, required=True, help='Onnx model folder path (must contain config.json and model.onnx)')
parser.add_argument("-p", "--provider", type=str, required=True, help="Provider to run model")
parser.add_argument('-pr', '--prompts', nargs='*', required=False, help='Input prompts to generate tokens from. Provide this parameter multiple times to batch multiple prompts')
parser.add_argument('-i', '--min_length', type=int, default=25, help='Min number of tokens to generate including the prompt')
diff --git a/examples/python/phi3v.py b/examples/python/phi3v.py
index 1c91b12e2a..1f59e6af5e 100644
--- a/examples/python/phi3v.py
+++ b/examples/python/phi3v.py
@@ -54,10 +54,10 @@ def run(args: argparse.Namespace):
if len(image_paths) == 0:
print("No image provided")
else:
- print(f"Loading images: {image_paths}")
for i, image_path in enumerate(image_paths):
if not os.path.exists(image_path):
raise FileNotFoundError(f"Image file not found: {image_path}")
+ print(f"Using image: {image_path}")
prompt += f"<|image_{i+1}|>\n"
images = og.Images.open(*image_paths)
From 311ed3f9c90167b5cf73701417f99fbe45a4c519 Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Fri, 15 Nov 2024 16:13:06 +0800
Subject: [PATCH 26/42] Cleanup
---
examples/python/phi3v.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/examples/python/phi3v.py b/examples/python/phi3v.py
index 1f59e6af5e..c134a1aa04 100644
--- a/examples/python/phi3v.py
+++ b/examples/python/phi3v.py
@@ -47,7 +47,6 @@ def run(args: argparse.Namespace):
image_paths = [str(REPO_ROOT / "test" / "test_models" / "images" / "australia.jpg")]
image_paths = [image_path for image_path in image_paths]
- print(image_paths)
images = None
prompt = "<|user|>\n"
From aa6e060e71d848e972a5ffcd2aa2fdfaa27fae71 Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Fri, 15 Nov 2024 16:29:04 +0800
Subject: [PATCH 27/42] Fix C# build
---
.../steps/utils/perform-nuget-validation-with-model.yml | 3 ++-
examples/csharp/HelloPhi3V/Program.cs | 8 ++++----
2 files changed, 6 insertions(+), 5 deletions(-)
diff --git a/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml b/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
index e035a1c106..0769f4ead4 100644
--- a/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
+++ b/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
@@ -43,11 +43,12 @@ steps:
NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS: 180
- bash: |
+ set -e -x
dotnet --info
cp $(Build.BinariesDirectory)/nuget/* ${{ parameters.CsprojFolder }}
cd ${{ parameters.CsprojFolder }}
dotnet restore -r $(os)-$(arch) /property:Configuration=${{ parameters.CsprojConfiguration }} --source https://api.nuget.org/v3/index.json --source https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/nuget/v3/index.json --source $PWD --disable-parallel --verbosity detailed
- dotnet build ./${{ parameters.CsprojName }}.csproj -r $(os)-$(arch) /property:Configuration=${{ parameters.CsprojConfiguration }} --no-restore --self-contained
+ dotnet build ./${{ parameters.CsprojName }}.csproj -r $(os)-$(arch) /property:Configuration=${{ parameters.CsprojConfiguration }} --no-restore --self-contained --verbosity normal
ls -l ./bin/${{ parameters.CsprojConfiguration }}/net6.0/$(os)-$(arch)/
displayName: 'Perform dotnet restore & build'
workingDirectory: '$(Build.Repository.LocalPath)'
diff --git a/examples/csharp/HelloPhi3V/Program.cs b/examples/csharp/HelloPhi3V/Program.cs
index 6c77bceecc..1c5b81f612 100644
--- a/examples/csharp/HelloPhi3V/Program.cs
+++ b/examples/csharp/HelloPhi3V/Program.cs
@@ -47,7 +47,7 @@ void PrintUsage()
{
if (i_arg + 1 < args.Length)
{
- imagePaths = args[i_arg+1].Split(',').ToList().Select(i => i.ToString().Trim()).ToArray();
+ imagePaths = args[i_arg + 1].Split(',').ToList().Select(i => i.ToString().Trim()).ToList();
}
}
i_arg++;
@@ -75,7 +75,7 @@ static string GetThisFilePath([CallerFilePath] string path = null)
if (interactive)
{
Console.WriteLine("Image Path (comma separated; leave empty if no image):");
- imagePaths = Console.ReadLine().Split(',').ToList().Select(i => i.ToString().Trim()).ToArray();
+ imagePaths = Console.ReadLine().Split(',').ToList().Select(i => i.ToString().Trim()).ToList();
}
if (imagePaths.Count == 0)
@@ -94,7 +94,7 @@ static string GetThisFilePath([CallerFilePath] string path = null)
Console.WriteLine("Using image: " + imagePath);
}
- Images images = imagePaths.Length > 0 ? Images.Load(imagePaths) : null;
+ Images images = imagePaths.Count > 0 ? Images.Load(imagePaths.ToArray()) : null;
string text = "What is shown in this image?";
if (interactive) {
@@ -105,7 +105,7 @@ static string GetThisFilePath([CallerFilePath] string path = null)
string prompt = "<|user|>\n";
if (images != null)
{
- for (int i = 0; i < imagePaths.Length; i++)
+ for (int i = 0; i < imagePaths.Count; i++)
{
prompt += "<|image_" + (i + 1) + "|>\n";
}
From efb37b6f90acee72edd71133428fe4be2bdc8cdb Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Fri, 15 Nov 2024 16:59:41 +0800
Subject: [PATCH 28/42] [skip ci] Fix C# build
---
examples/csharp/HelloPhi3V/Program.cs | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/examples/csharp/HelloPhi3V/Program.cs b/examples/csharp/HelloPhi3V/Program.cs
index 1c5b81f612..da27cbaf91 100644
--- a/examples/csharp/HelloPhi3V/Program.cs
+++ b/examples/csharp/HelloPhi3V/Program.cs
@@ -5,6 +5,12 @@
using System.Linq;
using System.Runtime.CompilerServices;
+// From https://stackoverflow.com/a/47841442
+static string GetThisFilePath([CallerFilePath] string path = null)
+{
+ return path;
+}
+
void PrintUsage()
{
Console.WriteLine("Usage:");
@@ -53,12 +59,6 @@ void PrintUsage()
i_arg++;
}
-// From https://stackoverflow.com/a/47841442
-static string GetThisFilePath([CallerFilePath] string path = null)
-{
- return path;
-}
-
Console.WriteLine("--------------------");
Console.WriteLine("Hello, Phi-3-Vision!");
Console.WriteLine("--------------------");
@@ -82,7 +82,7 @@ static string GetThisFilePath([CallerFilePath] string path = null)
{
Console.WriteLine("No image provided. Using default image.");
imagePaths.Add(Path.GetFullPath(Path.Combine(
- GetThisFilePath(), "../../..", "test_models", "images", "australia.jpg")));
+ GetThisFilePath(), "../../../..", "test", "test_models", "images", "australia.jpg")));
}
for (int i = 0; i < imagePaths.Count; i++)
{
From ce88751c3e72d279edf4aa4b330626928849565d Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Fri, 15 Nov 2024 17:21:48 +0800
Subject: [PATCH 29/42] [skip ci] Fix python build
---
.../steps/utils/perform-python-validation-with-model.yml | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
diff --git a/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml b/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
index 103743398f..63d22c7aab 100644
--- a/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
+++ b/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
@@ -23,6 +23,7 @@ steps:
workingDirectory: '$(Build.Repository.LocalPath)'
condition: and(eq(variables['os'], 'win'), eq(variables['ep'], 'cuda'))
- powershell: |
+ python -m pip install readline
python -m pip install -r test/python/requirements.txt
if ("$(ep)" -eq "cuda") {
$env:CUDA_PATH = '$(Build.Repository.LocalPath)\cuda_sdk\v$(cuda_version)'
@@ -42,7 +43,7 @@ steps:
cd ${{ parameters.PythonScriptFolder }}
python -m pip install --no-index --find-links=$(Build.BinariesDirectory)/wheel $(pip_package_name)
- if ("$(ep)" -eq "cuda") {
+ if ("$(ep)" -eq "directml") {
python ${{ parameters.PythonScriptName }} -m .\${{ parameters.LocalFolder }}\${{ parameters.ModelFolder }} --provider dml
} else {
python ${{ parameters.PythonScriptName }} -m .\${{ parameters.LocalFolder }}\${{ parameters.ModelFolder }} --provider $(ep)
@@ -67,6 +68,7 @@ steps:
-w /ort_genai_src/ $(cuda_docker_image) \
bash -c " \
export ORTGENAI_LOG_ORT_LIB=1 && \
+ $python_exe -m pip install readline && \
$python_exe -m pip install -r /ort_genai_src/test/python/requirements.txt && \
$python_exe -m pip install -r /ort_genai_src/test/python/cuda/torch/requirements.txt && \
$python_exe -m pip install -r /ort_genai_src/test/python/cuda/ort/requirements.txt && \
@@ -80,6 +82,7 @@ steps:
- bash: |
export ORTGENAI_LOG_ORT_LIB=1
+ python -m pip install readline
python -m pip install -r test/python/requirements.txt
if [[ "$(os)" == "linux" ]]; then
python -m pip install -r test/python/cpu/torch/requirements.txt
From b0526f42614bc0d1fba227542c52ff7f88ee0c43 Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Fri, 15 Nov 2024 19:48:06 +0800
Subject: [PATCH 30/42] [skip ci] readline not available
---
.../utils/perform-python-validation-with-model.yml | 3 ---
examples/python/phi3v.py | 11 +++++++----
2 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml b/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
index 63d22c7aab..f607a626f6 100644
--- a/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
+++ b/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
@@ -23,7 +23,6 @@ steps:
workingDirectory: '$(Build.Repository.LocalPath)'
condition: and(eq(variables['os'], 'win'), eq(variables['ep'], 'cuda'))
- powershell: |
- python -m pip install readline
python -m pip install -r test/python/requirements.txt
if ("$(ep)" -eq "cuda") {
$env:CUDA_PATH = '$(Build.Repository.LocalPath)\cuda_sdk\v$(cuda_version)'
@@ -68,7 +67,6 @@ steps:
-w /ort_genai_src/ $(cuda_docker_image) \
bash -c " \
export ORTGENAI_LOG_ORT_LIB=1 && \
- $python_exe -m pip install readline && \
$python_exe -m pip install -r /ort_genai_src/test/python/requirements.txt && \
$python_exe -m pip install -r /ort_genai_src/test/python/cuda/torch/requirements.txt && \
$python_exe -m pip install -r /ort_genai_src/test/python/cuda/ort/requirements.txt && \
@@ -82,7 +80,6 @@ steps:
- bash: |
export ORTGENAI_LOG_ORT_LIB=1
- python -m pip install readline
python -m pip install -r test/python/requirements.txt
if [[ "$(os)" == "linux" ]]; then
python -m pip install -r test/python/cpu/torch/requirements.txt
diff --git a/examples/python/phi3v.py b/examples/python/phi3v.py
index c134a1aa04..514207ce9f 100644
--- a/examples/python/phi3v.py
+++ b/examples/python/phi3v.py
@@ -3,7 +3,6 @@
import argparse
import os
-import readline
import glob
from pathlib import Path
@@ -30,10 +29,14 @@ def run(args: argparse.Namespace):
interactive = args.interactive
while True:
- readline.set_completer_delims(" \t\n;")
- readline.parse_and_bind("tab: complete")
- readline.set_completer(_complete)
if interactive:
+ try:
+ import readline
+ readline.set_completer_delims(" \t\n;")
+ readline.parse_and_bind("tab: complete")
+ readline.set_completer(_complete)
+ except ImportError:
+ pass
image_paths = [
image_path.strip()
for image_path in input(
From d9db677ce5ac4d8d23257c1402ada1b581609913 Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Mon, 18 Nov 2024 13:39:58 +0800
Subject: [PATCH 31/42] Make image paths docker friendly
---
examples/csharp/HelloPhi3V/Program.cs | 27 +++++++++++++++++++++-----
examples/python/phi3v.py | 12 ++++++++++--
test/csharp/TestOnnxRuntimeGenAIAPI.cs | 27 +++++++++++++++++++++-----
3 files changed, 54 insertions(+), 12 deletions(-)
diff --git a/examples/csharp/HelloPhi3V/Program.cs b/examples/csharp/HelloPhi3V/Program.cs
index da27cbaf91..3c383650b1 100644
--- a/examples/csharp/HelloPhi3V/Program.cs
+++ b/examples/csharp/HelloPhi3V/Program.cs
@@ -5,10 +5,27 @@
using System.Linq;
using System.Runtime.CompilerServices;
-// From https://stackoverflow.com/a/47841442
-static string GetThisFilePath([CallerFilePath] string path = null)
+static string GetDirectoryInTreeThatContains(string currentDirectory, string targetDirectoryName)
{
- return path;
+ bool found = false;
+ foreach (string d in Directory.GetDirectories(currentDirectory, searchPattern: targetDirectoryName))
+ {
+ found = true;
+ return Path.Combine(currentDirectory, targetDirectoryName);
+ }
+ if (!found)
+ {
+ DirectoryInfo dirInfo = new DirectoryInfo(currentDirectory);
+ if (dirInfo.Parent != null)
+ {
+ return GetDirectoryInTreeThatContains(Path.GetFullPath(Path.Combine(currentDirectory, "..")), targetDirectoryName);
+ }
+ else
+ {
+ return null;
+ }
+ }
+ return null;
}
void PrintUsage()
@@ -81,8 +98,8 @@ void PrintUsage()
if (imagePaths.Count == 0)
{
Console.WriteLine("No image provided. Using default image.");
- imagePaths.Add(Path.GetFullPath(Path.Combine(
- GetThisFilePath(), "../../../..", "test", "test_models", "images", "australia.jpg")));
+ imagePaths.Add(Path.Combine(
+ GetDirectoryInTreeThatContains(Directory.GetCurrentDirectory(), "test"), "test_models", "images", "australia.jpg"));
}
for (int i = 0; i < imagePaths.Count; i++)
{
diff --git a/examples/python/phi3v.py b/examples/python/phi3v.py
index 514207ce9f..22464c6d3f 100644
--- a/examples/python/phi3v.py
+++ b/examples/python/phi3v.py
@@ -8,7 +8,15 @@
import onnxruntime_genai as og
-REPO_ROOT = Path(__file__).parents[2]
+
+def _find_dir_contains_sub_dir(current_dir: Path, target_dir_name):
+ curr_path = Path(current_dir).absolute()
+ target_dir = glob.glob(str(curr_path / target_dir_name))
+ if target_dir:
+ return Path(target_dir[0]).absolute()
+ else:
+ return _find_dir_contains_sub_dir(curr_path / '..', target_dir_name)
+
def _complete(text, state):
return (glob.glob(text + "*") + [None])[state]
@@ -47,7 +55,7 @@ def run(args: argparse.Namespace):
if args.image_paths:
image_paths = args.image_paths
else:
- image_paths = [str(REPO_ROOT / "test" / "test_models" / "images" / "australia.jpg")]
+ image_paths = [str(_find_dir_contains_sub_dir(Path(__file__), "test") / "test_models" / "images" / "australia.jpg")]
image_paths = [image_path for image_path in image_paths]
diff --git a/test/csharp/TestOnnxRuntimeGenAIAPI.cs b/test/csharp/TestOnnxRuntimeGenAIAPI.cs
index efc8e78351..3ea5fd6635 100644
--- a/test/csharp/TestOnnxRuntimeGenAIAPI.cs
+++ b/test/csharp/TestOnnxRuntimeGenAIAPI.cs
@@ -15,14 +15,31 @@ public class OnnxRuntimeGenAITests
{
private readonly ITestOutputHelper output;
- // From https://stackoverflow.com/a/47841442
- private static string GetThisFilePath([CallerFilePath] string path = null)
+ private static string GetDirectoryInTreeThatContains(string currentDirectory, string targetDirectoryName)
{
- return path;
+ bool found = false;
+ foreach (string d in Directory.GetDirectories(currentDirectory, searchPattern: targetDirectoryName))
+ {
+ found = true;
+ return Path.Combine(currentDirectory, targetDirectoryName);
+ }
+ if (!found)
+ {
+ DirectoryInfo dirInfo = new DirectoryInfo(currentDirectory);
+ if (dirInfo.Parent != null)
+ {
+ return GetDirectoryInTreeThatContains(Path.GetFullPath(Path.Combine(currentDirectory, "..")), targetDirectoryName);
+ }
+ else
+ {
+ return null;
+ }
+ }
+ return null;
}
- private static readonly string _phi2Path = Path.GetFullPath(Path.Combine(
- GetThisFilePath(),"../..", "test_models", "phi-2", "int4", "cpu"));
+ private static readonly string _phi2Path = Path.Combine(
+ GetDirectoryInTreeThatContains(Directory.GetCurrentDirectory(), "test"), "test_models", "phi-2", "int4", "cpu");
public OnnxRuntimeGenAITests(ITestOutputHelper o)
{
From e7762c691b4729db7d6ff7fe0877578bdbfb11d5 Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Mon, 18 Nov 2024 16:46:09 +0800
Subject: [PATCH 32/42] [skip ci] check root
---
examples/python/phi3v.py | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/examples/python/phi3v.py b/examples/python/phi3v.py
index 22464c6d3f..d40a8b462a 100644
--- a/examples/python/phi3v.py
+++ b/examples/python/phi3v.py
@@ -15,6 +15,9 @@ def _find_dir_contains_sub_dir(current_dir: Path, target_dir_name):
if target_dir:
return Path(target_dir[0]).absolute()
else:
+ if curr_path.parent == curr_path:
+ # Root dir
+ return None
return _find_dir_contains_sub_dir(curr_path / '..', target_dir_name)
@@ -44,6 +47,7 @@ def run(args: argparse.Namespace):
readline.parse_and_bind("tab: complete")
readline.set_completer(_complete)
except ImportError:
+ # Not available on some platforms. Ignore it.
pass
image_paths = [
image_path.strip()
From 050549f6989b19ae36318fcc2284aaefeef8ee3c Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Mon, 18 Nov 2024 16:55:40 +0800
Subject: [PATCH 33/42] [skip ci] fix glob on linux
---
examples/python/phi3v.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/examples/python/phi3v.py b/examples/python/phi3v.py
index d40a8b462a..5ec7419036 100644
--- a/examples/python/phi3v.py
+++ b/examples/python/phi3v.py
@@ -11,7 +11,7 @@
def _find_dir_contains_sub_dir(current_dir: Path, target_dir_name):
curr_path = Path(current_dir).absolute()
- target_dir = glob.glob(str(curr_path / target_dir_name))
+ target_dir = glob.glob(target_dir_name, root_dir=curr_path)
if target_dir:
return Path(target_dir[0]).absolute()
else:
@@ -59,7 +59,7 @@ def run(args: argparse.Namespace):
if args.image_paths:
image_paths = args.image_paths
else:
- image_paths = [str(_find_dir_contains_sub_dir(Path(__file__), "test") / "test_models" / "images" / "australia.jpg")]
+ image_paths = [str(_find_dir_contains_sub_dir(Path(__file__).parent, "test") / "test_models" / "images" / "australia.jpg")]
image_paths = [image_path for image_path in image_paths]
From 9571f88337486fb77a847ee0d8eecef147a29a5d Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Mon, 18 Nov 2024 17:28:32 +0800
Subject: [PATCH 34/42] [skip ci] reverse interactive
---
.../steps/utils/perform-nuget-validation-with-model.yml | 6 +++---
.../steps/utils/perform-python-validation-with-model.yml | 8 ++++----
examples/csharp/HelloPhi/Program.cs | 8 ++++----
examples/csharp/HelloPhi3V/Program.cs | 8 ++++----
examples/python/phi3v.py | 6 +++---
5 files changed, 18 insertions(+), 18 deletions(-)
diff --git a/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml b/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
index 0769f4ead4..788a367328 100644
--- a/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
+++ b/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
@@ -34,7 +34,7 @@ steps:
Copy-Item -Force -Recurse -Verbose $(Build.BinariesDirectory)/nuget/* -Destination ${{ parameters.CsprojFolder }}
cd ${{ parameters.CsprojFolder }}
dotnet restore -r $(os)-$(arch) /property:Configuration=${{ parameters.CsprojConfiguration }} --source https://api.nuget.org/v3/index.json --source https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/ORT-Nightly/nuget/v3/index.json --source $PWD --disable-parallel --verbosity detailed
- dotnet run -r $(os)-$(arch) --configuration ${{ parameters.CsprojConfiguration }} --no-restore --verbosity normal -- -m ./${{ parameters.LocalFolder }}/${{ parameters.ModelFolder }}
+ dotnet run -r $(os)-$(arch) --configuration ${{ parameters.CsprojConfiguration }} --no-restore --verbosity normal -- -m ./${{ parameters.LocalFolder }}/${{ parameters.ModelFolder }} --non-interactive
displayName: 'Run ${{ parameters.CsprojName }} With Artifact on Windows'
workingDirectory: '$(Build.Repository.LocalPath)'
condition: eq(variables['os'], 'win')
@@ -73,7 +73,7 @@ steps:
export ORTGENAI_LOG_ORT_LIB=1 && \
cd /ort_genai_src/${{ parameters.CsprojFolder }} && \
chmod +x ./bin/Release_Cuda/net6.0/linux-x64/${{ parameters.CsprojName }} && \
- ./bin/Release_Cuda/net6.0/linux-x64/${{ parameters.CsprojName }} -m ./${{ parameters.LocalFolder }}/${{ parameters.ModelFolder }}"
+ ./bin/Release_Cuda/net6.0/linux-x64/${{ parameters.CsprojName }} -m ./${{ parameters.LocalFolder }}/${{ parameters.ModelFolder }} --non-interactive"
displayName: 'Run ${{ parameters.CsprojName }} With Artifact on Linux CUDA'
workingDirectory: '$(Build.Repository.LocalPath)'
@@ -82,7 +82,7 @@ steps:
- bash: |
export ORTGENAI_LOG_ORT_LIB=1
cd ${{ parameters.CsprojFolder }}
- dotnet run -r $(os)-$(arch) --configuration ${{ parameters.CsprojConfiguration }} --no-build --verbosity normal -- -m ./${{ parameters.LocalFolder }}/${{ parameters.ModelFolder }}
+ dotnet run -r $(os)-$(arch) --configuration ${{ parameters.CsprojConfiguration }} --no-build --verbosity normal -- -m ./${{ parameters.LocalFolder }}/${{ parameters.ModelFolder }} --non-interactive
displayName: 'Run ${{ parameters.CsprojName }} With Artifact on Linux/macOS CPU'
workingDirectory: '$(Build.Repository.LocalPath)'
condition: and(or(eq(variables['os'], 'linux'), eq(variables['os'], 'osx')), eq(variables['ep'], 'cpu'))
diff --git a/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml b/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
index f607a626f6..dc964b718e 100644
--- a/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
+++ b/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
@@ -43,9 +43,9 @@ steps:
python -m pip install --no-index --find-links=$(Build.BinariesDirectory)/wheel $(pip_package_name)
if ("$(ep)" -eq "directml") {
- python ${{ parameters.PythonScriptName }} -m .\${{ parameters.LocalFolder }}\${{ parameters.ModelFolder }} --provider dml
+ python ${{ parameters.PythonScriptName }} -m .\${{ parameters.LocalFolder }}\${{ parameters.ModelFolder }} --provider dml --non-interactive
} else {
- python ${{ parameters.PythonScriptName }} -m .\${{ parameters.LocalFolder }}\${{ parameters.ModelFolder }} --provider $(ep)
+ python ${{ parameters.PythonScriptName }} -m .\${{ parameters.LocalFolder }}\${{ parameters.ModelFolder }} --provider $(ep) --non-interactive
}
displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact on Windows'
workingDirectory: '$(Build.Repository.LocalPath)'
@@ -72,7 +72,7 @@ steps:
$python_exe -m pip install -r /ort_genai_src/test/python/cuda/ort/requirements.txt && \
cd /ort_genai_src/${{ parameters.PythonScriptFolder }} && \
$python_exe -m pip install --no-index --find-links=/ort_genai_binary/wheel $(pip_package_name) && \
- $python_exe ${{ parameters.PythonScriptName }} -m ./${{ parameters.LocalFolder }}/${{ parameters.ModelFolder }} --provider $(ep)"
+ $python_exe ${{ parameters.PythonScriptName }} -m ./${{ parameters.LocalFolder }}/${{ parameters.ModelFolder }} --provider $(ep) --non-interactive"
displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact on Linux CUDA'
workingDirectory: '$(Build.Repository.LocalPath)'
@@ -91,7 +91,7 @@ steps:
fi
cd ${{ parameters.PythonScriptFolder }}
python -m pip install --no-index --find-links=$(Build.BinariesDirectory)/wheel $(pip_package_name)
- python ${{ parameters.PythonScriptName }} -m ./${{ parameters.LocalFolder }}/${{ parameters.ModelFolder }} --provider $(ep)
+ python ${{ parameters.PythonScriptName }} -m ./${{ parameters.LocalFolder }}/${{ parameters.ModelFolder }} --provider $(ep) --non-interactive
displayName: 'Run ${{ parameters.PythonScriptName }} With Artifact on Linux/macOS CPU'
workingDirectory: '$(Build.Repository.LocalPath)'
condition: and(or(eq(variables['os'], 'linux'), eq(variables['os'], 'osx')), eq(variables['ep'], 'cpu'))
\ No newline at end of file
diff --git a/examples/csharp/HelloPhi/Program.cs b/examples/csharp/HelloPhi/Program.cs
index f5448d8b44..02190f21b1 100644
--- a/examples/csharp/HelloPhi/Program.cs
+++ b/examples/csharp/HelloPhi/Program.cs
@@ -8,7 +8,7 @@ void PrintUsage()
Console.WriteLine("Usage:");
Console.WriteLine(" -m model_path");
Console.WriteLine("\t\t\t\tPath to the model");
- Console.WriteLine(" --interactive (optional)");
+ Console.WriteLine(" --non-interactive (optional)");
Console.WriteLine("\t\t\t\tInteractive mode");
}
@@ -20,16 +20,16 @@ void PrintUsage()
Environment.Exit(-1);
}
-bool interactive = false;
+bool interactive = true;
string modelPath = string.Empty;
uint i = 0;
while (i < args.Length)
{
var arg = args[i];
- if (arg == "--interactive")
+ if (arg == "--non-interactive")
{
- interactive = true;
+ interactive = false;
}
else if (arg == "-m")
{
diff --git a/examples/csharp/HelloPhi3V/Program.cs b/examples/csharp/HelloPhi3V/Program.cs
index 3c383650b1..d04f2ec702 100644
--- a/examples/csharp/HelloPhi3V/Program.cs
+++ b/examples/csharp/HelloPhi3V/Program.cs
@@ -35,7 +35,7 @@ void PrintUsage()
Console.WriteLine("\t\t\t\tPath to the model");
Console.WriteLine(" --image_paths");
Console.WriteLine("\t\t\t\tPath to the images");
- Console.WriteLine(" --interactive (optional)");
+ Console.WriteLine(" --non-interactive (optional), mainly for CI usage");
Console.WriteLine("\t\t\t\tInteractive mode");
}
@@ -47,7 +47,7 @@ void PrintUsage()
Environment.Exit(-1);
}
-bool interactive = false;
+bool interactive = true;
string modelPath = string.Empty;
List imagePaths = new List();
@@ -55,9 +55,9 @@ void PrintUsage()
while (i_arg < args.Length)
{
var arg = args[i_arg];
- if (arg == "--interactive")
+ if (arg == "--non-interactive")
{
- interactive = true;
+ interactive = false;
}
else if (arg == "-m")
{
diff --git a/examples/python/phi3v.py b/examples/python/phi3v.py
index 5ec7419036..ef410088f0 100644
--- a/examples/python/phi3v.py
+++ b/examples/python/phi3v.py
@@ -13,7 +13,7 @@ def _find_dir_contains_sub_dir(current_dir: Path, target_dir_name):
curr_path = Path(current_dir).absolute()
target_dir = glob.glob(target_dir_name, root_dir=curr_path)
if target_dir:
- return Path(target_dir[0]).absolute()
+ return Path(curr_path / target_dir[0]).absolute()
else:
if curr_path.parent == curr_path:
# Root dir
@@ -37,7 +37,7 @@ def run(args: argparse.Namespace):
processor = model.create_multimodal_processor()
tokenizer_stream = processor.create_stream()
- interactive = args.interactive
+ interactive = not args.non_interactive
while True:
if interactive:
@@ -126,7 +126,7 @@ def run(args: argparse.Namespace):
'-pr', '--prompt', required=False, help='Input prompts to generate tokens from.'
)
parser.add_argument(
- '--interactive', default=False, required=False, help='Interactive mode'
+ '--non-interactive', action=argparse.BooleanOptionalAction, required=False, help='Non-interactive mode, mainly for CI usage'
)
args = parser.parse_args()
run(args)
From e273fb47ceb507711b85f0d7eed23bd98bfb7550 Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Mon, 18 Nov 2024 17:59:48 +0800
Subject: [PATCH 35/42] [skip ci] fix model-generate.py
---
examples/python/model-generate.py | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/examples/python/model-generate.py b/examples/python/model-generate.py
index e98d84dd9a..14d05220ff 100644
--- a/examples/python/model-generate.py
+++ b/examples/python/model-generate.py
@@ -19,9 +19,13 @@ def main(args):
if hasattr(args, 'prompts'):
prompts = args.prompts
else:
- prompts = ["I like walking my cute dog",
+ if args.non_interactive:
+ prompts = ["I like walking my cute dog",
"What is the best restaurant in town?",
"Hello, how are you today?"]
+ else:
+ text = input("Input: ")
+ prompts = [text]
if args.chat_template:
if args.chat_template.count('{') != 1 or args.chat_template.count('}') != 1:
@@ -78,6 +82,7 @@ def main(args):
parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Print verbose output and timing information. Defaults to false')
parser.add_argument('-b', '--batch_size_for_cuda_graph', type=int, default=1, help='Max batch size for CUDA graph')
parser.add_argument('-c', '--chat_template', type=str, default='', help='Chat template to use for the prompt. User input will be injected into {input}. If not set, the prompt is used as is.')
+ parser.add_argument('--non-interactive', action=argparse.BooleanOptionalAction, required=False, help='Non-interactive mode, mainly for CI usage')
args = parser.parse_args()
main(args)
\ No newline at end of file
From e0f07c2bf0668862e91a08358de121ec327dc818 Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Tue, 19 Nov 2024 08:15:26 +0800
Subject: [PATCH 36/42] [skip ci] total time in phi3v.py
---
examples/python/phi3v.py | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/examples/python/phi3v.py b/examples/python/phi3v.py
index ef410088f0..c58e9c8765 100644
--- a/examples/python/phi3v.py
+++ b/examples/python/phi3v.py
@@ -4,6 +4,7 @@
import argparse
import os
import glob
+import time
from pathlib import Path
import onnxruntime_genai as og
@@ -93,6 +94,7 @@ def run(args: argparse.Namespace):
params.set_search_options(max_length=7680)
generator = og.Generator(model, params)
+ start_time = time.time()
while not generator.is_done():
generator.compute_logits()
@@ -101,6 +103,10 @@ def run(args: argparse.Namespace):
new_token = generator.get_next_tokens()[0]
print(tokenizer_stream.decode(new_token), end="", flush=True)
+ print()
+ total_run_time = time.time() - start_time
+ print(f"Total Time : {total_run_time:.2f}")
+
for _ in range(3):
print()
From 283c285f496a67be2f6477fc5295ab564a1fb498 Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Tue, 19 Nov 2024 08:17:20 +0800
Subject: [PATCH 37/42] [skip ci] total time in phi3v csharp
---
examples/csharp/HelloPhi3V/Program.cs | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/examples/csharp/HelloPhi3V/Program.cs b/examples/csharp/HelloPhi3V/Program.cs
index d04f2ec702..61c23415e7 100644
--- a/examples/csharp/HelloPhi3V/Program.cs
+++ b/examples/csharp/HelloPhi3V/Program.cs
@@ -138,12 +138,16 @@ void PrintUsage()
generatorParams.SetInputs(inputTensors);
using var generator = new Generator(model, generatorParams);
+ var watch = System.Diagnostics.Stopwatch.StartNew();
while (!generator.IsDone())
{
generator.ComputeLogits();
generator.GenerateNextToken();
Console.Write(tokenizerStream.Decode(generator.GetSequence(0)[^1]));
}
+ watch.Stop();
+ var runTimeInSeconds = watch.Elapsed.TotalSeconds;
+ Console.WriteLine($"Total Time: {runTimeInSeconds:0.00}");
if (images != null)
{
From 4d9ba1df10a07a4f78acc00868e59406b5711f7a Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Tue, 19 Nov 2024 11:14:26 +0800
Subject: [PATCH 38/42] [skip ci] Update examples/python/phi3v.py
Co-authored-by: kunal-vaishnavi <115581922+kunal-vaishnavi@users.noreply.github.com>
---
examples/python/phi3v.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/examples/python/phi3v.py b/examples/python/phi3v.py
index c58e9c8765..5a61816ff9 100644
--- a/examples/python/phi3v.py
+++ b/examples/python/phi3v.py
@@ -126,7 +126,7 @@ def run(args: argparse.Namespace):
"-p", "--provider", type=str, required=True, help="Provider to run model"
)
parser.add_argument(
- "--image_paths", nargs='*', type=str, required=False, help="Path to the images"
+ "--image_paths", nargs='*', type=str, required=False, help="Path to the images, mainly for CI usage"
)
parser.add_argument(
'-pr', '--prompt', required=False, help='Input prompts to generate tokens from.'
From d73f9e794ed192b2d55a837597ddd32b8affc074 Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Tue, 19 Nov 2024 11:14:39 +0800
Subject: [PATCH 39/42] [skip ci] Update examples/python/phi3v.py
Co-authored-by: kunal-vaishnavi <115581922+kunal-vaishnavi@users.noreply.github.com>
---
examples/python/phi3v.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/examples/python/phi3v.py b/examples/python/phi3v.py
index 5a61816ff9..02ff24c71b 100644
--- a/examples/python/phi3v.py
+++ b/examples/python/phi3v.py
@@ -129,7 +129,7 @@ def run(args: argparse.Namespace):
"--image_paths", nargs='*', type=str, required=False, help="Path to the images, mainly for CI usage"
)
parser.add_argument(
- '-pr', '--prompt', required=False, help='Input prompts to generate tokens from.'
+ '-pr', '--prompt', required=False, help='Input prompts to generate tokens from, mainly for CI usage'
)
parser.add_argument(
'--non-interactive', action=argparse.BooleanOptionalAction, required=False, help='Non-interactive mode, mainly for CI usage'
From eaa010de75116993a8a5c25522ae0133b2cc3c56 Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Tue, 19 Nov 2024 11:24:12 +0800
Subject: [PATCH 40/42] [skip ci] refactor
---
.pipelines/stages/jobs/nuget-validation-job.yml | 4 ++--
.pipelines/stages/jobs/py-validation-job.yml | 4 ++--
...-validation-with-model.yml => nuget-validation-step.yml} | 0
...validation-with-model.yml => python-validation-step.yml} | 0
examples/csharp/HelloPhi/HelloPhi.csproj | 6 +++---
examples/csharp/HelloPhi3V/HelloPhi3V.csproj | 6 +++---
6 files changed, 10 insertions(+), 10 deletions(-)
rename .pipelines/stages/jobs/steps/{utils/perform-nuget-validation-with-model.yml => nuget-validation-step.yml} (100%)
rename .pipelines/stages/jobs/steps/{utils/perform-python-validation-with-model.yml => python-validation-step.yml} (100%)
diff --git a/.pipelines/stages/jobs/nuget-validation-job.yml b/.pipelines/stages/jobs/nuget-validation-job.yml
index 1b22a702b6..88a1c0c0bd 100644
--- a/.pipelines/stages/jobs/nuget-validation-job.yml
+++ b/.pipelines/stages/jobs/nuget-validation-job.yml
@@ -147,7 +147,7 @@ jobs:
HuggingFaceToken: $(HF_TOKEN)
os: ${{ parameters.os }}
- - template: steps/utils/perform-nuget-validation-with-model.yml
+ - template: steps/nuget-validation-step.yml
parameters:
CsprojFolder: "examples/csharp/HelloPhi"
CsprojName: "HelloPhi"
@@ -164,7 +164,7 @@ jobs:
HuggingFaceToken: $(HF_TOKEN)
os: ${{ parameters.os }}
- - template: steps/utils/perform-nuget-validation-with-model.yml
+ - template: steps/nuget-validation-step.yml
parameters:
CsprojFolder: "examples/csharp/HelloPhi3V"
CsprojName: "HelloPhi3V"
diff --git a/.pipelines/stages/jobs/py-validation-job.yml b/.pipelines/stages/jobs/py-validation-job.yml
index ab2d2a983e..2930b88854 100644
--- a/.pipelines/stages/jobs/py-validation-job.yml
+++ b/.pipelines/stages/jobs/py-validation-job.yml
@@ -164,7 +164,7 @@ jobs:
HuggingFaceToken: $(HF_TOKEN)
os: ${{ parameters.os }}
- - template: steps/utils/perform-python-validation-with-model.yml
+ - template: steps/python-validation-step.yml
parameters:
PythonScriptFolder: "examples/python"
PythonScriptName: "model-generate.py"
@@ -180,7 +180,7 @@ jobs:
HuggingFaceToken: $(HF_TOKEN)
os: ${{ parameters.os }}
- - template: steps/utils/perform-python-validation-with-model.yml
+ - template: steps/python-validation-step.yml
parameters:
PythonScriptFolder: "examples/python"
PythonScriptName: "phi3v.py"
diff --git a/.pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml b/.pipelines/stages/jobs/steps/nuget-validation-step.yml
similarity index 100%
rename from .pipelines/stages/jobs/steps/utils/perform-nuget-validation-with-model.yml
rename to .pipelines/stages/jobs/steps/nuget-validation-step.yml
diff --git a/.pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml b/.pipelines/stages/jobs/steps/python-validation-step.yml
similarity index 100%
rename from .pipelines/stages/jobs/steps/utils/perform-python-validation-with-model.yml
rename to .pipelines/stages/jobs/steps/python-validation-step.yml
diff --git a/examples/csharp/HelloPhi/HelloPhi.csproj b/examples/csharp/HelloPhi/HelloPhi.csproj
index 3c5855bda3..24e6c5bb9d 100644
--- a/examples/csharp/HelloPhi/HelloPhi.csproj
+++ b/examples/csharp/HelloPhi/HelloPhi.csproj
@@ -10,9 +10,9 @@
-
-
-
+
+
+
diff --git a/examples/csharp/HelloPhi3V/HelloPhi3V.csproj b/examples/csharp/HelloPhi3V/HelloPhi3V.csproj
index 2a85abc0e0..dd0f330696 100644
--- a/examples/csharp/HelloPhi3V/HelloPhi3V.csproj
+++ b/examples/csharp/HelloPhi3V/HelloPhi3V.csproj
@@ -9,9 +9,9 @@
-
-
-
+
+
+
From 7a9b3b0772a56d6e1437f0ad2c961aa45b787e80 Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Tue, 19 Nov 2024 13:19:10 +0800
Subject: [PATCH 41/42] Fix
---
.../stages/jobs/steps/utils/download-huggingface-model.yml | 3 +++
.../jobs/steps/utils/flex-download-pipeline-artifact.yml | 1 +
examples/csharp/HelloPhi3V/Program.cs | 1 +
3 files changed, 5 insertions(+)
diff --git a/.pipelines/stages/jobs/steps/utils/download-huggingface-model.yml b/.pipelines/stages/jobs/steps/utils/download-huggingface-model.yml
index 7f3d4dc14f..c537cd86e0 100644
--- a/.pipelines/stages/jobs/steps/utils/download-huggingface-model.yml
+++ b/.pipelines/stages/jobs/steps/utils/download-huggingface-model.yml
@@ -22,6 +22,8 @@ steps:
workingDirectory: ${{ parameters.WorkingDirectory }}
env:
HF_TOKEN: ${{ parameters.HuggingFaceToken }}
+ condition: succeededOrFailed() # Run this even if previous tasks failed.
+
- ${{ if eq(parameters.os, 'win') }}:
- powershell: |
python -m pip install "huggingface_hub[cli]"
@@ -32,3 +34,4 @@ steps:
workingDirectory: ${{ parameters.WorkingDirectory }}
env:
HF_TOKEN: ${{ parameters.HuggingFaceToken }}
+ condition: succeededOrFailed() # Run this even if previous tasks failed.
diff --git a/.pipelines/stages/jobs/steps/utils/flex-download-pipeline-artifact.yml b/.pipelines/stages/jobs/steps/utils/flex-download-pipeline-artifact.yml
index a83451a1b3..232e8549e9 100644
--- a/.pipelines/stages/jobs/steps/utils/flex-download-pipeline-artifact.yml
+++ b/.pipelines/stages/jobs/steps/utils/flex-download-pipeline-artifact.yml
@@ -30,3 +30,4 @@ steps:
pipeline: $(Build.DefinitionName)
runVersion: 'specific'
buildId: ${{ parameters.BuildId }}
+ condition: succeededOrFailed() # Run this even if previous tasks failed.
diff --git a/examples/csharp/HelloPhi3V/Program.cs b/examples/csharp/HelloPhi3V/Program.cs
index 61c23415e7..61f98b2d93 100644
--- a/examples/csharp/HelloPhi3V/Program.cs
+++ b/examples/csharp/HelloPhi3V/Program.cs
@@ -147,6 +147,7 @@ void PrintUsage()
}
watch.Stop();
var runTimeInSeconds = watch.Elapsed.TotalSeconds;
+ Console.WriteLine();
Console.WriteLine($"Total Time: {runTimeInSeconds:0.00}");
if (images != null)
From 110719f6367ac5f4861fdf67423a5ee7dc8e51c2 Mon Sep 17 00:00:00 2001
From: Chester Liu <4710575+skyline75489@users.noreply.github.com>
Date: Tue, 19 Nov 2024 15:53:21 +0800
Subject: [PATCH 42/42] [skip ci] Config not available untin 0.5.1
---
examples/python/model-generate.py | 17 ++++++++++-------
examples/python/phi3v.py | 15 +++++++++------
2 files changed, 19 insertions(+), 13 deletions(-)
diff --git a/examples/python/model-generate.py b/examples/python/model-generate.py
index 14d05220ff..930eba5dec 100644
--- a/examples/python/model-generate.py
+++ b/examples/python/model-generate.py
@@ -4,13 +4,16 @@
def main(args):
if args.verbose: print("Loading model...")
- config = og.Config(args.model_path)
- config.clear_providers()
- if args.provider != "cpu":
- if args.verbose:
- print(f"Setting model to {args.provider}...")
- config.append_provider(args.provider)
- model = og.Model(config)
+ if hasattr(og, 'Config'):
+ config = og.Config(args.model_path)
+ config.clear_providers()
+ if args.provider != "cpu":
+ if args.verbose:
+ print(f"Setting model to {args.provider}...")
+ config.append_provider(args.provider)
+ model = og.Model(config)
+ else:
+ model = og.Model(args.model_path)
if args.verbose: print("Model loaded")
tokenizer = og.Tokenizer(model)
diff --git a/examples/python/phi3v.py b/examples/python/phi3v.py
index 02ff24c71b..67b8beba72 100644
--- a/examples/python/phi3v.py
+++ b/examples/python/phi3v.py
@@ -28,12 +28,15 @@ def _complete(text, state):
def run(args: argparse.Namespace):
print("Loading model...")
- config = og.Config(args.model_path)
- config.clear_providers()
- if args.provider != "cpu":
- print(f"Setting model to {args.provider}...")
- config.append_provider(args.provider)
- model = og.Model(config)
+ if hasattr(og, 'Config'):
+ config = og.Config(args.model_path)
+ config.clear_providers()
+ if args.provider != "cpu":
+ print(f"Setting model to {args.provider}...")
+ config.append_provider(args.provider)
+ model = og.Model(config)
+ else:
+ model = og.Model(args.model_path)
print("Model loaded")
processor = model.create_multimodal_processor()
tokenizer_stream = processor.create_stream()