Finish readme text on pipeline provisioning; added keyvault secrets t… (#3186)

…ests

---------

Co-authored-by: Copilot <[email protected]>
Co-authored-by: Heath Stewart <[email protected]>

Author: 3 people

.gitignore

@@ -13,6 +13,7 @@ Cargo.lock
 .proxy/
 tests/data/
 review/
+results.json
 
 # Dependencies.
 vcpkg_installed/

sdk/core/azure_core/benches/http_transport_benchmarks.rs

@@ -15,6 +15,10 @@ use azure_identity::DeveloperToolsCredential;
 use criterion::{criterion_group, criterion_main, Criterion};
 use std::sync::Arc;
 
+#[cfg_attr(target_os = "macos", allow(dead_code))]
+const HTTP_ENDPOINT: &str = "https://azuresdkforcpp.azurewebsites.net";
+//const HTTP_ENDPOINT: &str = "http://httpbin.org";
+
 #[derive(Clone, SafeDebug)]
 pub struct TestServiceClientOptions {
     pub client_options: ClientOptions,
@@ -136,7 +140,7 @@ pub fn simple_http_transport_test(c: &mut Criterion) {
     {
         let rt = tokio::runtime::Runtime::new().unwrap();
 
-        let endpoint = "https://azuresdkforcpp.azurewebsites.net";
+        let endpoint = HTTP_ENDPOINT;
         let credential = DeveloperToolsCredential::new(None).unwrap();
         let options = TestServiceClientOptions::default();
 
@@ -163,7 +167,7 @@ pub fn disable_pooling_http_transport_test(c: &mut Criterion) {
     {
         let rt = tokio::runtime::Runtime::new().unwrap();
 
-        let endpoint = "https://azuresdkforcpp.azurewebsites.net";
+        let endpoint = HTTP_ENDPOINT;
         let credential = DeveloperToolsCredential::new(None).unwrap();
         let transport = new_reqwest_client_disable_connection_pool();
         let options = TestServiceClientOptions {
@@ -196,21 +200,50 @@ pub fn baseline_http_transport_test(c: &mut Criterion) {
     #[cfg(not(target_os = "macos"))]
     {
         let rt = tokio::runtime::Runtime::new().unwrap();
-        let endpoint = "https://azuresdkforcpp.azurewebsites.net";
+        let endpoint = HTTP_ENDPOINT;
 
         let http_client = new_default_reqwest_client();
 
+        let url = Url::parse(&format!("{}/get", endpoint)).unwrap();
+
         // Benchmark GET and POST requests
         c.bench_function("baseline_http_pipeline_test", |b| {
+            b.to_async(&rt).iter(|| {
+                // Clone the Url for this iteration so the async block can take ownership.
+                let url = url.clone();
+                let http_client = http_client.clone();
+                async move {
+                    let request = Request::new(url, Method::Get);
+                    let response = http_client.execute_request(&request).await;
+                    assert!(response.is_ok());
+                    let response = response.unwrap();
+                    assert_eq!(response.status(), azure_core::http::StatusCode::Ok);
+                }
+            });
+        });
+    }
+}
+
+#[cfg_attr(target_os = "macos", allow(unused_variables))]
+pub fn raw_reqwest_http_transport_test(c: &mut Criterion) {
+    #[cfg(target_os = "macos")]
+    return;
+
+    #[cfg(not(target_os = "macos"))]
+    {
+        let rt = tokio::runtime::Runtime::new().unwrap();
+        let endpoint = HTTP_ENDPOINT;
+
+        let client = ::reqwest::Client::new();
+
+        // Benchmark GET and POST requests
+        c.bench_function("raw_http_pipeline_test", |b| {
             b.to_async(&rt).iter(|| async {
-                let request = Request::new(
-                    Url::parse(&format!("{}/get", endpoint)).unwrap(),
-                    Method::Get,
-                );
-                let response = http_client.execute_request(&request).await;
+                let request = client.get(format!("{}/get", endpoint));
+                let response = request.send().await;
                 assert!(response.is_ok());
                 let response = response.unwrap();
-                assert_eq!(response.status(), azure_core::http::StatusCode::Ok);
+                assert_eq!(response.status(), reqwest::StatusCode::OK);
             });
         });
     }
@@ -219,10 +252,10 @@ pub fn baseline_http_transport_test(c: &mut Criterion) {
 // Main benchmark configuration
 criterion_group!(name=http_transport_benchmarks;
     config=Criterion::default()
-        .sample_size(100)
+        .sample_size(500)
         .warm_up_time(std::time::Duration::new(10, 0))
-        .measurement_time(std::time::Duration::new(50, 0));
-    targets=simple_http_transport_test, disable_pooling_http_transport_test, baseline_http_transport_test
+        .measurement_time(std::time::Duration::new(60, 0));
+    targets=simple_http_transport_test, disable_pooling_http_transport_test, baseline_http_transport_test, raw_reqwest_http_transport_test
 );
 
 criterion_main!(http_transport_benchmarks);

sdk/core/azure_core_test/src/perf/README.md

@@ -159,9 +159,22 @@ Before you can declare your test pipeline, you need to create some infrastructur
 
 Test pipelines are defined using a [`perf.yml`](https://github.com/Azure/azure-sdk-for-rust/blob/main/sdk/storage/azure_storage_blob/perf.yml) file declared in the package directory.
 
-For example, from the `storage/azure_storage_blob` package:
+For example (from the `storage/azure_storage_blob` package):
 
 ```yml
+trigger: none
+
+pr: none
+
+# Schedule the pipeline to run at UTC+7 Hours (Midnight Pacific time)
+schedules:
+- cron: "0 7 * * *"
+  displayName: Daily midnight run.
+  branches:
+    include:
+    - main
+  always: true
+
 parameters:
 - name: PackageVersions
   displayName: PackageVersions (regex of package versions to run)
@@ -199,9 +212,7 @@ extends:
     Profile: ${{ parameters.Profile }}
 ```
 
-***TODO Update this to include yml based triggers***
-
-You'll want to configure the `ServiceDirectory` field to match the location of your package.
+You'll want to configure the `ServiceDirectory` field to match the location of your package, and tweak the default values for the variables to match your performance tests.
 
 #### Performance Test Yaml Configuration
 
@@ -251,7 +262,8 @@ And FINALLY, you need to create a pull request containing this file and find the
 
 #### Creating the performance pipeline
 
-Once the pull request
+Once the pull request you created above has been committed to the branch, you can start to create the performance pipelines (note: DO NOT ATTEMPT TO CREATE THE PIPELINE UNTIL THE `perf.yml` file mentioned above is in `main` - if you don't, you are highly likely to disrupt all operations in the repository).
+
 Navigate to the `azure-sdk` Azure DevOps instance, and select the `internal` project.
 
 Within the `internal` project, select `Pipelines`, select "All" from the right hand pane. This will show a tree structured hierarchy of pipelines.
@@ -268,4 +280,34 @@ Next select `Azure/azure-sdk-for-rust` to specify the Rust SDK and configure you
 
 Select your pipeline file from the main branch of the repository and you're almost done.
 
-The next thing you want to do is to "save" the new pipeline.
+The next thing you want to do is to "save" the new pipeline. This will cause your pipeline to be created. You can also attempt to `run` the pipeline at this point but it is likely to fail.
+
+You now need to set the required variables or the pipeline. Performance pipelines require the `Secrets for Resource Provisioner` variable group added to the pipeline. To add this, select the newly created pipeline, and click on `Edit`. Navigate to the `...` menu and select `Triggers`. This brings up the `Yaml`, `Variables`, `Triggers` and `History` edit. Make sure that all the triggers (included scheduled triggers) are cleared from the `Triggers` - Rust performance pipeline triggers are managed by the pipeline yaml file, rather than in the Azure DevOps user interface.
+
+Select `Variables`, which allows you to add variables to the pipeline. You want to select `Variable groups` in the left hand column and select `Link variable group` in the right hand column.
+
+That will bring up a pane on the right with a number of variable groups. You want to select the `Secrets for Resource Provisioner` variable group and click the `Link` button.
+
+Once you've saved these changes, your pipeline should be ready to run.
+
+You may need to ask for help from the Azure SDK Engineering Systems team to enable access to test resources for your pipeline.
+
+## Running the test automation locally
+
+It is possible to run the performance test automation locally, that is often helpful when debugging performance tests.
+
+Running the performance tests locally requires a clone of the `Azure/azure-sdk-tools` repo.
+
+Start at the root of the `azure-sdk-tools` repo and navigate to the `tools/perf-automation/Azure.Sdk.Tools/PerfAutomation` directory:
+
+```bash
+cd tools/perf-automation/Azure.Sdk.Tools/PerfAutomation
+```
+
+Then run the perf automation tool, replacing your repo on the command line:
+
+```bash
+ dotnet run . -- -l rust --language-version N/A --tests-file {Path to perf-tests.yml file in your repo} --repo-root {Path to the root of your repo}
+ ```
+
+ This will run your performance tests and save the results in the `results` directory locally.

sdk/core/azure_core_test/src/perf/mod.rs

@@ -505,7 +505,7 @@ impl PerfRunner {
             .arg(
                 clap::arg!(--"test-results" <FILE> "The file to write test results to")
                     .required(false)
-                    .default_value("./tests/results.json")
+                    .default_value("./results.json")
                     .global(false),
             )
             .arg(clap::arg!(--"no-cleanup" "Disable test cleanup")

sdk/keyvault/azure_security_keyvault_keys/Cargo.toml

@@ -42,6 +42,7 @@ rustc_version.workspace = true
 [lints]
 workspace = true
 
-[[bench]]
-name = "benchmarks"
+[[test]]
+name = "perf"
+path = "perf/perf_tests.rs"
 harness = false