Skip to content

FIx perf pipeline #838

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lucyzhang929 merged 4 commits into from
May 4, 2023
Merged

FIx perf pipeline #838

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
05f758d
dont start func host in fixture for perf
lucyzhang929 May 1, 2023
8f4e34f
don't use fixture in perf test
lucyzhang929 May 3, 2023
b3c3819
Merge branch 'release/trigger' into luczhan/fixperfpipeline
lucyzhang929 May 3, 2023
8e6519b
finally block
lucyzhang929 May 4, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 44 additions & 34 deletions performance/SqlBindingBenchmarks.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License. See License.txt in the project root for license information.

using System.Diagnostics;
using System.Linq;
using BenchmarkDotNet.Running;
using Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Integration;
using Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Common;

namespace Microsoft.Azure.WebJobs.Extensions.Sql.Performance
{
Expand All @@ -13,40 +14,49 @@ public static void Main(string[] args)
{
bool runAll = args.Length == 0;

using var testFixture = new IntegrationTestFixture();
Process azuriteHost = TestUtils.StartAzurite();
TestUtils.SetupDatabase(out string masterConnectionString, out string connectionString);
try
{
// **IMPORTANT** If changing these make sure to update template-steps-performance.yml as well
if (runAll || args.Contains("input"))
{
BenchmarkRunner.Run<SqlInputBindingPerformance>();
}
if (runAll || args.Contains("output"))
{
BenchmarkRunner.Run<SqlOutputBindingPerformance>();
}
if (runAll || args.Contains("trigger"))
{
BenchmarkRunner.Run<SqlTriggerBindingPerformance>();
}
if (runAll || args.Contains("trigger_batch"))
{
BenchmarkRunner.Run<SqlTriggerBindingPerformance_BatchOverride>();
}
if (runAll || args.Contains("trigger_poll"))
{
BenchmarkRunner.Run<SqlTriggerBindingPerformance_PollingIntervalOverride>();
}
if (runAll || args.Contains("trigger_overrides"))
{
BenchmarkRunner.Run<SqlTriggerPerformance_Overrides>();
}
if (runAll || args.Contains("trigger_parallel"))
{
BenchmarkRunner.Run<SqlTriggerBindingPerformance_Parallelization>();
}
if (runAll || args.Contains("trigger_changerate"))
{
BenchmarkRunner.Run<SqlTriggerBindingPerformance_ChangeRate>();
}
}
finally
{
TestUtils.StopAzurite(azuriteHost);
TestUtils.DropDatabase(masterConnectionString, connectionString);

// **IMPORTANT** If changing these make sure to update template-steps-performance.yml as well
if (runAll || args.Contains("input"))
{
BenchmarkRunner.Run<SqlInputBindingPerformance>();
}
if (runAll || args.Contains("output"))
{
BenchmarkRunner.Run<SqlOutputBindingPerformance>();
}
if (runAll || args.Contains("trigger"))
{
BenchmarkRunner.Run<SqlTriggerBindingPerformance>();
}
if (runAll || args.Contains("trigger_batch"))
{
BenchmarkRunner.Run<SqlTriggerBindingPerformance_BatchOverride>();
}
if (runAll || args.Contains("trigger_poll"))
{
BenchmarkRunner.Run<SqlTriggerBindingPerformance_PollingIntervalOverride>();
}
if (runAll || args.Contains("trigger_overrides"))
{
BenchmarkRunner.Run<SqlTriggerPerformance_Overrides>();
}
if (runAll || args.Contains("trigger_parallel"))
{
BenchmarkRunner.Run<SqlTriggerBindingPerformance_Parallelization>();
}
if (runAll || args.Contains("trigger_changerate"))
{
BenchmarkRunner.Run<SqlTriggerBindingPerformance_ChangeRate>();
}
}
}
Expand Down
120 changes: 120 additions & 0 deletions test/Common/TestUtils.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Azure.WebJobs.Extensions.Sql.Samples.Common;
using Microsoft.Data.SqlClient;

namespace Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Common
{
Expand All @@ -22,6 +23,125 @@ public static class TestUtils
{
internal static int ThreadId;

/// <summary>
/// This returns a running Azurite storage emulator.
/// </summary>
public static Process StartAzurite()
{
Console.WriteLine("Starting Azurite Host...");
var azuriteHost = new Process()
{
StartInfo = new ProcessStartInfo
{
FileName = "azurite",
WindowStyle = ProcessWindowStyle.Hidden,
UseShellExecute = true
}
};

azuriteHost.Start();
return azuriteHost;
}

public static void StopAzurite(Process azuriteHost)
{
Console.WriteLine("Stopping Azurite Host...");
try
{
azuriteHost.Kill(true);
azuriteHost.Dispose();
}
catch (Exception e)
{
Console.WriteLine($"Failed to stop Azurite, Error: {e.Message}");
}
}

/// <summary>
/// Sets up a test database for the tests to use.
/// </summary>
/// <remarks>
/// The server the database will be created on can be set by the environment variable "TEST_SERVER", otherwise localhost will be used by default.
/// By default, integrated authentication will be used to connect to the server, unless the env variable "SA_PASSWORD" is set.
/// In this case, connection will be made using SQL login with user "SA" and the provided password.
/// </remarks>
public static void SetupDatabase(out string MasterConnectionString, out string DatabaseName)
{
SqlConnectionStringBuilder connectionStringBuilder;
string connectionString = Environment.GetEnvironmentVariable("TEST_CONNECTION_STRING");
string masterConnectionString;
if (connectionString != null)
{
masterConnectionString = connectionString;
connectionStringBuilder = new SqlConnectionStringBuilder(connectionString);
}
else
{
// Get the test server name from environment variable "TEST_SERVER", default to localhost if not set
string testServer = Environment.GetEnvironmentVariable("TEST_SERVER");
if (string.IsNullOrEmpty(testServer))
{
testServer = "localhost";
}

// First connect to master to create the database
connectionStringBuilder = new SqlConnectionStringBuilder()
{
DataSource = testServer,
InitialCatalog = "master",
Pooling = false,
Encrypt = SqlConnectionEncryptOption.Optional
};

// Either use integrated auth or SQL login depending if SA_PASSWORD is set
string userId = "SA";
string password = Environment.GetEnvironmentVariable("SA_PASSWORD");
if (string.IsNullOrEmpty(password))
{
connectionStringBuilder.IntegratedSecurity = true;
}
else
{
connectionStringBuilder.UserID = userId;
connectionStringBuilder.Password = password;
}
masterConnectionString = connectionStringBuilder.ToString();
}

// Create database
// Retry this in case the server isn't fully initialized yet
string databaseName = GetUniqueDBName("SqlBindingsTest");
Retry(() =>
{
using var masterConnection = new SqlConnection(masterConnectionString);
masterConnection.Open();
ExecuteNonQuery(masterConnection, $"CREATE DATABASE [{databaseName}]", Console.WriteLine);
// Enable change tracking for trigger tests
ExecuteNonQuery(masterConnection, $"ALTER DATABASE [{databaseName}] SET CHANGE_TRACKING = ON (CHANGE_RETENTION = 2 DAYS, AUTO_CLEANUP = ON);", Console.WriteLine);
}, Console.WriteLine);

connectionStringBuilder.InitialCatalog = databaseName;

// Set SqlConnectionString env var for the tests to use
Environment.SetEnvironmentVariable("SqlConnectionString", connectionStringBuilder.ToString());
MasterConnectionString = masterConnectionString;
DatabaseName = databaseName;
}

public static void DropDatabase(string masterConnectionString, string databaseName)
{
try
{
using var masterConnection = new SqlConnection(masterConnectionString);
masterConnection.Open();
ExecuteNonQuery(masterConnection, $"DROP DATABASE IF EXISTS {databaseName}", Console.WriteLine);
}
catch (Exception e)
{
Console.WriteLine($"Failed to drop {databaseName}, Error: {e.Message}");
}
}

/// <summary>
/// Returns a mangled name that unique based on Prefix + Machine + Process
/// </summary>
Expand Down
122 changes: 7 additions & 115 deletions test/Integration/IntegrationTestFixture.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
using System.IO;
using System.Threading.Tasks;
using Microsoft.Azure.WebJobs.Extensions.Sql.Tests.Common;
using Microsoft.Data.SqlClient;
using Xunit;
using static Microsoft.Azure.WebJobs.Extensions.Sql.Telemetry.Telemetry;

Expand All @@ -22,17 +21,17 @@ public class IntegrationTestFixture : IDisposable
/// Host process for Azurite local storage emulator. This is required for non-HTTP trigger functions:
/// https://docs.microsoft.com/azure/azure-functions/functions-develop-local
/// </summary>
private Process AzuriteHost;
private readonly Process AzuriteHost;

/// <summary>
/// Connection string to the master database on the test server, mainly used for database setup and teardown.
/// </summary>
private string MasterConnectionString;
private readonly string MasterConnectionString;

/// <summary>
/// Name of the database used.
/// </summary>
private string DatabaseName;
private readonly string DatabaseName;

/// <summary>
/// List of all functions in the samples folder that will be started before the
Expand All @@ -56,98 +55,11 @@ public class IntegrationTestFixture : IDisposable

public IntegrationTestFixture()
{
this.StartAzurite();
this.SetupDatabase();
this.AzuriteHost = TestUtils.StartAzurite();
TestUtils.SetupDatabase(out this.MasterConnectionString, out this.DatabaseName);
this.StartFunctionHosts();
}

/// <summary>
/// This starts the Azurite storage emulator.
/// </summary>
protected void StartAzurite()
{
Console.WriteLine("Starting Azurite Host...");
this.AzuriteHost = new Process()
{
StartInfo = new ProcessStartInfo
{
FileName = "azurite",
WindowStyle = ProcessWindowStyle.Hidden,
UseShellExecute = true
}
};

this.AzuriteHost.Start();
}

/// <summary>
/// Sets up a test database for the tests to use.
/// </summary>
/// <remarks>
/// The server the database will be created on can be set by the environment variable "TEST_SERVER", otherwise localhost will be used by default.
/// By default, integrated authentication will be used to connect to the server, unless the env variable "SA_PASSWORD" is set.
/// In this case, connection will be made using SQL login with user "SA" and the provided password.
/// </remarks>
private void SetupDatabase()
{
SqlConnectionStringBuilder connectionStringBuilder;
string connectionString = Environment.GetEnvironmentVariable("TEST_CONNECTION_STRING");
if (connectionString != null)
{
this.MasterConnectionString = connectionString;
connectionStringBuilder = new SqlConnectionStringBuilder(connectionString);
}
else
{
// Get the test server name from environment variable "TEST_SERVER", default to localhost if not set
string testServer = Environment.GetEnvironmentVariable("TEST_SERVER");
if (string.IsNullOrEmpty(testServer))
{
testServer = "localhost";
}

// First connect to master to create the database
connectionStringBuilder = new SqlConnectionStringBuilder()
{
DataSource = testServer,
InitialCatalog = "master",
Pooling = false,
Encrypt = SqlConnectionEncryptOption.Optional
};

// Either use integrated auth or SQL login depending if SA_PASSWORD is set
string userId = "SA";
string password = Environment.GetEnvironmentVariable("SA_PASSWORD");
if (string.IsNullOrEmpty(password))
{
connectionStringBuilder.IntegratedSecurity = true;
}
else
{
connectionStringBuilder.UserID = userId;
connectionStringBuilder.Password = password;
}
this.MasterConnectionString = connectionStringBuilder.ToString();
}

// Create database
// Retry this in case the server isn't fully initialized yet
this.DatabaseName = TestUtils.GetUniqueDBName("SqlBindingsTest");
TestUtils.Retry(() =>
{
using var masterConnection = new SqlConnection(this.MasterConnectionString);
masterConnection.Open();
TestUtils.ExecuteNonQuery(masterConnection, $"CREATE DATABASE [{this.DatabaseName}]", Console.WriteLine);
// Enable change tracking for trigger tests
TestUtils.ExecuteNonQuery(masterConnection, $"ALTER DATABASE [{this.DatabaseName}] SET CHANGE_TRACKING = ON (CHANGE_RETENTION = 2 DAYS, AUTO_CLEANUP = ON);", Console.WriteLine);
}, Console.WriteLine);

connectionStringBuilder.InitialCatalog = this.DatabaseName;

// Set SqlConnectionString env var for the tests to use
Environment.SetEnvironmentVariable("SqlConnectionString", connectionStringBuilder.ToString());
}

/// <summary>
/// This starts the function hosts for each language.
/// </summary>
Expand Down Expand Up @@ -257,28 +169,8 @@ void TestOutputHandler(object sender, DataReceivedEventArgs e)
public void Dispose()
{
this.DisposeFunctionHosts();

try
{
this.AzuriteHost.Kill(true);
this.AzuriteHost.Dispose();
}
catch (Exception e)
{
Console.WriteLine($"Failed to stop Azurite, Error: {e.Message}");
}

try
{
// Drop the test database
using var masterConnection = new SqlConnection(this.MasterConnectionString);
masterConnection.Open();
TestUtils.ExecuteNonQuery(masterConnection, $"DROP DATABASE IF EXISTS {this.DatabaseName}", Console.WriteLine);
}
catch (Exception e4)
{
Console.WriteLine($"Failed to drop {this.DatabaseName}, Error: {e4.Message}");
}
TestUtils.StopAzurite(this.AzuriteHost);
TestUtils.DropDatabase(this.MasterConnectionString, this.DatabaseName);

GC.SuppressFinalize(this);
}
Expand Down