Allow specifying default versions on ClientTestFixture

sdk/core/Azure.Core.TestFramework/src/ClientTestFixtureAttribute.cs

@@ -25,7 +25,18 @@ public class ClientTestFixtureAttribute : NUnitAttribute, IFixtureBuilder2, IPre
 
         private readonly object[] _additionalParameters;
         private readonly object[] _serviceVersions;
-        private readonly int? _maxServiceVersion;
+        private int? _actualPlaybackServiceVersion;
+        private int[] _actualLiveServiceVersions;
+
+        /// <summary>
+        /// Specifies which service version is run during recording/playback runs.
+        /// </summary>
+        public object RecordingServiceVersion { get; set; }
+
+        /// <summary>
+        /// Specifies which service version is run during live runs.
+        /// </summary>
+        public object[] LiveServiceVersions { get; set; }
 
         /// <summary>
         /// Initializes an instance of the <see cref="ClientTestFixtureAttribute"/> accepting additional fixture parameters.
@@ -43,8 +54,6 @@ public ClientTestFixtureAttribute(object[] serviceVersions, object[] additionalP
         {
             _additionalParameters = additionalParameters ?? new object[] { };
             _serviceVersions = serviceVersions ?? new object[] { };
-
-            _maxServiceVersion = _serviceVersions.Any() ? _serviceVersions.Max(s => Convert.ToInt32(s)) : (int?)null;
         }
 
         public IEnumerable<TestSuite> BuildFrom(ITypeInfo typeInfo)
@@ -54,6 +63,23 @@ public IEnumerable<TestSuite> BuildFrom(ITypeInfo typeInfo)
 
         public IEnumerable<TestSuite> BuildFrom(ITypeInfo typeInfo, IPreFilter filter)
         {
+            var latestVersion = _serviceVersions.Any() ? _serviceVersions.Max(Convert.ToInt32) : (int?)null;
+            _actualPlaybackServiceVersion = RecordingServiceVersion != null ? Convert.ToInt32(RecordingServiceVersion) : latestVersion;
+
+            var liveVersions = LiveServiceVersions ?? _serviceVersions;
+
+            if (liveVersions.Any())
+            {
+                if (OnlyTestLatestServiceVersionLazy.Value)
+                {
+                    _actualLiveServiceVersions = new[] { liveVersions.Max(Convert.ToInt32) };
+                }
+                else
+                {
+                    _actualLiveServiceVersions = liveVersions.Select(Convert.ToInt32).ToArray();
+                }
+            }
+
             var suitePermutations = GeneratePermutations();
 
             foreach (var (fixture, isAsync, serviceVersion, parameter) in suitePermutations)
@@ -114,6 +140,8 @@ public IEnumerable<TestSuite> BuildFrom(ITypeInfo typeInfo, IPreFilter filter)
         private void Process(TestSuite testSuite, object serviceVersion, bool isAsync, object parameter)
         {
             var serviceVersionNumber = Convert.ToInt32(serviceVersion);
+            ApplyLimits(serviceVersionNumber, testSuite);
+
             foreach (Test test in testSuite.Tests)
             {
                 if (test is ParameterizedMethodSuite parameterizedMethodSuite)
@@ -157,18 +185,23 @@ private void ProcessTest(object serviceVersion, bool isAsync, int serviceVersion
                 return;
             }
 
-            if (serviceVersionNumber != _maxServiceVersion)
+            if (serviceVersionNumber != _actualPlaybackServiceVersion)
             {
-                test.Properties.Add("SkipRecordings", $"Test is ignored when not running live because the service version {serviceVersion} is not the latest.");
+                test.Properties.Add("SkipRecordings", $"Test is ignored when not running live because the service version {serviceVersion} is not {_actualPlaybackServiceVersion}.");
             }
 
-            if (OnlyTestLatestServiceVersionLazy.Value && serviceVersionNumber != _maxServiceVersion)
+            if (_actualLiveServiceVersions != null &&
+                !_actualLiveServiceVersions.Contains(serviceVersionNumber))
             {
-                test.RunState = RunState.Ignored;
-                test.Properties.Set("_SKIPREASON",
-                    $"Test ignored because {OnlyTestLatestServiceVersionKey} is set in the environment and version {serviceVersion} is not the latest.");
+                test.Properties.Set("SkipLive",
+                    $"Test ignored when running live service version {serviceVersion} is not one of {string.Join(", " , _actualLiveServiceVersions)}.");
             }
 
+            ApplyLimits(serviceVersionNumber, test);
+        }
+
+        private static void ApplyLimits(int serviceVersionNumber, Test test)
+        {
             var minServiceVersion = test.GetCustomAttributes<ServiceVersionAttribute>(true);
             foreach (ServiceVersionAttribute serviceVersionAttribute in minServiceVersion)
             {
@@ -179,7 +212,7 @@ private void ProcessTest(object serviceVersion, bool isAsync, int serviceVersion
                     test.Properties.Set("_SKIPREASON", $"Test ignored because it's minimum service version is set to {serviceVersionAttribute.Min}");
                 }
 
-                if (serviceVersionAttribute.Max != null &&
+                if (serviceVersionAttribute.Max != null &
                     Convert.ToInt32(serviceVersionAttribute.Max) < serviceVersionNumber)
                 {
                     test.RunState = RunState.Ignored;

sdk/core/Azure.Core.TestFramework/src/RecordedTestBase.cs

@@ -143,6 +143,13 @@ public virtual void StartTestRecording()
             {
                 throw new IgnoreException((string) test.Properties.Get("SkipRecordings"));
             }
+
+            if (Mode == RecordedTestMode.Live &&
+                test.Properties.ContainsKey("SkipLive"))
+            {
+                throw new IgnoreException((string) test.Properties.Get("SkipLive"));
+            }
+
             Recording = new TestRecording(Mode, GetSessionFilePath(), Sanitizer, Matcher);
             ValidateClientInstrumentation = Recording.HasRequests;
         }

sdk/core/Azure.Core.TestFramework/src/ServiceVersionAttribute.cs

@@ -6,7 +6,7 @@
 
 namespace Azure.Core.TestFramework
 {
-    [AttributeUsage(AttributeTargets.Method, AllowMultiple = false, Inherited = false)]
+    [AttributeUsage(AttributeTargets.Method | AttributeTargets.Class, AllowMultiple = false, Inherited = false)]
     public class ServiceVersionAttribute : NUnitAttribute
     {
         public object Min { get; set; }

sdk/core/Azure.Core/tests/ClientTestBaseMultiVersionTests.cs

@@ -6,7 +6,8 @@
 
 namespace Azure.Core.Tests
 {
-    [ClientTestFixture(FakeClientVersion.V1, FakeClientVersion.V2, FakeClientVersion.V3)]
+    [ClientTestFixture(FakeClientVersion.V0, FakeClientVersion.V1, FakeClientVersion.V2, FakeClientVersion.V3, FakeClientVersion.V4)]
+    [ServiceVersion(Min = FakeClientVersion.V1, Max = FakeClientVersion.V3)]
     public class ClientTestBaseMultiVersionTests : ClientTestBase
     {
         private readonly FakeClientVersion _version;
@@ -59,9 +60,11 @@ public void SyncOnlyWorks()
 
         public enum FakeClientVersion
         {
+            V0 = 0,
             V1 = 1,
             V2 = 2,
-            V3 = 3
+            V3 = 3,
+            V4 = 4
         }
     }
 }

sdk/core/Azure.Core/tests/ClientTestBaseMultiVersionTestsWithSpecificVersions.cs

@@ -0,0 +1,94 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+using System;
+using System.Threading;
+using System.Threading.Tasks;
+using Azure.Core.Pipeline;
+using Azure.Core.TestFramework;
+using NUnit.Framework;
+
+namespace Azure.Core.Tests
+{
+    [ClientTestFixture(
+        TestClientOptions.ServiceVersion.V0,
+        TestClientOptions.ServiceVersion.V1,
+        TestClientOptions.ServiceVersion.V2,
+        TestClientOptions.ServiceVersion.V3,
+        RecordingServiceVersion = TestClientOptions.ServiceVersion.V2,
+        LiveServiceVersions = new object[] { TestClientOptions.ServiceVersion.V1, TestClientOptions.ServiceVersion.V0 })]
+    public class ClientTestBaseMultiVersionTestsWithSpecificVersions : RecordedTestBase
+    {
+        private readonly TestClientOptions.ServiceVersion _version;
+
+        public ClientTestBaseMultiVersionTestsWithSpecificVersions(bool isAsync, TestClientOptions.ServiceVersion version) : base(isAsync)
+        {
+            _version = version;
+            TestDiagnostics = false;
+        }
+
+        [Test]
+        public async Task HasValidVersion()
+        {
+            var testClientOptions = new TestClientOptions(_version) { Transport = new MockTransport(new MockResponse(200))};
+            var client = InstrumentClient(new TestClient(InstrumentClientOptions(testClientOptions)));
+            await client.GetAsync(default);
+            if (Mode == RecordedTestMode.Playback || Mode == RecordedTestMode.Record)
+            {
+                Assert.IsTrue(_version == TestClientOptions.ServiceVersion.V2);
+            }
+            else
+            {
+                Assert.IsTrue(_version == TestClientOptions.ServiceVersion.V1 || _version == TestClientOptions.ServiceVersion.V0);
+            }
+        }
+
+        public class TestClientOptions: ClientOptions
+        {
+            public readonly ServiceVersion Version;
+
+            public enum ServiceVersion
+            {
+                V0 = 0,
+                V1 = 1,
+                V2 = 2,
+                V3 = 3
+            }
+
+            public TestClientOptions(ServiceVersion serviceVersion)
+            {
+                Version = serviceVersion;
+            }
+        }
+        public class TestClient
+        {
+            private readonly TestClientOptions _options;
+            private HttpPipeline _pipeline;
+
+            protected TestClient(){}
+            public TestClient(TestClientOptions options)
+            {
+                _options = options;
+                _pipeline = HttpPipelineBuilder.Build(options);
+            }
+
+            public virtual Response Get(CancellationToken cancellationToken)
+            {
+                using var request = _pipeline.CreateRequest();
+                request.Method = RequestMethod.Get;
+                request.Uri.Reset(new Uri("http://localhost"));
+                request.Uri.AppendQuery("api", _options.Version.ToString());
+                return _pipeline.SendRequest(request, cancellationToken);
+            }
+
+            public virtual async Task<Response> GetAsync(CancellationToken cancellationToken)
+            {
+                using var request = _pipeline.CreateRequest();
+                request.Method = RequestMethod.Get;
+                request.Uri.Reset(new Uri("http://localhost"));
+                request.Uri.AppendQuery("api", _options.Version.ToString());
+                return await _pipeline.SendRequestAsync(request, cancellationToken);
+            }
+        }
+    }
+}