diff --git a/tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/AzPredictorTests.cs b/tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/AzPredictorTests.cs
index df52a44ba337..d1f9f3f4aa55 100644
--- a/tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/AzPredictorTests.cs
+++ b/tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/AzPredictorTests.cs
@@ -37,12 +37,12 @@ public sealed class AzPredictorTests
         /// </summary>
         public AzPredictorTests(ModelFixture modelFixture)
         {
-            this._fixture = modelFixture;
+            _fixture = modelFixture;
             var startHistory = $"{AzPredictorConstants.CommandPlaceholder}{AzPredictorConstants.CommandConcatenator}{AzPredictorConstants.CommandPlaceholder}";
 
-            this._service = new MockAzPredictorService(startHistory, this._fixture.PredictionCollection[startHistory], this._fixture.CommandCollection);
-            this._telemetryClient = new MockAzPredictorTelemetryClient();
-            this._azPredictor = new AzPredictor(this._service, this._telemetryClient, new Settings()
+            _service = new MockAzPredictorService(startHistory, _fixture.PredictionCollection[startHistory], _fixture.CommandCollection);
+            _telemetryClient = new MockAzPredictorTelemetryClient();
+            _azPredictor = new AzPredictor(_service, _telemetryClient, new Settings()
             {
                 SuggestionCount = 1,
                 MaxAllowedCommandDuplicate = 1,
@@ -51,84 +51,250 @@ public AzPredictorTests(ModelFixture modelFixture)
         }
 
         /// <summary>
-        /// Verifies when the last command in history are not supported.
-        /// We don't collect the telemetry and only request prediction while StartEarlyProcess is called.
+        /// Verify we replace unsupported command with <see cref="AzPredictorConstants.CommandPlaceholder"/>.
         /// </summary>
-        [Theory]
-        [InlineData("start_of_snippet\nstart_of_snippet\nstart_of_snippet")]
-        [InlineData("start_of_snippet")]
-        [InlineData("")]
-        [InlineData("git status")]
-        [InlineData("git status\nGet-ChildItem")]
-        [InlineData("^29a9l2")]
-        [InlineData("'Get-AzResource'")]
-        [InlineData("Get-AzResource\ngit log")]
-        [InlineData("Get-ChildItem")]
-        public void VerifyWithNonSupportedCommand(string historyLine)
+        [Fact]
+        public void VerifyRequestPredictionForOneUnsupportedCommandInHistory()
         {
-            IReadOnlyList<string> history = historyLine.Split('\n');
+            IReadOnlyList<string> history = new List<string>()
+            {
+                "git status"
+            };
 
-            this._telemetryClient.RecordedSuggestion = null;
-            this._service.IsPredictionRequested = false;
+            _telemetryClient.RecordedSuggestion = null;
+            _service.Commands = null;
+            _service.History = null;
 
-            this._azPredictor.StartEarlyProcessing(history);
+            _azPredictor.StartEarlyProcessing(history);
 
-            Assert.True(this._service.IsPredictionRequested);
-            Assert.NotNull(this._telemetryClient.RecordedSuggestion);
+            Assert.Equal(new List<string>() { AzPredictorConstants.CommandPlaceholder, AzPredictorConstants.CommandPlaceholder }, _service.Commands);
+            Assert.Equal(AzPredictorConstants.CommandPlaceholder, _telemetryClient.RecordedSuggestion.HistoryLine);
+            Assert.Null(_service.History);
         }
 
         /// <summary>
-        /// Verifies when the last command in history are not supported.
-        /// We don't collect the telemetry and only request prediction while StartEarlyProcess is called.
+        /// Verify that we masked the supported command in requesting prediction and telemetry.
         /// </summary>
-        [Theory]
-        [InlineData("start_of_snippet\nConnect-AzAccount")]
-        [InlineData("Get-AzResource")]
-        [InlineData("git status\nGet-AzContext")]
-        [InlineData("Get-AzContext\nGet-AzLog")]
-        public void VerifyWithOneSupportedCommand(string historyLine)
+        [Fact]
+        public void VerifyRequestPredictionForOneSupportedCommandInHistory()
+        {
+            IReadOnlyList<string> history = new List<string>()
+            {
+                "New-AzVM -Name hello -Location WestUS"
+            };
+
+            _telemetryClient.RecordedSuggestion = null;
+            _service.Commands = null;
+            _service.History = null;
+
+            _azPredictor.StartEarlyProcessing(history);
+
+            string maskedCommand = "New-AzVM -Location *** -Name ***";
+
+            Assert.Equal(new List<string>() { AzPredictorConstants.CommandPlaceholder, maskedCommand }, _service.Commands);
+            Assert.Equal(maskedCommand, _telemetryClient.RecordedSuggestion.HistoryLine);
+            Assert.Equal(history[0], _service.History.ToString());
+        }
+
+        /// <summary>
+        ///  Verify that we can handle the two supported command in sequences.
+        /// </summary>
+        [Fact]
+        public void VerifyRequestPredictionForTwoSupportedCommandInHistory()
+        {
+            IReadOnlyList<string> history = new List<string>()
+            {
+                "New-AzResourceGroup -Name 'resourceGroup01'",
+                "New-AzVM -Name:hello -Location:WestUS"
+            };
+
+            _telemetryClient.RecordedSuggestion = null;
+            _service.Commands = null;
+            _service.History = null;
+
+            _azPredictor.StartEarlyProcessing(history);
+
+            var maskedCommands = new List<string>()
+            {
+                "New-AzResourceGroup -Name ***",
+                "New-AzVM -Location:*** -Name:***"
+            };
+
+            Assert.Equal(maskedCommands, _service.Commands);
+            Assert.Equal(maskedCommands[1], _telemetryClient.RecordedSuggestion.HistoryLine);
+            Assert.Equal(history[1], _service.History.ToString());
+        }
+
+        /// <summary>
+        ///  Verify that we can handle the two unsupported command in sequences.
+        /// </summary>
+        [Fact]
+        public void VerifyRequestPredictionForTwoUnsupportedCommandInHistory()
+        {
+            IReadOnlyList<string> history = new List<string>()
+            {
+                "git status",
+                @"$a='ResourceGroup01'",
+            };
+
+            _telemetryClient.RecordedSuggestion = null;
+            _service.Commands = null;
+            _service.History = null;
+
+            _azPredictor.StartEarlyProcessing(history);
+
+            var maskedCommands = new List<string>()
+            {
+                AzPredictorConstants.CommandPlaceholder,
+                AzPredictorConstants.CommandPlaceholder,
+            };
+
+            Assert.Equal(maskedCommands, _service.Commands);
+            Assert.Equal(maskedCommands[1], _telemetryClient.RecordedSuggestion.HistoryLine);
+            Assert.Null(_service.History);
+        }
+
+        /// <summary>
+        /// Verify that we skip the unsupported commands.
+        /// </summary>
+        [Fact]
+        public void VerifyNotTakeUnsupportedCommands()
+        {
+            var history = new List<string>()
+            {
+                "New-AzResourceGroup -Name:resourceGroup01",
+                "New-AzVM -Name hello -Location WestUS"
+            };
+
+            _telemetryClient.RecordedSuggestion = null;
+            _service.Commands = null;
+            _service.History = null;
+
+            _azPredictor.StartEarlyProcessing(history);
+
+            history.Add("git status");
+            _azPredictor.StartEarlyProcessing(history);
+
+            history.Add(@"$a='NewResourceName'");
+            _azPredictor.StartEarlyProcessing(history);
+
+            // We don't take the last two unsupported command to request predictions.
+            // But we send the masked one in telemetry.
+
+            var maskedCommands = new List<string>()
+            {
+                "New-AzResourceGroup -Name:***",
+                "New-AzVM -Location *** -Name ***"
+            };
+
+            Assert.Equal(maskedCommands, _service.Commands);
+            Assert.Equal(AzPredictorConstants.CommandPlaceholder, _telemetryClient.RecordedSuggestion.HistoryLine);
+            Assert.Equal(history[1], _service.History.ToString());
+
+            // When there is a new supported command, we'll use that for prediction.
+
+            history.Add("Get-AzResourceGroup -Name ResourceGroup01");
+            _azPredictor.StartEarlyProcessing(history);
+
+            maskedCommands = new List<string>()
+            {
+                "New-AzVM -Location *** -Name ***",
+                "Get-AzResourceGroup -Name ***",
+            };
+
+            Assert.Equal(maskedCommands, _service.Commands);
+            Assert.Equal(maskedCommands[1], _telemetryClient.RecordedSuggestion.HistoryLine);
+            Assert.Equal(history.Last(), _service.History.ToString());
+        }
+
+        /// <summary>
+        /// Verify that we handle the three supported command in the same order.
+        /// </summary>
+        [Fact]
+        public void VerifyThreeSupportedCommands()
         {
-            IReadOnlyList<string> history = historyLine.Split('\n');
+            var history = new List<string>()
+            {
+                "New-AzResourceGroup -Name resourceGroup01",
+                "New-AzVM -Name:hello -Location:WestUS"
+            };
 
-            this._telemetryClient.RecordedSuggestion = null;
-            this._service.IsPredictionRequested = false;
+            _telemetryClient.RecordedSuggestion = null;
+            _service.Commands = null;
+            _service.History = null;
 
-            this._azPredictor.StartEarlyProcessing(history);
+            _azPredictor.StartEarlyProcessing(history);
 
-            Assert.True(this._service.IsPredictionRequested);
-            Assert.NotNull(this._telemetryClient.RecordedSuggestion);
+            history.Add("Get-AzResourceGroup -Name resourceGroup01");
+            _azPredictor.StartEarlyProcessing(history);
+
+            var maskedCommands = new List<string>()
+            {
+                "New-AzVM -Location:*** -Name:***",
+                "Get-AzResourceGroup -Name ***",
+            };
+
+            Assert.Equal(maskedCommands, _service.Commands);
+            Assert.Equal(maskedCommands[1], _telemetryClient.RecordedSuggestion.HistoryLine);
+            Assert.Equal(history.Last(), _service.History.ToString());
         }
 
         /// <summary>
-        /// Verify that the supported commands parameter values are masked.
+        /// Verify that we handle the sequence of one unsupported command and one supported command.
         /// </summary>
         [Fact]
-        public void VerifySupportedCommandMasked()
+        public void VerifyUnsupportedAndSupportedCommands()
         {
-            var input = "Get-AzVMExtension -ResourceGroupName 'ResourceGroup11' -VMName 'VirtualMachine22'";
-            var expected = "Get-AzVMExtension -ResourceGroupName *** -VMName ***";
+            var history = new List<string>()
+            {
+                "git status",
+                "New-AzVM -Name:hello -Location:WestUS"
+            };
 
-            this._telemetryClient.RecordedSuggestion = null;
-            this._service.IsPredictionRequested = false;
+            _telemetryClient.RecordedSuggestion = null;
+            _service.Commands = null;
+            _service.History = null;
 
-            this._azPredictor.StartEarlyProcessing(new List<string> { input } );
+            _azPredictor.StartEarlyProcessing(history);
 
-            Assert.True(this._service.IsPredictionRequested);
-            Assert.NotNull(this._telemetryClient.RecordedSuggestion);
-            Assert.Equal(expected, this._telemetryClient.RecordedSuggestion.HistoryLine);
+            var maskedCommands = new List<string>()
+            {
+                AzPredictorConstants.CommandPlaceholder,
+                "New-AzVM -Location:*** -Name:***"
+            };
 
-            input = "Get-AzStorageAccountKey -Name:'ContosoStorage' -ResourceGroupName:'ContosoGroup02'";
-            expected = "Get-AzStorageAccountKey -Name:*** -ResourceGroupName:***";
+            Assert.Equal(maskedCommands, _service.Commands);
+            Assert.Equal(maskedCommands[1], _telemetryClient.RecordedSuggestion.HistoryLine);
+            Assert.Equal(history.Last(), _service.History.ToString());
+        }
 
+        /// <summary>
+        /// Verify that we handle the sequence of one supported command and one unsupported command.
+        /// </summary>
+        [Fact]
+        public void VerifySupportedAndUnsupportedCommands()
+        {
+            var history = new List<string>()
+            {
+                "New-AzVM -Name hello -Location WestUS",
+                "git status",
+            };
+
+            _telemetryClient.RecordedSuggestion = null;
+            _service.Commands = null;
+            _service.History = null;
 
-            this._telemetryClient.RecordedSuggestion = null;
-            this._service.IsPredictionRequested = false;
+            _azPredictor.StartEarlyProcessing(history);
 
-            this._azPredictor.StartEarlyProcessing(new List<string> { input } );
+            var maskedCommands = new List<string>()
+            {
+                AzPredictorConstants.CommandPlaceholder,
+                "New-AzVM -Location *** -Name ***",
+            };
 
-            Assert.True(this._service.IsPredictionRequested);
-            Assert.NotNull(this._telemetryClient.RecordedSuggestion);
-            Assert.Equal(expected, this._telemetryClient.RecordedSuggestion.HistoryLine);
+            Assert.Equal(maskedCommands, _service.Commands);
+            Assert.Equal(AzPredictorConstants.CommandPlaceholder, _telemetryClient.RecordedSuggestion.HistoryLine);
+            Assert.Equal(history.First(), _service.History.ToString());
         }
 
         /// <summary>
@@ -140,8 +306,8 @@ public void VerifySupportedCommandMasked()
         public void VerifySuggestion(string userInput)
         {
             var predictionContext = PredictionContext.Create(userInput);
-            var expected = this._service.GetSuggestion(predictionContext.InputAst, 1, 1, CancellationToken.None);
-            var actual = this._azPredictor.GetSuggestion(predictionContext, CancellationToken.None);
+            var expected = _service.GetSuggestion(predictionContext.InputAst, 1, 1, CancellationToken.None);
+            var actual = _azPredictor.GetSuggestion(predictionContext, CancellationToken.None);
 
             Assert.Equal(expected.Count, actual.Count);
             Assert.Equal(expected.PredictiveSuggestions.First().SuggestionText, actual.First().SuggestionText);
@@ -154,7 +320,7 @@ public void VerifySuggestion(string userInput)
         public void VerifySuggestionOnIncompleteCommand()
         {
             // We need to get the suggestions for more than one. So we create a local version az predictor.
-            var localAzPredictor = new AzPredictor(this._service, this._telemetryClient, new Settings()
+            var localAzPredictor = new AzPredictor(_service, _telemetryClient, new Settings()
             {
                 SuggestionCount = 7,
                 MaxAllowedCommandDuplicate = 1,
@@ -170,7 +336,6 @@ public void VerifySuggestionOnIncompleteCommand()
             Assert.Equal(expected, actual.First().SuggestionText);
         }
 
-
         /// <summary>
         /// Verify when we cannot parse the user input correctly.
         /// </summary>
@@ -183,7 +348,7 @@ public void VerifySuggestionOnIncompleteCommand()
         public void VerifyMalFormattedCommandLine(string userInput)
         {
             var predictionContext = PredictionContext.Create(userInput);
-            var actual = this._azPredictor.GetSuggestion(predictionContext, CancellationToken.None);
+            var actual = _azPredictor.GetSuggestion(predictionContext, CancellationToken.None);
 
             Assert.Empty(actual);
         }
diff --git a/tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/Mocks/MockAzPredictorService.cs b/tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/Mocks/MockAzPredictorService.cs
index 0d0d3d74d724..f431390cfe0f 100644
--- a/tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/Mocks/MockAzPredictorService.cs
+++ b/tools/Az.Tools.Predictor/Az.Tools.Predictor.Test/Mocks/MockAzPredictorService.cs
@@ -12,8 +12,8 @@
 // limitations under the License.
 // ----------------------------------------------------------------------------------
 
-using System;
 using System.Collections.Generic;
+using System.Management.Automation.Language;
 
 namespace Microsoft.Azure.PowerShell.Tools.AzPredictor.Test.Mocks
 {
@@ -23,9 +23,14 @@ namespace Microsoft.Azure.PowerShell.Tools.AzPredictor.Test.Mocks
     sealed class MockAzPredictorService : AzPredictorService
     {
         /// <summary>
-        /// Gets or sets if a predictions is requested.
+        /// Gets or sets the commands in history to request prediction for.
         /// </summary>
-        public bool IsPredictionRequested { get; set; }
+        public IEnumerable<string> Commands { get; set; }
+
+        /// <summary>
+        /// Gets or sets the commands that's recorded in history.
+        /// </summary>
+        public CommandAst History { get; set; }
 
         /// <summary>
         /// Constructs a new instance of <see cref="MockAzPredictorService"/>
@@ -52,9 +57,9 @@ public MockAzPredictorService(string history, IList<PredictiveCommand> suggestio
         }
 
         /// <inheritdoc/>
-        public override void RequestPredictions(IEnumerable<string> history)
+        public override void RequestPredictions(IEnumerable<string> commands)
         {
-            this.IsPredictionRequested = true;
+            Commands = commands;
         }
 
         /// <inheritdoc/>
@@ -62,5 +67,11 @@ protected override void RequestAllPredictiveCommands()
         {
             // Do nothing since we've set the command and suggestion predictors.
         }
+
+        /// <inheritdoc/>
+        public override void RecordHistory(CommandAst history)
+        {
+            History = history;
+        }
     }
 }
diff --git a/tools/Az.Tools.Predictor/Az.Tools.Predictor/AzPredictor.cs b/tools/Az.Tools.Predictor/Az.Tools.Predictor/AzPredictor.cs
index a03eed3391a2..9fa881d3ed49 100644
--- a/tools/Az.Tools.Predictor/Az.Tools.Predictor/AzPredictor.cs
+++ b/tools/Az.Tools.Predictor/Az.Tools.Predictor/AzPredictor.cs
@@ -83,11 +83,21 @@ public void StartEarlyProcessing(IReadOnlyList<string> history)
 
             if (history.Count > 0)
             {
-                if (_lastTwoMaskedCommands.Any())
-                {
-                    _lastTwoMaskedCommands.Dequeue();
-                }
-                else
+                // We try to find the commands to request predictions for.
+                // We should only have "start_of_snippet" when there are no enough Az commands for prediction.
+                // We then ignore that when there are new "start_of_snippet".
+                // This is the scenario.
+                // 1. New-AzResourceGroup -Name ****
+                // 2. $resourceName="Test"
+                // 3. $resourceLocation="westus2"
+                // 4. New-AzVM -Name $resourceName -Location $resourceLocation
+                //
+                // We'll replace 2 and 3 with "start_of_snippet" but if we request prediction using 2 and 3, that'll reset the
+                // workflow. We want to predict only by Az commands. That's to use commands 1 and 4.
+
+                bool isLastTwoCommandsChanged = false;
+
+                if (_lastTwoMaskedCommands.Count == 0)
                 {
                     // This is the first time we populate our record. Push the second to last command in history to the
                     // queue. If there is only one command in history, push the command placeholder.
@@ -97,7 +107,11 @@ public void StartEarlyProcessing(IReadOnlyList<string> history)
                         string secondToLastLine = history.TakeLast(AzPredictorConstants.CommandHistoryCountToProcess).First();
                         var secondToLastCommand = GetAstAndMaskedCommandLine(secondToLastLine);
                         _lastTwoMaskedCommands.Enqueue(secondToLastCommand.Item2);
-                        _service.RecordHistory(secondToLastCommand.Item1);
+
+                        if (!string.Equals(AzPredictorConstants.CommandPlaceholder, secondToLastCommand.Item2, StringComparison.Ordinal))
+                        {
+                            _service.RecordHistory(secondToLastCommand.Item1);
+                        }
                     }
                     else
                     {
@@ -105,20 +119,41 @@ public void StartEarlyProcessing(IReadOnlyList<string> history)
                         // We only extract parameter values from the command line in _service.RecordHistory.
                         // So we don't need to do that for a placeholder.
                     }
+
+                    isLastTwoCommandsChanged = true;
                 }
 
                 string lastLine = history.Last();
                 var lastCommand = GetAstAndMaskedCommandLine(lastLine);
+                bool isLastCommandSupported = !string.Equals(AzPredictorConstants.CommandPlaceholder, lastCommand.Item2, StringComparison.Ordinal);
 
-                _lastTwoMaskedCommands.Enqueue(lastCommand.Item2);
-
-                if ((lastCommand.Item2 != null) && !string.Equals(AzPredictorConstants.CommandPlaceholder, lastCommand.Item2, StringComparison.Ordinal))
+                if (isLastCommandSupported)
                 {
+                    if (_lastTwoMaskedCommands.Count == 2)
+                    {
+                        // There are already two commands, dequeue the oldest one.
+                        _lastTwoMaskedCommands.Dequeue();
+                    }
+
+                    _lastTwoMaskedCommands.Enqueue(lastCommand.Item2);
+                    isLastTwoCommandsChanged = true;
+
                     _service.RecordHistory(lastCommand.Item1);
                 }
+                else if (_lastTwoMaskedCommands.Count == 1)
+                {
+                    isLastTwoCommandsChanged = true;
+                    var existingInQueue = _lastTwoMaskedCommands.Dequeue();
+                    _lastTwoMaskedCommands.Enqueue(AzPredictorConstants.CommandPlaceholder);
+                    _lastTwoMaskedCommands.Enqueue(existingInQueue);
+                }
 
                 _telemetryClient.OnHistory(new HistoryTelemetryData(lastCommand.Item2));
-                _service.RequestPredictions(_lastTwoMaskedCommands);
+
+                if (isLastTwoCommandsChanged)
+                {
+                    _service.RequestPredictions(_lastTwoMaskedCommands);
+                }
             }
 
             ValueTuple<CommandAst, string> GetAstAndMaskedCommandLine(string commandLine)
@@ -126,7 +161,7 @@ ValueTuple<CommandAst, string> GetAstAndMaskedCommandLine(string commandLine)
                 var asts = Parser.ParseInput(commandLine, out _, out _);
                 var allNestedAsts = asts?.FindAll((ast) => ast is CommandAst, true);
                 var commandAst = allNestedAsts?.LastOrDefault() as CommandAst;
-                string maskedCommandLine = null;
+                string maskedCommandLine = AzPredictorConstants.CommandPlaceholder;
 
                 var commandName = commandAst?.CommandElements?.FirstOrDefault().ToString();
 
@@ -134,10 +169,6 @@ ValueTuple<CommandAst, string> GetAstAndMaskedCommandLine(string commandLine)
                 {
                     maskedCommandLine = CommandLineUtilities.MaskCommandLine(commandAst);
                 }
-                else
-                {
-                    maskedCommandLine = AzPredictorConstants.CommandPlaceholder;
-                }
 
                 return ValueTuple.Create(commandAst, maskedCommandLine);
             }