Introduce Target Based Scaling

Directory.Packages.props

@@ -4,7 +4,7 @@
     <PackageVersion Include="Microsoft.NET.Sdk.Functions" Version="4.1.3" />
     <PackageVersion Include="Microsoft.Azure.Functions.Worker.Extensions.Abstractions" Version="1.1.0" />
     <PackageVersion Include="Microsoft.ApplicationInsights" Version="2.17.0" />
-    <PackageVersion Include="Microsoft.Azure.WebJobs" Version="3.0.32" />
+    <PackageVersion Include="Microsoft.Azure.WebJobs" Version="3.0.36" />
     <PackageVersion Include="Microsoft.Data.SqlClient" Version="5.0.1" />
     <PackageVersion Include="Microsoft.SourceLink.GitHub" Version="1.1.1" />
     <PackageVersion Include="morelinq" Version="3.3.2" />

nuget.config

@@ -4,4 +4,5 @@
     <clear />
     <add key="nuget.org" value="https://api.nuget.org/v3/index.json" protocolVersion="3" />
   </packageSources>
-</configuration>
+</configuration>
+

performance/packages.lock.json

@@ -288,11 +288,12 @@
       },
       "Microsoft.Azure.WebJobs.Core": {
         "type": "Transitive",
-        "resolved": "3.0.32",
-        "contentHash": "pW5lyF0Tno1cC2VkmBLyv7E3o5ObDdbn3pfpUpKdksJo9ysCdQTpgc0Ib99wPHca6BgvoglicGbDYXuatanMfg==",
+        "resolved": "3.0.36",
+        "contentHash": "4ht/u677qxDL5rviXafYI5K9qtxee8peXQd8JDqG5KBRr5VXct2pgpEs5SnNu+gDO8dAVp+83N1IRiGG2Fl7Nw==",
         "dependencies": {
           "System.ComponentModel.Annotations": "4.4.0",
-          "System.Diagnostics.TraceSource": "4.3.0"
+          "System.Diagnostics.TraceSource": "4.3.0",
+          "System.Memory.Data": "1.0.1"
         }
       },
       "Microsoft.Azure.WebJobs.Extensions": {
@@ -2081,37 +2082,37 @@
       "microsoft.azure.webjobs.extensions.sql": {
         "type": "Project",
         "dependencies": {
-          "Microsoft.ApplicationInsights": "[2.17.0, )",
-          "Microsoft.AspNetCore.Http": "[2.2.2, )",
-          "Microsoft.Azure.WebJobs": "[3.0.32, )",
-          "Microsoft.Data.SqlClient": "[5.0.1, )",
-          "Newtonsoft.Json": "[13.0.1, )",
-          "System.Runtime.Caching": "[5.0.0, )",
-          "morelinq": "[3.3.2, )"
+          "Microsoft.ApplicationInsights": "2.17.0",
+          "Microsoft.AspNetCore.Http": "2.2.2",
+          "Microsoft.Azure.WebJobs": "3.0.36",
+          "Microsoft.Data.SqlClient": "5.0.1",
+          "Newtonsoft.Json": "13.0.1",
+          "System.Runtime.Caching": "5.0.0",
+          "morelinq": "3.3.2"
         }
       },
       "microsoft.azure.webjobs.extensions.sql.samples": {
         "type": "Project",
         "dependencies": {
-          "Microsoft.AspNetCore.Http": "[2.2.2, )",
-          "Microsoft.Azure.WebJobs.Extensions.Sql": "[99.99.99, )",
-          "Microsoft.Azure.WebJobs.Extensions.Storage": "[5.0.0, )",
-          "Microsoft.NET.Sdk.Functions": "[4.1.3, )",
-          "Newtonsoft.Json": "[13.0.1, )"
+          "Microsoft.AspNetCore.Http": "2.2.2",
+          "Microsoft.Azure.WebJobs.Extensions.Sql": "99.99.99",
+          "Microsoft.Azure.WebJobs.Extensions.Storage": "5.0.0",
+          "Microsoft.NET.Sdk.Functions": "4.1.3",
+          "Newtonsoft.Json": "13.0.1"
         }
       },
       "microsoft.azure.webjobs.extensions.sql.tests": {
         "type": "Project",
         "dependencies": {
-          "Microsoft.AspNetCore.Http": "[2.2.2, )",
-          "Microsoft.Azure.WebJobs.Extensions.Sql": "[99.99.99, )",
-          "Microsoft.Azure.WebJobs.Extensions.Sql.Samples": "[1.0.0, )",
-          "Microsoft.NET.Sdk.Functions": "[4.1.3, )",
-          "Microsoft.NET.Test.Sdk": "[17.0.0, )",
-          "Moq": "[4.14.3, )",
-          "Newtonsoft.Json": "[13.0.1, )",
-          "xunit": "[2.4.0, )",
-          "xunit.runner.visualstudio": "[2.4.0, )"
+          "Microsoft.AspNetCore.Http": "2.2.2",
+          "Microsoft.Azure.WebJobs.Extensions.Sql": "99.99.99",
+          "Microsoft.Azure.WebJobs.Extensions.Sql.Samples": "1.0.0",
+          "Microsoft.NET.Sdk.Functions": "4.1.3",
+          "Microsoft.NET.Test.Sdk": "17.0.0",
+          "Moq": "4.14.3",
+          "Newtonsoft.Json": "13.0.1",
+          "xunit": "2.4.0",
+          "xunit.runner.visualstudio": "2.4.0"
         }
       },
       "Microsoft.ApplicationInsights": {
@@ -2161,11 +2162,11 @@
       },
       "Microsoft.Azure.WebJobs": {
         "type": "CentralTransitive",
-        "requested": "[3.0.32, )",
-        "resolved": "3.0.32",
-        "contentHash": "uN8GsFqPFHHcSrwwj/+0tGe6F6cOwugqUiePPw7W3TL9YC594+Hw8GBK5S/fcDWXacqvRRGf9nDX8xP94/Yiyw==",
+        "requested": "[3.0.36, )",
+        "resolved": "3.0.36",
+        "contentHash": "S5wppmL8sUfanGmo/zDzpoWrcM+IOvGNFjpkD5XJFCFzrBqktZTEX6MpP/vF3RN6VZm2sFKegZYyce+RNhgE4Q==",
         "dependencies": {
-          "Microsoft.Azure.WebJobs.Core": "3.0.32",
+          "Microsoft.Azure.WebJobs.Core": "3.0.36",
           "Microsoft.Extensions.Configuration": "2.1.1",
           "Microsoft.Extensions.Configuration.Abstractions": "2.1.1",
           "Microsoft.Extensions.Configuration.EnvironmentVariables": "2.1.0",
@@ -2174,8 +2175,8 @@
           "Microsoft.Extensions.Logging": "2.1.1",
           "Microsoft.Extensions.Logging.Abstractions": "2.1.1",
           "Microsoft.Extensions.Logging.Configuration": "2.1.0",
-          "Newtonsoft.Json": "11.0.2",
-          "System.Memory.Data": "1.0.1",
+          "Newtonsoft.Json": "13.0.1",
+          "System.Memory.Data": "1.0.2",
           "System.Threading.Tasks.Dataflow": "4.8.0"
         }
       },

samples/samples-csharp/packages.lock.json

@@ -276,11 +276,12 @@
       },
       "Microsoft.Azure.WebJobs.Core": {
         "type": "Transitive",
-        "resolved": "3.0.32",
-        "contentHash": "pW5lyF0Tno1cC2VkmBLyv7E3o5ObDdbn3pfpUpKdksJo9ysCdQTpgc0Ib99wPHca6BgvoglicGbDYXuatanMfg==",
+        "resolved": "3.0.36",
+        "contentHash": "4ht/u677qxDL5rviXafYI5K9qtxee8peXQd8JDqG5KBRr5VXct2pgpEs5SnNu+gDO8dAVp+83N1IRiGG2Fl7Nw==",
         "dependencies": {
           "System.ComponentModel.Annotations": "4.4.0",
-          "System.Diagnostics.TraceSource": "4.3.0"
+          "System.Diagnostics.TraceSource": "4.3.0",
+          "System.Memory.Data": "1.0.1"
         }
       },
       "Microsoft.Azure.WebJobs.Extensions": {
@@ -1720,13 +1721,13 @@
       "microsoft.azure.webjobs.extensions.sql": {
         "type": "Project",
         "dependencies": {
-          "Microsoft.ApplicationInsights": "[2.17.0, )",
-          "Microsoft.AspNetCore.Http": "[2.2.2, )",
-          "Microsoft.Azure.WebJobs": "[3.0.32, )",
-          "Microsoft.Data.SqlClient": "[5.0.1, )",
-          "Newtonsoft.Json": "[13.0.1, )",
-          "System.Runtime.Caching": "[5.0.0, )",
-          "morelinq": "[3.3.2, )"
+          "Microsoft.ApplicationInsights": "2.17.0",
+          "Microsoft.AspNetCore.Http": "2.2.2",
+          "Microsoft.Azure.WebJobs": "3.0.36",
+          "Microsoft.Data.SqlClient": "5.0.1",
+          "Newtonsoft.Json": "13.0.1",
+          "System.Runtime.Caching": "5.0.0",
+          "morelinq": "3.3.2"
         }
       },
       "Microsoft.ApplicationInsights": {
@@ -1763,11 +1764,11 @@
       },
       "Microsoft.Azure.WebJobs": {
         "type": "CentralTransitive",
-        "requested": "[3.0.32, )",
-        "resolved": "3.0.32",
-        "contentHash": "uN8GsFqPFHHcSrwwj/+0tGe6F6cOwugqUiePPw7W3TL9YC594+Hw8GBK5S/fcDWXacqvRRGf9nDX8xP94/Yiyw==",
+        "requested": "[3.0.36, )",
+        "resolved": "3.0.36",
+        "contentHash": "S5wppmL8sUfanGmo/zDzpoWrcM+IOvGNFjpkD5XJFCFzrBqktZTEX6MpP/vF3RN6VZm2sFKegZYyce+RNhgE4Q==",
         "dependencies": {
-          "Microsoft.Azure.WebJobs.Core": "3.0.32",
+          "Microsoft.Azure.WebJobs.Core": "3.0.36",
           "Microsoft.Extensions.Configuration": "2.1.1",
           "Microsoft.Extensions.Configuration.Abstractions": "2.1.1",
           "Microsoft.Extensions.Configuration.EnvironmentVariables": "2.1.0",
@@ -1776,8 +1777,8 @@
           "Microsoft.Extensions.Logging": "2.1.1",
           "Microsoft.Extensions.Logging.Abstractions": "2.1.1",
           "Microsoft.Extensions.Logging.Configuration": "2.1.0",
-          "Newtonsoft.Json": "11.0.2",
-          "System.Memory.Data": "1.0.1",
+          "Newtonsoft.Json": "13.0.1",
+          "System.Memory.Data": "1.0.2",
           "System.Threading.Tasks.Dataflow": "4.8.0"
         }
       },

samples/samples-outofproc/packages.lock.json

@@ -830,9 +830,9 @@
       "microsoft.azure.functions.worker.extension.sql": {
         "type": "Project",
         "dependencies": {
-          "Microsoft.Azure.Functions.Worker.Extensions.Abstractions": "[1.1.0, )",
-          "Microsoft.Data.SqlClient": "[5.0.1, )",
-          "System.Drawing.Common": "[5.0.3, )"
+          "Microsoft.Azure.Functions.Worker.Extensions.Abstractions": "1.1.0",
+          "Microsoft.Data.SqlClient": "5.0.1",
+          "System.Drawing.Common": "5.0.3"
         }
       },
       "Microsoft.Azure.Functions.Worker.Extensions.Abstractions": {

src/TriggerBinding/SqlTableChangeMonitor.cs

@@ -169,69 +169,6 @@ public void Dispose()
             this._cancellationTokenSourceCheckForChanges.Cancel();
         }
 
-        public async Task<long> GetUnprocessedChangeCountAsync()
-        {
-            long unprocessedChangeCount = 0L;
-
-            try
-            {
-                long getUnprocessedChangesDurationMs = 0L;
-
-                using (var connection = new SqlConnection(this._connectionString))
-                {
-                    this._logger.LogDebugWithThreadId("BEGIN OpenGetUnprocessedChangesConnection");
-                    await connection.OpenAsync();
-                    this._logger.LogDebugWithThreadId("END OpenGetUnprocessedChangesConnection");
-
-                    // Use a transaction to automatically release the app lock when we're done executing the query
-                    using (SqlTransaction transaction = connection.BeginTransaction(IsolationLevel.RepeatableRead))
-                    {
-                        try
-                        {
-                            using (SqlCommand getUnprocessedChangesCommand = this.BuildGetUnprocessedChangesCommand(connection, transaction))
-                            {
-                                this._logger.LogInformation("Getting change count");
-                                this._logger.LogDebugWithThreadId($"BEGIN GetUnprocessedChangeCount Query={getUnprocessedChangesCommand.CommandText}");
-                                var commandSw = Stopwatch.StartNew();
-                                unprocessedChangeCount = (long)await getUnprocessedChangesCommand.ExecuteScalarAsync();
-                                getUnprocessedChangesDurationMs = commandSw.ElapsedMilliseconds;
-                            }
-
-                            this._logger.LogDebugWithThreadId($"END GetUnprocessedChangeCount Duration={getUnprocessedChangesDurationMs}ms Count={unprocessedChangeCount}");
-                            transaction.Commit();
-                        }
-                        catch (Exception)
-                        {
-                            try
-                            {
-                                transaction.Rollback();
-                            }
-                            catch (Exception ex2)
-                            {
-                                this._logger.LogError($"GetUnprocessedChangeCount : Failed to rollback transaction due to exception: {ex2.GetType()}. Exception message: {ex2.Message}");
-                                TelemetryInstance.TrackException(TelemetryErrorName.GetUnprocessedChangeCountRollback, ex2, this._telemetryProps);
-                            }
-                            throw;
-                        }
-                    }
-                }
-
-                var measures = new Dictionary<TelemetryMeasureName, double>
-                {
-                    [TelemetryMeasureName.GetUnprocessedChangesDurationMs] = getUnprocessedChangesDurationMs,
-                    [TelemetryMeasureName.UnprocessedChangeCount] = unprocessedChangeCount,
-                };
-            }
-            catch (Exception ex)
-            {
-                this._logger.LogError($"Failed to query count of unprocessed changes for table '{this._userTable.FullName}' due to exception: {ex.GetType()}. Exception message: {ex.Message}");
-                TelemetryInstance.TrackException(TelemetryErrorName.GetUnprocessedChangeCount, ex, this._telemetryProps);
-                throw;
-            }
-
-            return unprocessedChangeCount;
-        }
-
         /// <summary>
         /// Executed once every <see cref="_pollingIntervalInMs"/> period. If the state of the change monitor is
         /// <see cref="State.CheckingForChanges"/>, then the method query the change/leases tables for changes on the
@@ -819,38 +756,6 @@ LEFT OUTER JOIN {this._userTable.BracketQuotedFullName} AS u ON {userTableJoinCo
             return new SqlCommand(getChangesQuery, connection, transaction);
         }
 
-        /// <summary>
-        /// Builds the query to get count of unprocessed changes in the user's table. This one mimics the query that is
-        /// used by workers to get the changes for processing.
-        /// </summary>
-        /// <param name="connection">The connection to add to the returned SqlCommand</param>
-        /// <param name="transaction">The transaction to add to the returned SqlCommand</param>
-        /// <returns>The SqlCommand populated with the query and appropriate parameters</returns>
-        private SqlCommand BuildGetUnprocessedChangesCommand(SqlConnection connection, SqlTransaction transaction)
-        {
-            string leasesTableJoinCondition = string.Join(" AND ", this._primaryKeyColumns.Select(col => $"c.{col.name.AsBracketQuotedString()} = l.{col.name.AsBracketQuotedString()}"));
-
-            string getUnprocessedChangesQuery = $@"
-                {AppLockStatements}
-
-                DECLARE @last_sync_version bigint;
-                SELECT @last_sync_version = LastSyncVersion
-                FROM {GlobalStateTableName}
-                WHERE UserFunctionID = '{this._userFunctionId}' AND UserTableID = {this._userTableId};
-
-                SELECT COUNT_BIG(*)
-                FROM CHANGETABLE(CHANGES {this._userTable.BracketQuotedFullName}, @last_sync_version) AS c
-                LEFT OUTER JOIN {this._leasesTableName} AS l ON {leasesTableJoinCondition}
-                WHERE
-                    (l.{LeasesTableLeaseExpirationTimeColumnName} IS NULL AND
-                       (l.{LeasesTableChangeVersionColumnName} IS NULL OR l.{LeasesTableChangeVersionColumnName} < c.{SysChangeVersionColumnName}) OR
-                        l.{LeasesTableLeaseExpirationTimeColumnName} < SYSDATETIME()) AND
-                    (l.{LeasesTableAttemptCountColumnName} IS NULL OR l.{LeasesTableAttemptCountColumnName} < {MaxChangeProcessAttemptCount});
-            ";
-
-            return new SqlCommand(getUnprocessedChangesQuery, connection, transaction);
-        }
-
         /// <summary>
         /// Builds the query to acquire leases on the rows in "_rows" if changes are detected in the user's table
         /// (<see cref="RunChangeConsumptionLoopAsync()"/>).