Add Prometheus provider for external metrics check during autoscaling

cmd/server/command.go

@@ -26,6 +26,7 @@ func RegisterCommand(rootCmd *cobra.Command) error {
 	serverCfg.RegisterTLSConfig(cmd)
 	serverCfg.RegisterTelemetryConfig(cmd)
 	serverCfg.RegisterClusterConfig(cmd)
+	serverCfg.RegisterMetricProviderConfig(cmd)
 	logCfg.RegisterConfig(cmd)
 	rootCmd.AddCommand(cmd)
 
@@ -37,6 +38,7 @@ func runServer(_ *cobra.Command, _ []string) {
 	tlsConfig := serverCfg.GetTLSConfig()
 	telemetryConfig := serverCfg.GetTelemetryConfig()
 	clusterConfig := serverCfg.GetClusterConfig()
+	metricProviderConfig := serverCfg.GetMetricProviderConfig()
 
 	if err := verifyServerConfig(serverConfig); err != nil {
 		fmt.Println(err)
@@ -51,10 +53,11 @@ func runServer(_ *cobra.Command, _ []string) {
 	}
 
 	cfg := &server.Config{
-		Cluster:   &clusterConfig,
-		Server:    &serverConfig,
-		TLS:       &tlsConfig,
-		Telemetry: &telemetryConfig,
+		Cluster:        &clusterConfig,
+		MetricProvider: metricProviderConfig,
+		Server:         &serverConfig,
+		TLS:            &tlsConfig,
+		Telemetry:      &telemetryConfig,
 	}
 	srv := server.New(log.Logger, cfg)
 

docs/configuration/README.md

@@ -14,6 +14,7 @@ The Sherpa server can be configured by supplying either CLI flags or using envir
 * `--log-format` (string: "auto") - Specify the log format ("auto", "zerolog" or "human").
 * `--log-level` (string: "info") - Change the level used for logging.
 * `--log-use-color` (bool: true) - Use ANSI colors in logging output.
+* `--metric-provider-prometheus-addr` (string: "") The address of the Prometheus endpoint in the form <protocol>://<addr>:<port>.
 * `--policy-engine-api-enabled` (bool: true) - Enable the Sherpa API to manage scaling policies.
 * `--policy-engine-nomad-meta-enabled` (bool: false) - Enable Nomad job meta lookups to manage scaling policies.
 * `--policy-engine-strict-checking-enabled` (bool: true) - When enabled, all scaling activities must pass through policy checks.

docs/configuration/telemetry.md

@@ -278,16 +278,10 @@ Autoscale metrics allow operators to get insight into how the autoscaler is func
     <td>Summary</td>
   </tr>
   <tr>
-    <td>`sherpa.autoscale.evaluation.error`</td>
-    <td>Number of autoscaling evaluation errors across all jobs</td>
-    <td>Number of errors</td>
-    <td>Counter</td>
-  </tr>
-  <tr>
-    <td>`sherpa.autoscale.{job}.evaluation.error`</td>
-    <td>Number of autoscaling evaluation errors for the job named {job}</td>
-    <td>Number of errors</td>
-    <td>Counter</td>
+    <td>`sherpa.autoscale.{job}.{group}.evaluation`</td>
+    <td>The time taken to perform the autoscaling evaluation for the job named {job} and group named {group}</td>
+    <td>Milliseconds</td>
+    <td>Summary</td>
   </tr>
   <tr>
     <td>`sherpa.autoscale.trigger.error`</td>
@@ -313,4 +307,22 @@ Autoscale metrics allow operators to get insight into how the autoscaler is func
     <td>Number of successes</td>
     <td>Counter</td>
   </tr>
+  <tr>
+    <td>`sherpa.autoscale.prometheus.get_value`</td>
+    <td>The time taken to query Prometheus for a metric value</td>
+    <td>Milliseconds</td>
+    <td>Summary</td>
+  </tr>
+  <tr>
+    <td>`sherpa.autoscale.prometheus.error`</td>
+    <td>Number of errors querying Prometheus for a metric value</td>
+    <td>Number of errors</td>
+    <td>Counter</td>
+  </tr>
+    <tr>
+      <td>`sherpa.autoscale.prometheus.success`</td>
+      <td>Number of successful queries of Prometheus for a metric value</td>
+      <td>Number of successes</td>
+      <td>Counter</td>
+    </tr>
 </table>

pkg/api/policy.go

@@ -20,10 +20,21 @@ type JobGroupPolicy struct {
 	MinCount                          int
 	ScaleOutCount                     int
 	ScaleInCount                      int
-	ScaleOutCPUPercentageThreshold    int
-	ScaleOutMemoryPercentageThreshold int
-	ScaleInCPUPercentageThreshold     int
-	ScaleInMemoryPercentageThreshold  int
+	ScaleOutCPUPercentageThreshold    *int
+	ScaleOutMemoryPercentageThreshold *int
+	ScaleInCPUPercentageThreshold     *int
+	ScaleInMemoryPercentageThreshold  *int
+	ExternalChecks                    map[string]*ExternalCheck
+}
+
+// ExternalCheck represents an individual external check within a group scaling policy.
+type ExternalCheck struct {
+	Enabled            bool
+	Provider           string
+	Query              string
+	ComparisonOperator string
+	ComparisonValue    int
+	Action             string
 }
 
 func (p *Policies) List() (*map[string]map[string]*JobGroupPolicy, error) {