add slow starting warm-up feature

Signed-off-by: Cybwan <[email protected]>
flomesh-io · Jan 6, 2025 · e8a9717 · e8a9717
1 parent 2cf92eb
commit e8a9717
Show file tree

Hide file tree

Showing 42 changed files with 1,079 additions and 35 deletions.
diff --git a/charts/fsm/README.md b/charts/fsm/README.md
@@ -118,6 +118,7 @@ The following table lists the configurable parameters of the fsm chart and their
 | fsm.featureFlags.enableSidecarActiveHealthChecks | bool | `false` | Enable Sidecar active health checks |
 | fsm.featureFlags.enableSidecarPrettyConfig | bool | `true` | Enable Sidecar Pretty Config |
 | fsm.featureFlags.enableSnapshotCacheMode | bool | `false` | Enables SnapshotCache feature for Sidecar xDS server. |
+| fsm.featureFlags.enableTrafficWarmupPolicy | bool | `false` | Enables traffic warmup feature |
 | fsm.featureFlags.enableValidateGRPCRouteHostnames | bool | `true` | Enable validate GRPC route hostnames, enforce the hostname is DNS name not IP address |
 | fsm.featureFlags.enableValidateGatewayListenerHostname | bool | `true` | Enable validate Gateway listener hostname, enforce the hostname is DNS name not IP address |
 | fsm.featureFlags.enableValidateHTTPRouteHostnames | bool | `true` | Enable validate HTTP route hostnames, enforce the hostname is DNS name not IP address |
@@ -487,6 +488,9 @@ The following table lists the configurable parameters of the fsm chart and their
 | fsm.vault.secret.key | string | `""` | The Kubernetes secret key with the value bring the Vault token |
 | fsm.vault.secret.name | string | `""` | The Kubernetes secret name storing the Vault token used in FSM |
 | fsm.vault.token | string | `""` | token that should be used to connect to Vault |
+| fsm.warmup | object | `{"duration":"90s","enable":false,"maxWeight":100,"minWeight":10}` | Global Traffic Warmup policy |
+| fsm.warmup.maxWeight | int | `100` | MaxWeight configures the maximum percentage of origin weight -- If unspecified, defaults to 100 |
+| fsm.warmup.minWeight | int | `10` | MinWeight configures the minimum percentage of origin weight -- If unspecified, defaults to 10 |
 | fsm.webhookConfigNamePrefix | string | `"fsm-webhook"` | Prefix used in name of the webhook configuration resources |
 | smi.validateTrafficTarget | bool | `true` | Enables validation of SMI Traffic Target |
 

diff --git a/charts/fsm/templates/fsm-rbac.yaml b/charts/fsm/templates/fsm-rbac.yaml
@@ -89,10 +89,10 @@ rules:
 
   # FSM's custom policy API
   - apiGroups: ["policy.flomesh.io"]
-    resources: ["egresses", "egressgateways", "ingressbackends", "accesscontrols", "accesscerts", "isolations", "retries", "upstreamtrafficsettings"]
+    resources: ["egresses", "egressgateways", "ingressbackends", "accesscontrols", "accesscerts", "isolations", "retries", "upstreamtrafficsettings", "trafficwarmups"]
     verbs: ["list", "get", "watch"]
   - apiGroups: ["policy.flomesh.io"]
-    resources: ["ingressbackends/status", "accesscontrols/status", "accesscerts/status", "upstreamtrafficsettings/status"]
+    resources: ["ingressbackends/status", "accesscontrols/status", "accesscerts/status", "upstreamtrafficsettings/status", "trafficwarmup/status"]
     verbs: ["update"]
 
   # FSM's MultiCluster resource API

diff --git a/charts/fsm/templates/preset-mesh-config.yaml b/charts/fsm/templates/preset-mesh-config.yaml
@@ -46,6 +46,12 @@ data:
         "outboundIPRangeInclusionList": {{.Values.fsm.outboundIPRangeInclusionList | mustToJson}},
         "networkInterfaceExclusionList": {{.Values.fsm.networkInterfaceExclusionList | mustToJson}}
       },
+      "warmup": {
+        "enable": {{.Values.fsm.warmup.enable | mustToJson}},
+        "duration": {{.Values.fsm.warmup.duration | mustToJson}},
+        "minWeight": {{.Values.fsm.warmup.minWeight | mustToJson}},
+        "maxWeight": {{.Values.fsm.warmup.maxWeight | mustToJson}}
+      },
       "observability": {
         "fsmLogLevel": {{.Values.fsm.controllerLogLevel | mustToJson}},
         "tracing": {
@@ -86,6 +92,7 @@ data:
         "enableIngressBackendPolicy": {{.Values.fsm.featureFlags.enableIngressBackendPolicy | mustToJson}},
         "enableAccessControlPolicy": {{.Values.fsm.featureFlags.enableAccessControlPolicy | mustToJson}},
         "enableAccessCertPolicy": {{.Values.fsm.featureFlags.enableAccessCertPolicy | mustToJson}},
+        "enableTrafficWarmupPolicy": {{.Values.fsm.featureFlags.enableTrafficWarmupPolicy | mustToJson}},
         "enableSidecarPrettyConfig": {{.Values.fsm.featureFlags.enableSidecarPrettyConfig | mustToJson}},
         "enableSidecarActiveHealthChecks": {{.Values.fsm.featureFlags.enableSidecarActiveHealthChecks | mustToJson}},
         "enableRetryPolicy": {{.Values.fsm.featureFlags.enableRetryPolicy | mustToJson}},

diff --git a/charts/fsm/values.schema.json b/charts/fsm/values.schema.json
@@ -1140,6 +1140,47 @@
                       }
                     }
                 },
+                "warmup": {
+                  "$id": "#/properties/fsm/properties/warmup",
+                  "type": "object",
+                  "title": "The traffic warmup schema",
+                  "required": [
+                    "enable",
+                    "duration",
+                    "minWeight",
+                    "maxWeight"
+                  ],
+                  "properties": {
+                    "enable": {
+                      "$id": "#/properties/fsm/properties/warmup/properties/enable",
+                      "type": "boolean",
+                      "title": "The enable schema for traffic warmup",
+                      "examples": [
+                        true
+                      ]
+                    },
+                    "duration": {
+                      "$id": "#/properties/fsm/properties/warmup/properties/duration",
+                      "type": "string",
+                      "title": "The duration schema",
+                      "description": "The traffic warmup duration.",
+                      "examples": [
+                        "90s"
+                      ]
+                    },
+                    "minWeight": {
+                      "$id": "#/properties/fsm/properties/warmup/properties/minWeight",
+                      "type": "integer",
+                      "title": "minWeight configures the minimum percentage"
+                    },
+                    "maxWeight": {
+                      "$id": "#/properties/fsm/properties/warmup/properties/maxWeight",
+                      "type": "integer",
+                      "title": "maxWeight configures the maximum percentage"
+                    }
+                  },
+                  "additionalProperties": false
+                },
                 "trustDomain": {
                     "$id": "#/properties/fsm/properties/trustDomain",
                     "type": "string",
@@ -2417,6 +2458,7 @@
                         "enableIngressBackendPolicy",
                         "enableAccessControlPolicy",
                         "enableAccessCertPolicy",
+                        "enableTrafficWarmupPolicy",
                         "enableSidecarPrettyConfig",
                         "enableSidecarActiveHealthChecks",
                         "enableSnapshotCacheMode",
@@ -2477,6 +2519,15 @@
                                 true
                             ]
                         },
+                        "enableTrafficWarmupPolicy": {
+                          "$id": "#/properties/fsm/properties/featureFlags/properties/enableTrafficWarmupPolicy",
+                          "type": "boolean",
+                          "title": "Enable traffic warmup feature",
+                          "description": "Enable traffic warmup feature",
+                          "examples": [
+                            true
+                          ]
+                        },
                         "enableSidecarPrettyConfig": {
                           "$id": "#/properties/fsm/properties/featureFlags/properties/enableSidecarPrettyConfig",
                           "type": "boolean",

diff --git a/charts/fsm/values.yaml b/charts/fsm/values.yaml
@@ -112,6 +112,17 @@ fsm:
     # -- codebase is the folder used by fsmController.
     codebase: ""
 
+  # -- Global Traffic Warmup policy
+  warmup:
+    enable: false
+    duration: 90s
+    # -- MinWeight configures the minimum percentage of origin weight
+    # -- If unspecified, defaults to 10
+    minWeight: 10
+    # -- MaxWeight configures the maximum percentage of origin weight
+    # -- If unspecified, defaults to 100
+    maxWeight: 100
+
   pluginChains:
     inbound-tcp:
       - plugin: modules/inbound-tls-termination
@@ -780,6 +791,8 @@ fsm:
     enableAccessControlPolicy: true
     # When enabled, FSM can issue certificates for external services.
     enableAccessCertPolicy: false
+    # -- Enables traffic warmup feature
+    enableTrafficWarmupPolicy: false
     # -- Enable Sidecar Pretty Config
     enableSidecarPrettyConfig: true
     # -- Enable Sidecar active health checks

diff --git a/cmd/fsm-bootstrap/crds/config.flomesh.io_meshconfigs.yaml b/cmd/fsm-bootstrap/crds/config.flomesh.io_meshconfigs.yaml
@@ -1473,6 +1473,10 @@ spec:
                     description: EnableSnapshotCacheMode defines if XDS server starts
                       with snapshot cache.
                     type: boolean
+                  enableTrafficWarmupPolicy:
+                    description: EnableTrafficWarmupPolicy defines if FSM will use
+                      the TrafficWarmup API to allow traffic warmup
+                    type: boolean
                   enableValidateGRPCRouteHostnames:
                     description: EnableValidateGRPCRouteHostnames defines if validate
                       grpc route hostnames is enabled.
@@ -1506,6 +1510,7 @@ spec:
                 - enableSidecarActiveHealthChecks
                 - enableSidecarPrettyConfig
                 - enableSnapshotCacheMode
+                - enableTrafficWarmupPolicy
                 - enableValidateGRPCRouteHostnames
                 - enableValidateGatewayListenerHostname
                 - enableValidateHTTPRouteHostnames
@@ -2451,6 +2456,43 @@ spec:
                 - outboundPortExclusionList
                 - serviceAccessMode
                 type: object
+              warmup:
+                description: Warmup defines the traffic warm up policy
+                properties:
+                  aggression:
+                    default: 1
+                    description: |-
+                      Aggression controls the speed of traffic increase over the warmup duration. Defaults to 1.0, so that endpoints would
+                      get linearly increasing amount of traffic. When increasing the value for this parameter,
+                      the speed of traffic ramp-up increases non-linearly.
+                    minimum: 1
+                    type: number
+                  duration:
+                    default: 90s
+                    format: duration
+                    type: string
+                  enable:
+                    default: true
+                    type: boolean
+                  maxWeight:
+                    default: 100
+                    description: |-
+                      MaxWeight configures the maximum percentage of origin weight
+                      If unspecified, defaults to 100
+                    format: int64
+                    maximum: 100
+                    minimum: 0
+                    type: integer
+                  minWeight:
+                    default: 10
+                    description: |-
+                      MinWeight configures the minimum percentage of origin weight
+                      If unspecified, defaults to 10
+                    format: int64
+                    maximum: 100
+                    minimum: 0
+                    type: integer
+                type: object
             required:
             - connector
             - image

diff --git a/cmd/fsm-bootstrap/crds/policy.flomesh.io_trafficwarmups.yaml b/cmd/fsm-bootstrap/crds/policy.flomesh.io_trafficwarmups.yaml
@@ -0,0 +1,97 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.16.5
+  labels:
+    app.kubernetes.io/name: flomesh.io
+  name: trafficwarmups.policy.flomesh.io
+spec:
+  group: policy.flomesh.io
+  names:
+    kind: TrafficWarmup
+    listKind: TrafficWarmupList
+    plural: trafficwarmups
+    shortNames:
+    - trafficwarmup
+    singular: trafficwarmup
+  scope: Namespaced
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: TrafficWarmup is the type used to represent a traffic warmup
+          policy.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: |-
+              Specification of the desired behavior of the traffic raffic warmup.
+              More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status
+            properties:
+              aggression:
+                default: 1
+                description: |-
+                  Aggression controls the speed of traffic increase over the warmup duration. Defaults to 1.0, so that endpoints would
+                  get linearly increasing amount of traffic. When increasing the value for this parameter,
+                  the speed of traffic ramp-up increases non-linearly.
+                minimum: 1
+                type: number
+              duration:
+                default: 90s
+                format: duration
+                type: string
+              enable:
+                default: true
+                type: boolean
+              maxWeight:
+                default: 100
+                description: |-
+                  MaxWeight configures the maximum percentage of origin weight
+                  If unspecified, defaults to 100
+                format: int64
+                maximum: 100
+                minimum: 0
+                type: integer
+              minWeight:
+                default: 10
+                description: |-
+                  MinWeight configures the minimum percentage of origin weight
+                  If unspecified, defaults to 10
+                format: int64
+                maximum: 100
+                minimum: 0
+                type: integer
+            type: object
+          status:
+            description: Status defines the current state of TrafficWarmup.
+            properties:
+              currentStatus:
+                description: CurrentStatus defines the current status of a traffic
+                  warmup resource.
+                type: string
+              reason:
+                description: Reason defines the reason for the current status of a
+                  raffic warmup resource.
+                type: string
+            type: object
+        type: object
+    served: true
+    storage: true
diff --git a/dockerfiles/Dockerfile.fsm-bootstrap b/dockerfiles/Dockerfile.fsm-bootstrap
@@ -7,6 +7,7 @@ ARG TARGETARCH
 
 WORKDIR /fsm
 COPY . .
+RUN go env -w GOPROXY=https://goproxy.cn
 RUN --mount=type=cache,target=/root/.cache/go-build \
     --mount=type=cache,target=/go/pkg \
     CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -v -o fsm-bootstrap -ldflags "$LDFLAGS" ./cmd/fsm-bootstrap

diff --git a/dockerfiles/Dockerfile.fsm-connector b/dockerfiles/Dockerfile.fsm-connector
@@ -7,6 +7,7 @@ ARG TARGETARCH
 
 WORKDIR /fsm
 COPY . .
+RUN go env -w GOPROXY=https://goproxy.cn
 RUN --mount=type=cache,target=/root/.cache/go-build \
     --mount=type=cache,target=/go/pkg \
     CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -v -o fsm-connector -ldflags "$LDFLAGS" ./cmd/fsm-connector

diff --git a/dockerfiles/Dockerfile.fsm-controller b/dockerfiles/Dockerfile.fsm-controller
@@ -7,6 +7,7 @@ ARG TARGETARCH
 
 WORKDIR /fsm
 COPY . .
+RUN go env -w GOPROXY=https://goproxy.cn
 RUN --mount=type=cache,target=/root/.cache/go-build \
     --mount=type=cache,target=/go/pkg \
     CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -v -o fsm-controller -ldflags "$LDFLAGS" ./cmd/fsm-controller

diff --git a/dockerfiles/Dockerfile.fsm-gateway b/dockerfiles/Dockerfile.fsm-gateway
@@ -12,6 +12,7 @@ WORKDIR /fsm
 # Copy the Go Modules manifests
 COPY go.mod go.mod
 COPY go.sum go.sum
+RUN go env -w GOPROXY=https://goproxy.cn
 # cache deps before building and copying source so that we don't need to re-download as much
 # and so that source changes don't invalidate our downloaded layer
 RUN --mount=type=cache,target=/go/pkg go mod download

diff --git a/dockerfiles/Dockerfile.fsm-injector b/dockerfiles/Dockerfile.fsm-injector
@@ -7,6 +7,7 @@ ARG TARGETARCH
 
 WORKDIR /fsm
 COPY . .
+RUN go env -w GOPROXY=https://goproxy.cn
 RUN --mount=type=cache,target=/root/.cache/go-build \
     --mount=type=cache,target=/go/pkg \
     CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -v -o fsm-injector -ldflags "$LDFLAGS" ./cmd/fsm-injector

diff --git a/dockerfiles/Dockerfile.fsm-preinstall b/dockerfiles/Dockerfile.fsm-preinstall
@@ -7,6 +7,7 @@ ARG TARGETARCH
 
 WORKDIR /fsm
 COPY . .
+RUN go env -w GOPROXY=https://goproxy.cn
 RUN --mount=type=cache,target=/root/.cache/go-build \
     --mount=type=cache,target=/go/pkg \
     CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -v -o fsm-preinstall -ldflags "$LDFLAGS" ./cmd/fsm-preinstall

diff --git a/dockerfiles/Dockerfile.fsm-xnetmgmt b/dockerfiles/Dockerfile.fsm-xnetmgmt
@@ -7,6 +7,7 @@ ARG TARGETARCH
 
 WORKDIR /fsm
 COPY . .
+RUN go env -w GOPROXY=https://goproxy.cn
 RUN --mount=type=cache,target=/root/.cache/go-build \
     --mount=type=cache,target=/go/pkg \
     CGO_ENABLED=0 GOOS=$TARGETOS GOARCH=$TARGETARCH go build -v -o fsm-xnet-mgmt -ldflags "$LDFLAGS" ./cmd/fsm-xnetmgmt

diff --git a/pkg/announcements/types.go b/pkg/announcements/types.go
@@ -333,6 +333,15 @@ const (
 	// RetryPolicyUpdated is the type of announcement emitted when we observe an update to retries.policy.flomesh.io
 	RetryPolicyUpdated Kind = "retry-updated"
 
+	// TrafficWarmupAdded is the type of announcement emitted when we observe an addition of trafficwarmups.policy.flomesh.io
+	TrafficWarmupAdded Kind = "trafficwarmup-added"
+
+	// TrafficWarmupDeleted the type of announcement emitted when we observe a deletion of trafficwarmups.policy.flomesh.io
+	TrafficWarmupDeleted Kind = "trafficwarmup-deleted"
+
+	// TrafficWarmupUpdated is the type of announcement emitted when we observe an update to trafficwarmups.policy.flomesh.io
+	TrafficWarmupUpdated Kind = "trafficwarmup-updated"
+
 	// UpstreamTrafficSettingAdded is the type of announcement emitted when we observe an addition of upstreamtrafficsettings.policy.flomesh.io
 	UpstreamTrafficSettingAdded Kind = "upstreamtrafficsetting-added"