feat: init node level sidecar. (#481)

Co-authored-by: Lin Yang <[email protected]>

Signed-off-by: Bai Li <[email protected]>

.env.example

@@ -84,10 +84,6 @@ export BOOKWAREHOUSE_NAMESPACE=bookwarehouse
 # Default: 0 (200 with permissive traffic policy mode)
 # export BOOKTHIEF_EXPECTED_RESPONSE_CODE=0
 
-# optional: ENABLE_DEBUG_SERVER (true/false)
-# Default: false
-# export ENABLE_DEBUG_SERVER=true
-
 # optional: ENABLE_EGRESS (true/false)
 # Default: true
 # export ENABLE_EGRESS=false

charts/fsm/README.md

@@ -98,7 +98,6 @@ The following table lists the configurable parameters of the fsm chart and their
 | fsm.egressGateway.port | int | `1080` |  |
 | fsm.egressGateway.replicaCount | int | `1` | FSM Egress Gateway's replica count (ignored when autoscale.enable is true) |
 | fsm.egressGateway.resources | object | `{"limits":{"cpu":"1000m","memory":"512M"},"requests":{"cpu":"300m","memory":"128M"}}` | FSM Egress Gateway's container resource parameters. |
-| fsm.enableDebugServer | bool | `false` | Enable the debug HTTP server on FSM controller |
 | fsm.enableEgress | bool | `true` | Enable egress in the mesh |
 | fsm.enableFluentbit | bool | `false` | Enable Fluent Bit sidecar deployment on FSM controller's pod |
 | fsm.enableMultiClusters | bool | `false` |  |
@@ -251,6 +250,7 @@ The following table lists the configurable parameters of the fsm chart and their
 | fsm.fsmXnetwork.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.labelSelector.matchExpressions[0].values[0] | string | `"fsm-xnetwork"` |  |
 | fsm.fsmXnetwork.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].podAffinityTerm.topologyKey | string | `"kubernetes.io/hostname"` |  |
 | fsm.fsmXnetwork.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution[0].weight | int | `100` |  |
+| fsm.fsmXnetwork.initResources | object | `{"limits":{"cpu":"500m","memory":"512M"},"requests":{"cpu":"200m","memory":"128M"}}` | FSM xnetwork's init-container resource parameters. |
 | fsm.fsmXnetwork.tolerations | list | `[]` | Node tolerations applied to control plane pods. The specified tolerations allow pods to schedule onto nodes with matching taints. |
 | fsm.fsmXnetwork.xmgt | object | `{"resource":{"limits":{"cpu":"1.5","memory":"1G"},"requests":{"cpu":"0.5","memory":"256M"}}}` | xmgt |
 | fsm.fsmXnetwork.xnet | object | `{"filter":{"ports":{"inbound":"mesh","outbound":"mesh"}},"flush":{"conntrack":{"tcp":{"batchSize":4096,"crontab":"30 3 */1 * *","idleSeconds":3600},"udp":{"batchSize":4096,"crontab":"*/2 * * * *","idleSeconds":120}}},"image":{"name":"xnet","registry":"flomesh","tag":"latest"},"nodePaths":{"k3s":{"cniBin":"/bin","cniNetd":"/var/lib/rancher/k3s/agent/etc/cni/net.d","enable":true,"kubeToken":"/var/lib/rancher/k3s/server/token","sysFs":"/opt","sysRun":"/var/run"},"k8s":{"cniBin":"/opt/cni/bin","cniNetd":"/etc/cni/net.d","kubeToken":"/var/run/secrets/kubernetes.io/serviceaccount/token","sysFs":"/opt","sysRun":"/var/run"}},"resource":{"limits":{"cpu":"1.5","memory":"1G"},"requests":{"cpu":"0.5","memory":"256M"}}}` | xnet |

charts/fsm/templates/fsm-rbac.yaml

@@ -132,6 +132,11 @@ rules:
     resources: ["consulconnectors/status", "eurekaconnectors/status", "nacosconnectors/status", "machineconnectors/status", "gatewayconnectors/status"]
     verbs: ["update"]
 
+  # FSM's custom xnetwork API
+  - apiGroups: ["xnetwork.flomesh.io"]
+    resources: ["accesscontrols" ]
+    verbs: ["list", "get", "watch"]
+
   # FSM's NamespacedIngress API
   - apiGroups: [ "networking.flomesh.io" ]
     resources: [ "namespacedingresses" ]

charts/fsm/templates/fsm-xnetwork.yaml

@@ -22,6 +22,13 @@ spec:
       {{- toYaml .Values.fsm.fsmXnetwork.affinity | nindent 8 }}
       {{- end }}
       hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      initContainers:
+      - name: fsm-init
+        image: "{{ include "fsmCurl.image" . }}"
+        command: [ "curl", "http://fsm-bootstrap.{{ include "fsm.namespace" . }}:9091/healthz", "--connect-timeout", "2", "--retry", "50", "--retry-connrefused", "--retry-delay", "5"]
+        resources:
+          {{- toYaml .Values.fsm.fsmXnetwork.initResources | nindent 12 }}
       containers:
       - name: fsm-xmgt
         image: "{{ include "fsmXnetwork.xmgt.image" . }}"
@@ -92,7 +99,6 @@ spec:
           - mountPath: /host/run
             name: host-run
             mountPropagation: Bidirectional
-      dnsPolicy: ClusterFirst
       priorityClassName: system-node-critical
       restartPolicy: Always
       serviceAccountName: {{ .Release.Name }}

charts/fsm/templates/preset-mesh-config.yaml

@@ -45,7 +45,6 @@ data:
         "networkInterfaceExclusionList": {{.Values.fsm.networkInterfaceExclusionList | mustToJson}}
       },
       "observability": {
-        "enableDebugServer": {{.Values.fsm.enableDebugServer | mustToJson}},
         "fsmLogLevel": {{.Values.fsm.controllerLogLevel | mustToJson}},
         "tracing": {
           "enable": {{.Values.fsm.tracing.enable | mustToJson}}{{- if .Values.fsm.tracing.enable }},{{- end }}

charts/fsm/values.schema.json

@@ -169,7 +169,6 @@
                 "sidecar",
                 "pluginChains",
                 "caBundleSecretName",
-                "enableDebugServer",
                 "enablePermissiveTrafficPolicy",
                 "http1PerRequestLoadBalancing",
                 "http2PerRequestLoadBalancing",
@@ -267,7 +266,8 @@
                     "description": "The details of the fsmXnetwork.",
                     "required": [
                       "xmgt",
-                      "xnet"
+                      "xnet",
+                      "initResources"
                     ],
                     "properties": {
                         "podLabels": {
@@ -561,6 +561,9 @@
                               "additionalProperties": false
                             }
                           }
+                        },
+                        "initResources": {
+                          "$ref": "#/definitions/containerResources"
                         }
                     },
                     "additionalProperties": false
@@ -1178,15 +1181,6 @@
                         "fsm-ca-bundle"
                     ]
                 },
-                "enableDebugServer": {
-                    "$id": "#/properties/fsm/properties/enableDebugServer",
-                    "type": "boolean",
-                    "title": "The enableDebugServer schema",
-                    "description": "Indicates whether the Debug Server should be enabled or not.",
-                    "examples": [
-                        false
-                    ]
-                },
                 "enablePermissiveTrafficPolicy": {
                     "$id": "#/properties/fsm/properties/enablePermissiveTrafficPolicy",
                     "type": "boolean",

charts/fsm/values.yaml

@@ -68,6 +68,10 @@ fsm:
 
   # -- `fsm-controller` image pull secret
   imagePullSecrets: [ ]
+
+  # -- Traffic interception mode in the mesh
+  trafficInterceptionMode: PodLevel
+
   # -- Sidecar supported by fsm
   sidecar:
     image:
@@ -328,6 +332,15 @@ fsm:
     # -- Node tolerations applied to control plane pods.
     # The specified tolerations allow pods to schedule onto nodes with matching taints.
     tolerations: [ ]
+
+    # -- FSM xnetwork's init-container resource parameters.
+    initResources:
+      limits:
+        cpu: "500m"
+        memory: "512M"
+      requests:
+        cpu: "200m"
+        memory: "128M"
   #
   # -- Prometheus parameters
   prometheus:
@@ -477,9 +490,6 @@ fsm:
     # The specified tolerations allow pods to schedule onto nodes with matching taints.
     tolerations: [ ]
 
-  # -- Enable the debug HTTP server on FSM controller
-  enableDebugServer: false
-
   # -- Enable permissive traffic policy mode
   enablePermissiveTrafficPolicy: true
 
@@ -492,9 +502,6 @@ fsm:
   # -- Service access mode
   serviceAccessMode: mixed
 
-  # -- Traffic interception mode in the mesh
-  trafficInterceptionMode: PodLevel
-
   # -- Enable egress in the mesh
   enableEgress: true
 

cmd/fsm-bootstrap/crds/config.flomesh.io_meshconfigs.yaml

@@ -172,10 +172,6 @@ spec:
                 description: Observalility defines the observability configurations
                   for a mesh instance.
                 properties:
-                  enableDebugServer:
-                    description: EnableDebugServer defines if the debug endpoint on
-                      the FSM controller pod is enabled.
-                    type: boolean
                   fsmLogLevel:
                     description: FSMLogLevel defines the log level for FSM control
                       plane logs.
@@ -233,8 +229,6 @@ spec:
                     required:
                     - enable
                     type: object
-                required:
-                - enableDebugServer
                 type: object
               pluginChains:
                 description: PluginChains defines the default plugin chains.
@@ -729,10 +723,6 @@ spec:
                 description: Observalility defines the observability configurations
                   for a mesh instance.
                 properties:
-                  enableDebugServer:
-                    description: EnableDebugServer defines if the debug endpoint on
-                      the FSM controller pod is enabled.
-                    type: boolean
                   fsmLogLevel:
                     description: FSMLogLevel defines the log level for FSM control
                       plane logs.
@@ -790,8 +780,6 @@ spec:
                     required:
                     - enable
                     type: object
-                required:
-                - enableDebugServer
                 type: object
               pluginChains:
                 description: PluginChains defines the default plugin chains.
@@ -1754,10 +1742,6 @@ spec:
                 description: Observalility defines the observability configurations
                   for a mesh instance.
                 properties:
-                  enableDebugServer:
-                    description: EnableDebugServer defines if the debug endpoint on
-                      the FSM controller pod is enabled.
-                    type: boolean
                   fsmLogLevel:
                     description: FSMLogLevel defines the log level for FSM control
                       plane logs.
@@ -1828,8 +1812,6 @@ spec:
                     required:
                     - enable
                     type: object
-                required:
-                - enableDebugServer
                 type: object
               pluginChains:
                 description: PluginChains defines the default plugin chains.

cmd/fsm-bootstrap/crds/connector.flomesh.io_consulconnectors.yaml

@@ -359,7 +359,7 @@ spec:
                     format: int32
                     type: integer
                   generateInternalServiceHealthCheck:
-                    default: true
+                    default: false
                     type: boolean
                   passingOnly:
                     default: true

cmd/fsm-bootstrap/crds/xnetwork.flomesh.io_accesscontrols.yaml

@@ -0,0 +1,82 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.16.5
+  labels:
+    app.kubernetes.io/name: flomesh.io
+  name: accesscontrols.xnetwork.flomesh.io
+spec:
+  group: xnetwork.flomesh.io
+  names:
+    kind: AccessControl
+    listKind: AccessControlList
+    plural: accesscontrols
+    shortNames:
+    - accesscontrol
+    singular: accesscontrol
+  scope: Namespaced
+  versions:
+  - name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: |-
+          AccessControl is the type used to represent an AccessControl policy.
+          An AccessControl policy authorizes one or more backends to accept
+          ingress traffic from one or more sources.
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: Spec is the Ingress backend policy specification
+            properties:
+              services:
+                description: Services defines the list of sources the AccessControl
+                  policy applies to.
+                items:
+                  description: |-
+                    AccessControlServiceSpec is the type used to represent the Source in the list of Sources specified in an
+                    AccessControl policy specification.
+                  properties:
+                    name:
+                      description: Name defines the name of the source for the given
+                        Kind.
+                      type: string
+                    namespace:
+                      description: Namespace defines the namespace for the given source.
+                      type: string
+                    withClusterIPs:
+                      default: true
+                      type: boolean
+                    withEndpointIPs:
+                      default: false
+                      type: boolean
+                    withExternalIPs:
+                      default: false
+                      type: boolean
+                  required:
+                  - name
+                  type: object
+                type: array
+            required:
+            - services
+            type: object
+        type: object
+    served: true
+    storage: true

cmd/fsm-bootstrap/fsm-bootstrap.go

@@ -22,7 +22,7 @@ import (
 	admissionv1 "k8s.io/api/admissionregistration/v1"
 	corev1 "k8s.io/api/core/v1"
 	apiv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
-	clientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
+	"k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	apiclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

cmd/fsm-bootstrap/fsm-bootstrap_test.go

@@ -66,7 +66,6 @@ var testPresetMeshConfigMap = &corev1.ConfigMap{
 	"outboundIPRangeExclusionList": []
 },
 "observability": {
-	"enableDebugServer": false,
 	"fsmLogLevel": "trace",
 	"tracing": {
 	 "enable": false
@@ -137,7 +136,6 @@ func TestBuildDefaultMeshConfig(t *testing.T) {
 	assert.False(meshConfig.Spec.Sidecar.EnablePrivilegedInitContainer)
 	assert.True(meshConfig.Spec.Traffic.EnablePermissiveTrafficPolicyMode)
 	assert.True(meshConfig.Spec.Traffic.EnableEgress)
-	assert.False(meshConfig.Spec.Observability.EnableDebugServer)
 	assert.Equal(meshConfig.Spec.Certificate.ServiceCertValidityDuration, "23h")
 	assert.True(meshConfig.Spec.FeatureFlags.EnableIngressBackendPolicy)
 	assert.True(meshConfig.Spec.FeatureFlags.EnableAccessControlPolicy)

cmd/fsm-connector/fsm-connector.go

@@ -37,7 +37,6 @@ import (
 	"github.com/flomesh-io/fsm/pkg/logger"
 	"github.com/flomesh-io/fsm/pkg/messaging"
 	"github.com/flomesh-io/fsm/pkg/service"
-	_ "github.com/flomesh-io/fsm/pkg/sidecar/providers/pipy/driver"
 	"github.com/flomesh-io/fsm/pkg/signals"
 	"github.com/flomesh-io/fsm/pkg/version"
 )