Add support for dataset,runtime,webhook pprof

charts/fluid/fluid/templates/controller/alluxioruntime_controller.yaml

@@ -38,6 +38,7 @@ spec:
           - --development=false
           - --runtime-node-port-range={{ .Values.runtime.alluxio.portRange }}
           - --runtime-workers={{ .Values.runtime.alluxio.runtimeWorkers }}
+          - --pprof-addr=:6060
         env:
           {{- if .Values.workdir }}
           - name: FLUID_WORKDIR

charts/fluid/fluid/templates/controller/dataset_controller.yaml

@@ -41,6 +41,7 @@ spec:
         command: ["dataset-controller", "start"]
         args:
           - --development=false
+          - --pprof-addr=:6060
         env:
           {{- if .Values.workdir }}
           - name: FLUID_WORKDIR

charts/fluid/fluid/templates/controller/goosefsruntime_controller.yaml

@@ -39,6 +39,7 @@ spec:
           - --development=false
           - --runtime-node-port-range={{ .Values.runtime.goosefs.portRange }}
           - --runtime-workers={{ .Values.runtime.goosefs.runtimeWorkers }}
+          - --pprof-addr=:6060
         env:
           {{- if .Values.workdir }}
           - name: FLUID_WORKDIR

charts/fluid/fluid/templates/controller/jindoruntime_controller.yaml

@@ -37,6 +37,7 @@ spec:
             - --development=false
             - --runtime-node-port-range={{ .Values.runtime.jindo.portRange }}
             - --runtime-workers={{ .Values.runtime.jindo.runtimeWorkers }}
+            - --pprof-addr=:6060
           env:
           {{- if .Values.workdir }}
           - name: FLUID_WORKDIR

charts/fluid/fluid/templates/controller/juicefsruntime_controller.yaml

@@ -35,6 +35,7 @@ spec:
         name: manager
         args:
           - --development=false
+          - --pprof-addr=:6060
         command: ["juicefsruntime-controller", "start"]
         env:
           {{- if .Values.runtime.juicefs.fuse.image }}

charts/fluid/fluid/templates/webhook/webhook.yaml

@@ -23,6 +23,7 @@ spec:
           command: ["fluid-webhook", "start"]
           args:
             - --development=false
+            - --pprof-addr=:6060
           env:
             - name: MY_POD_NAMESPACE
               valueFrom:

cmd/alluxio/main.go

@@ -18,13 +18,12 @@ package main
 
 import (
 	"fmt"
-	"os"
-
 	"github.com/fluid-cloudnative/fluid"
 	alluxioctl "github.com/fluid-cloudnative/fluid/pkg/controllers/v1alpha1/alluxio"
 	"github.com/fluid-cloudnative/fluid/pkg/ddc/base/portallocator"
 	"github.com/spf13/cobra"
 	"k8s.io/apimachinery/pkg/util/net"
+	"os"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 
 	"go.uber.org/zap/zapcore"
@@ -41,6 +40,7 @@ import (
 	datav1alpha1 "github.com/fluid-cloudnative/fluid/api/v1alpha1"
 	"github.com/fluid-cloudnative/fluid/pkg/ddc/alluxio"
 	"github.com/fluid-cloudnative/fluid/pkg/ddc/base"
+	"github.com/fluid-cloudnative/fluid/pkg/utils"
 )
 
 var (
@@ -55,6 +55,7 @@ var (
 	development             bool
 	portRange               string
 	maxConcurrentReconciles int
+	pprofAddr               string
 )
 
 var cmd = &cobra.Command{
@@ -88,7 +89,7 @@ func init() {
 	startCmd.Flags().StringVar(&portRange, "runtime-node-port-range", "20000-25000", "Set available port range for Alluxio")
 	startCmd.Flags().IntVar(&maxConcurrentReconciles, "runtime-workers", 3, "Set max concurrent workers for AlluxioRuntime controller")
 	versionCmd.Flags().BoolVar(&short, "short", false, "print just the short version info")
-
+	startCmd.Flags().StringVarP(&pprofAddr, "pprof-addr", "", "", "The address for pprof to use while exporting profiling results")
 	cmd.AddCommand(startCmd)
 	cmd.AddCommand(versionCmd)
 }
@@ -116,6 +117,8 @@ func handle() {
 		}
 	}))
 
+	utils.NewPprofServer(setupLog, pprofAddr)
+
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:             scheme,
 		MetricsBindAddress: metricsAddr,

cmd/dataset/main.go

@@ -18,14 +18,14 @@ package main
 import (
 	"fmt"
 	"github.com/fluid-cloudnative/fluid"
+	"github.com/fluid-cloudnative/fluid/pkg/utils"
 	"github.com/spf13/cobra"
-	"go.uber.org/zap/zapcore"
-	"os"
-
 	zapOpt "go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
+	"os"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
 
@@ -47,6 +47,7 @@ var (
 	metricsAddr          string
 	enableLeaderElection bool
 	development          bool
+	pprofAddr            string
 )
 
 var cmd = &cobra.Command{
@@ -78,7 +79,7 @@ func init() {
 	startCmd.Flags().BoolVarP(&enableLeaderElection, "enable-leader-election", "", false, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
 	startCmd.Flags().BoolVarP(&development, "development", "", true, "Enable development mode for fluid controller.")
 	versionCmd.Flags().BoolVar(&short, "short", false, "print just the short version info")
-
+	startCmd.Flags().StringVarP(&pprofAddr, "pprof-addr", "", "", "The address for pprof to use while exporting profiling results")
 	cmd.AddCommand(startCmd)
 	cmd.AddCommand(versionCmd)
 
@@ -107,6 +108,8 @@ func handle() {
 		}
 	}))
 
+	utils.NewPprofServer(setupLog, pprofAddr)
+
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:             scheme,
 		MetricsBindAddress: metricsAddr,

cmd/goosefs/app/goosefs.go

@@ -16,20 +16,20 @@ limitations under the License.
 package app
 
 import (
-	"os"
-
 	"github.com/fluid-cloudnative/fluid"
 	datav1alpha1 "github.com/fluid-cloudnative/fluid/api/v1alpha1"
 	goosefsctl "github.com/fluid-cloudnative/fluid/pkg/controllers/v1alpha1/goosefs"
 	"github.com/fluid-cloudnative/fluid/pkg/ddc/base"
 	"github.com/fluid-cloudnative/fluid/pkg/ddc/base/portallocator"
 	"github.com/fluid-cloudnative/fluid/pkg/ddc/goosefs"
+	"github.com/fluid-cloudnative/fluid/pkg/utils"
 	"github.com/spf13/cobra"
 	zapOpt "go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/net"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"os"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
@@ -46,6 +46,7 @@ var (
 	development             bool
 	portRange               string
 	maxConcurrentReconciles int
+	pprofAddr               string
 )
 
 var cmd = &cobra.Command{
@@ -69,6 +70,7 @@ func init() {
 	startCmd.Flags().BoolVarP(&enableLeaderElection, "enable-leader-election", "", false, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
 	startCmd.Flags().BoolVarP(&development, "development", "", true, "Enable development mode for fluid controller.")
 	startCmd.Flags().StringVar(&portRange, "runtime-node-port-range", "20000-25000", "Set available port range for GooseFS")
+	startCmd.Flags().StringVarP(&pprofAddr, "pprof-addr", "", "", "The address for pprof to use while exporting profiling results")
 	startCmd.Flags().IntVar(&maxConcurrentReconciles, "runtime-workers", 3, "Set max concurrent workers for GooseFSRuntime controller")
 	cmd.AddCommand(startCmd)
 }
@@ -89,6 +91,8 @@ func handle() {
 		}
 	}))
 
+	utils.NewPprofServer(setupLog, pprofAddr)
+
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:             scheme,
 		MetricsBindAddress: metricsAddr,

cmd/jindo/main.go

@@ -17,20 +17,20 @@ package main
 
 import (
 	"fmt"
-	"os"
-
 	"github.com/fluid-cloudnative/fluid"
 	datav1alpha1 "github.com/fluid-cloudnative/fluid/api/v1alpha1"
 	jindoctl "github.com/fluid-cloudnative/fluid/pkg/controllers/v1alpha1/jindo"
 	"github.com/fluid-cloudnative/fluid/pkg/ddc/base"
 	"github.com/fluid-cloudnative/fluid/pkg/ddc/base/portallocator"
 	"github.com/fluid-cloudnative/fluid/pkg/ddc/jindo"
+	"github.com/fluid-cloudnative/fluid/pkg/utils"
 	"github.com/spf13/cobra"
 	zapOpt "go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/net"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"os"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
@@ -50,6 +50,7 @@ var (
 	eventDriven             bool
 	portRange               string
 	maxConcurrentReconciles int
+	pprofAddr               string
 )
 
 var cmd = &cobra.Command{
@@ -82,6 +83,7 @@ func init() {
 	startCmd.Flags().BoolVarP(&development, "development", "", true, "Enable development mode for fluid controller.")
 	startCmd.Flags().StringVar(&portRange, "runtime-node-port-range", "18000-19999", "Set available port range for Jindo")
 	startCmd.Flags().IntVar(&maxConcurrentReconciles, "runtime-workers", 3, "Set max concurrent workers for JindoRuntime controller")
+	startCmd.Flags().StringVarP(&pprofAddr, "pprof-addr", "", "", "The address for pprof to use while exporting profiling results")
 	startCmd.Flags().BoolVar(&eventDriven, "event-driven", true, "The reconciler's loop strategy. if it's false, it indicates period driven.")
 	versionCmd.Flags().BoolVar(&short, "short", false, "print just the short version info")
 
@@ -112,6 +114,8 @@ func handle() {
 		}
 	}))
 
+	utils.NewPprofServer(setupLog, pprofAddr)
+
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:             scheme,
 		MetricsBindAddress: metricsAddr,

cmd/juicefs/app/juicefs.go

@@ -17,13 +17,13 @@ limitations under the License.
 package app
 
 import (
-	"os"
-
+	"github.com/fluid-cloudnative/fluid/pkg/utils"
 	"github.com/spf13/cobra"
 	zapOpt "go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"os"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
@@ -45,6 +45,7 @@ var (
 	enableLeaderElection    bool
 	development             bool
 	maxConcurrentReconciles int
+	pprofAddr               string
 )
 
 var startCmd = &cobra.Command{
@@ -61,6 +62,7 @@ func init() {
 
 	startCmd.Flags().StringVarP(&metricsAddr, "metrics-addr", "", ":8080", "The address the metric endpoint binds to.")
 	startCmd.Flags().BoolVarP(&enableLeaderElection, "enable-leader-election", "", false, "Enable leader election for controller manager. Enabling this will ensure there is only one active controller manager.")
+	startCmd.Flags().StringVarP(&pprofAddr, "pprof-addr", "", "", "The address for pprof to use while exporting profiling results")
 	startCmd.Flags().BoolVarP(&development, "development", "", true, "Enable development mode for fluid controller.")
 }
 
@@ -80,6 +82,8 @@ func handle() {
 		}
 	}))
 
+	utils.NewPprofServer(setupLog, pprofAddr)
+
 	mgr, err := ctrl.NewManager(ctrl.GetConfigOrDie(), ctrl.Options{
 		Scheme:             scheme,
 		MetricsBindAddress: metricsAddr,

cmd/webhook/app/webhook.go

@@ -17,18 +17,18 @@ package app
 
 import (
 	"flag"
-	"os"
-
 	"github.com/fluid-cloudnative/fluid"
 	datav1alpha1 "github.com/fluid-cloudnative/fluid/api/v1alpha1"
 	"github.com/fluid-cloudnative/fluid/pkg/common"
+	"github.com/fluid-cloudnative/fluid/pkg/utils"
 	fluidwebhook "github.com/fluid-cloudnative/fluid/pkg/webhook"
 	"github.com/fluid-cloudnative/fluid/pkg/webhook/handler"
 	"github.com/spf13/cobra"
 	zapOpt "go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 	"k8s.io/apimachinery/pkg/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
+	"os"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
@@ -48,6 +48,7 @@ var (
 	metricsAddr string
 	webhookPort int
 	certDir     string
+	pprofAddr   string
 )
 
 var webhookCmd = &cobra.Command{
@@ -67,7 +68,7 @@ func init() {
 	webhookCmd.Flags().BoolVarP(&development, "development", "", true, "Enable development mode for fluid controller.")
 	webhookCmd.Flags().IntVar(&webhookPort, "webhook-port", 9443, "Admission webhook listen address.")
 	webhookCmd.Flags().StringVar(&certDir, "webhook-cert-dir", "/etc/k8s-webhook-server/certs", "Admission webhook cert/key dir.")
-
+	webhookCmd.Flags().StringVarP(&pprofAddr, "pprof-addr", "", "", "The address for pprof to use while exporting profiling results")
 	webhookCmd.Flags().AddGoFlagSet(flag.CommandLine)
 }
 
@@ -82,6 +83,8 @@ func handle() {
 		os.Exit(1)
 	}
 
+	utils.NewPprofServer(setupLog, pprofAddr)
+
 	mgr, err := ctrl.NewManager(cfg, ctrl.Options{
 		Scheme:             scheme,
 		MetricsBindAddress: metricsAddr,

pkg/utils/kubeclient/exec.go

@@ -46,7 +46,7 @@ var (
 	restConfig     *restclient.Config
 	log            logr.Logger = ctrl.Log.WithName("kubeclient")
 	kubeconfigPath             = "~/.kube/config"
-	mutex =&sync.Mutex{}
+	mutex                      = &sync.Mutex{}
 )
 
 // ExecOptions passed to ExecWithOptions

pkg/utils/pprof.go

@@ -0,0 +1,62 @@
+/*
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package utils
+
+import (
+	"context"
+	"errors"
+	"github.com/go-logr/logr"
+	"net/http"
+	"net/http/pprof"
+	"time"
+)
+
+func NewPprofServer(setupLog logr.Logger, pprofAddr string) {
+	if pprofAddr != "" {
+		setupLog.Info("Enabling pprof", "pprof address", pprofAddr)
+		mux := http.NewServeMux()
+		mux.HandleFunc("/debug/pprof/", pprof.Index)
+		mux.HandleFunc("/debug/pprof/cmdline", pprof.Cmdline)
+		mux.HandleFunc("/debug/pprof/profile", pprof.Profile)
+		mux.HandleFunc("/debug/pprof/symbol", pprof.Symbol)
+		mux.HandleFunc("/debug/pprof/trace", pprof.Trace)
+
+		pprofServer := http.Server{
+			Addr:    pprofAddr,
+			Handler: mux,
+		}
+		setupLog.Info("Starting pprof HTTP server", "pprof server address", pprofServer.Addr)
+
+		go func() {
+			go func() {
+				ctx := context.Background()
+				<-ctx.Done()
+
+				ctx, cancelFunc := context.WithTimeout(context.Background(), 60*time.Minute)
+				defer cancelFunc()
+
+				if err := pprofServer.Shutdown(ctx); err != nil {
+					setupLog.Error(err, "Failed to shutdown debug HTTP server")
+				}
+			}()
+
+			if err := pprofServer.ListenAndServe(); !errors.Is(http.ErrServerClosed, err) {
+				setupLog.Error(err, "Failed to start debug HTTP server")
+				panic(err)
+			}
+		}()
+	}
+}