From 96ec67fa83e0dc2f2d437ef6c627b3b82edb1434 Mon Sep 17 00:00:00 2001 From: Jerry Jacobs Date: Mon, 30 Jul 2018 19:37:38 +0200 Subject: [PATCH 1/5] cmd/serve: pprof memory profile dumps to disk (usefull for #4450) --- cmd/serv.go | 19 +++++++++++++++++++ custom/conf/app.ini.sample | 4 ++++ 2 files changed, 23 insertions(+) diff --git a/cmd/serv.go b/cmd/serv.go index 990355be98cf6..31753e2cb5c72 100644 --- a/cmd/serv.go +++ b/cmd/serv.go @@ -8,9 +8,12 @@ package cmd import ( "encoding/json" "fmt" + "io/ioutil" "os" "os/exec" "path/filepath" + "runtime" + "runtime/pprof" "strings" "time" @@ -93,6 +96,18 @@ func fail(userMessage, logMessage string, args ...interface{}) { os.Exit(1) } +func pprofDumpMemProfileForUsername(username string) { + f, err := ioutil.TempFile("", fmt.Sprintf("gitea_serve_pprof_%s_", username)) + if err != nil { + log.GitLogger.Fatal(4, "Could not create memory profile: %v", err) + } + runtime.GC() // get up-to-date statistics + if err := pprof.WriteHeapProfile(f); err != nil { + log.GitLogger.Fatal(4, "Could not write memory profile: %v", err) + } + f.Close() +} + func runServ(c *cli.Context) error { if c.IsSet("config") { setting.CustomConf = c.String("config") @@ -143,6 +158,10 @@ func runServ(c *cli.Context) error { username := strings.ToLower(rr[0]) reponame := strings.ToLower(strings.TrimSuffix(rr[1], ".git")) + if setting.EnablePprof { + defer pprofDumpMemProfileForUsername(username) + } + isWiki := false unitType := models.UnitTypeCode if strings.HasSuffix(reponame, ".wiki") { diff --git a/custom/conf/app.ini.sample b/custom/conf/app.ini.sample index 97a0eac02c530..05cbc59eb2f62 100644 --- a/custom/conf/app.ini.sample +++ b/custom/conf/app.ini.sample @@ -185,6 +185,10 @@ STATIC_ROOT_PATH = APP_DATA_PATH = data ; Application level GZIP support ENABLE_GZIP = false +; Application profiling (memory and cpu) +; For "web" command it listens on localhost:6060 +; For "serve" command it dumps to disk at $TMPDIR/gitea_serve_pprof__id +ENABLE_PPROF = false ; Landing page, can be "home", "explore", or "organizations" LANDING_PAGE = home ; Enables git-lfs support. true or false, default is false. From 4808ca3f59314f20abf1b40b983d1a68cd0c94b4 Mon Sep 17 00:00:00 2001 From: Jerry Jacobs Date: Mon, 30 Jul 2018 20:30:25 +0200 Subject: [PATCH 2/5] cmd/serve: Allow pprof per-key --- cmd/serv.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/serv.go b/cmd/serv.go index 31753e2cb5c72..49d4129de8324 100644 --- a/cmd/serv.go +++ b/cmd/serv.go @@ -45,6 +45,9 @@ var CmdServ = cli.Command{ Value: "custom/conf/app.ini", Usage: "Custom configuration file path", }, + cli.BoolFlag{ + Name: "enable-pprof", + }, }, } @@ -158,7 +161,7 @@ func runServ(c *cli.Context) error { username := strings.ToLower(rr[0]) reponame := strings.ToLower(strings.TrimSuffix(rr[1], ".git")) - if setting.EnablePprof { + if setting.EnablePprof || c.Bool("enable-pprof") { defer pprofDumpMemProfileForUsername(username) } From 88933945b60794ba0ef430632b50509c7217b5ee Mon Sep 17 00:00:00 2001 From: Jerry Jacobs Date: Wed, 1 Aug 2018 12:13:16 +0200 Subject: [PATCH 3/5] cmd/serve: Add setting for PPROF_DATA_PATH and break pprof code into its own module for reuse and clarity --- cmd/serv.go | 22 ++++++-------------- custom/conf/app.ini.sample | 6 ++++-- modules/pprof/pprof.go | 42 ++++++++++++++++++++++++++++++++++++++ modules/setting/setting.go | 8 ++++++++ 4 files changed, 60 insertions(+), 18 deletions(-) create mode 100644 modules/pprof/pprof.go diff --git a/cmd/serv.go b/cmd/serv.go index 49d4129de8324..6c9e770241e44 100644 --- a/cmd/serv.go +++ b/cmd/serv.go @@ -8,12 +8,9 @@ package cmd import ( "encoding/json" "fmt" - "io/ioutil" "os" "os/exec" "path/filepath" - "runtime" - "runtime/pprof" "strings" "time" @@ -22,6 +19,7 @@ import ( "code.gitea.io/gitea/modules/private" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" + "code.gitea.io/gitea/modules/pprof" "github.com/Unknwon/com" "github.com/dgrijalva/jwt-go" @@ -99,18 +97,6 @@ func fail(userMessage, logMessage string, args ...interface{}) { os.Exit(1) } -func pprofDumpMemProfileForUsername(username string) { - f, err := ioutil.TempFile("", fmt.Sprintf("gitea_serve_pprof_%s_", username)) - if err != nil { - log.GitLogger.Fatal(4, "Could not create memory profile: %v", err) - } - runtime.GC() // get up-to-date statistics - if err := pprof.WriteHeapProfile(f); err != nil { - log.GitLogger.Fatal(4, "Could not write memory profile: %v", err) - } - f.Close() -} - func runServ(c *cli.Context) error { if c.IsSet("config") { setting.CustomConf = c.String("config") @@ -162,7 +148,11 @@ func runServ(c *cli.Context) error { reponame := strings.ToLower(strings.TrimSuffix(rr[1], ".git")) if setting.EnablePprof || c.Bool("enable-pprof") { - defer pprofDumpMemProfileForUsername(username) + stopCPUProfiler := pprof.DumpCPUProfileForUsername(setting.PprofDataPath, username) + defer func() { + stopCPUProfiler() + pprof.DumpMemProfileForUsername(setting.PprofDataPath, username) + }() } isWiki := false diff --git a/custom/conf/app.ini.sample b/custom/conf/app.ini.sample index 05cbc59eb2f62..40945674b549f 100644 --- a/custom/conf/app.ini.sample +++ b/custom/conf/app.ini.sample @@ -187,8 +187,10 @@ APP_DATA_PATH = data ENABLE_GZIP = false ; Application profiling (memory and cpu) ; For "web" command it listens on localhost:6060 -; For "serve" command it dumps to disk at $TMPDIR/gitea_serve_pprof__id +; For "serve" command it dumps to disk at PPROF_DATA_PATH as (cpuprofile|memprofile)__ ENABLE_PPROF = false +; PPROF_DATA_PATH, use an absolute path when you start gitea as service +PPROF_DATA_PATH = data/tmp/pprof ; Landing page, can be "home", "explore", or "organizations" LANDING_PAGE = home ; Enables git-lfs support. true or false, default is false. @@ -217,7 +219,7 @@ USER = root PASSWD = ; For "postgres" only, either "disable", "require" or "verify-full" SSL_MODE = disable -; For "sqlite3" and "tidb", use absolute path when you start gitea as service +; For "sqlite3" and "tidb", use an absolute path when you start gitea as service PATH = data/gitea.db ; For "sqlite3" only. Query timeout SQLITE_TIMEOUT = 500 diff --git a/modules/pprof/pprof.go b/modules/pprof/pprof.go new file mode 100644 index 0000000000000..7f4fc653f3509 --- /dev/null +++ b/modules/pprof/pprof.go @@ -0,0 +1,42 @@ +// Copyright 2018 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package pprof + +import ( + "fmt" + "io/ioutil" + "runtime" + "runtime/pprof" + + "code.gitea.io/gitea/modules/log" +) + +// DumpMemProfileForUsername dumps a memory profile at pprofDataPath as memprofile__ +func DumpMemProfileForUsername(pprofDataPath, username string) { + f, err := ioutil.TempFile(pprofDataPath, fmt.Sprintf("memprofile_%s_", username)) + if err != nil { + log.GitLogger.Fatal(4, "Could not create memory profile: %v", err) + } + defer f.Close() + runtime.GC() // get up-to-date statistics + if err := pprof.WriteHeapProfile(f); err != nil { + log.GitLogger.Fatal(4, "Could not write memory profile: %v", err) + } +} + +// DumpCPUProfileForUsername dumps a CPU profile at pprofDataPath as cpuprofile__ +// it returns the stop function which stops, writes and closes the CPU profile file +func DumpCPUProfileForUsername(pprofDataPath, username string) func() { + f, err := ioutil.TempFile(pprofDataPath, fmt.Sprintf("cpuprofile_%s_", username)) + if err != nil { + log.GitLogger.Fatal(4, "Could not create cpu profile: %v", err) + } + + pprof.StartCPUProfile(f) + return func() { + pprof.StopCPUProfile() + f.Close() + } +} diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 1b9919404cfee..2b9fbe8f30bbd 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -111,6 +111,7 @@ var ( LandingPageURL LandingPage UnixSocketPermission uint32 EnablePprof bool + PprofDataPath string SSH = struct { Disabled bool `ini:"DISABLE_SSH"` @@ -773,6 +774,7 @@ func NewContext() { AppDataPath = sec.Key("APP_DATA_PATH").MustString(path.Join(AppWorkPath, "data")) EnableGzip = sec.Key("ENABLE_GZIP").MustBool() EnablePprof = sec.Key("ENABLE_PPROF").MustBool(false) + PprofDataPath = sec.Key("PPROF_DATA_PATH").MustString(path.Join(AppWorkPath, "data/tmp/pprof")) switch sec.Key("LANDING_PAGE").MustString("home") { case "explore": @@ -916,6 +918,12 @@ func NewContext() { } } + if PprofDataPath != "" { + if err := os.MkdirAll(PprofDataPath, os.ModePerm); err != nil { + log.Fatal(4, "Failed to create '%s': %v", PprofDataPath, err) + } + } + sec = Cfg.Section("security") InstallLock = sec.Key("INSTALL_LOCK").MustBool(false) SecretKey = sec.Key("SECRET_KEY").MustString("!#@FDEWREWR&*(") From 3c19346f4de6811875a3cb4713eece8b822e8645 Mon Sep 17 00:00:00 2001 From: Jerry Jacobs Date: Sun, 5 Aug 2018 23:14:46 +0200 Subject: [PATCH 4/5] Process last review comments --- cmd/serv.go | 6 +++++- modules/setting/setting.go | 9 +++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/cmd/serv.go b/cmd/serv.go index 6c9e770241e44..59405dbc3b8d8 100644 --- a/cmd/serv.go +++ b/cmd/serv.go @@ -16,10 +16,10 @@ import ( "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/pprof" "code.gitea.io/gitea/modules/private" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" - "code.gitea.io/gitea/modules/pprof" "github.com/Unknwon/com" "github.com/dgrijalva/jwt-go" @@ -148,6 +148,10 @@ func runServ(c *cli.Context) error { reponame := strings.ToLower(strings.TrimSuffix(rr[1], ".git")) if setting.EnablePprof || c.Bool("enable-pprof") { + if err := os.MkdirAll(PprofDataPath, os.ModePerm); err != nil { + fail("Error while trying to create PPROF_DATA_PATH", "Error while trying to create PPROF_DATA_PATH: %v", err) + } + stopCPUProfiler := pprof.DumpCPUProfileForUsername(setting.PprofDataPath, username) defer func() { stopCPUProfiler() diff --git a/modules/setting/setting.go b/modules/setting/setting.go index 2b9fbe8f30bbd..f8e2e4e0e9917 100644 --- a/modules/setting/setting.go +++ b/modules/setting/setting.go @@ -775,6 +775,9 @@ func NewContext() { EnableGzip = sec.Key("ENABLE_GZIP").MustBool() EnablePprof = sec.Key("ENABLE_PPROF").MustBool(false) PprofDataPath = sec.Key("PPROF_DATA_PATH").MustString(path.Join(AppWorkPath, "data/tmp/pprof")) + if !filepath.IsAbs(PprofDataPath) { + PprofDataPath = filepath.Join(AppWorkPath, PprofDataPath) + } switch sec.Key("LANDING_PAGE").MustString("home") { case "explore": @@ -918,12 +921,6 @@ func NewContext() { } } - if PprofDataPath != "" { - if err := os.MkdirAll(PprofDataPath, os.ModePerm); err != nil { - log.Fatal(4, "Failed to create '%s': %v", PprofDataPath, err) - } - } - sec = Cfg.Section("security") InstallLock = sec.Key("INSTALL_LOCK").MustBool(false) SecretKey = sec.Key("SECRET_KEY").MustString("!#@FDEWREWR&*(") From da761397355b543472fb6c7801790f3ebefbaaac Mon Sep 17 00:00:00 2001 From: Jerry Jacobs Date: Sun, 5 Aug 2018 23:19:59 +0200 Subject: [PATCH 5/5] cmd/serv: Fix typo --- cmd/serv.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/serv.go b/cmd/serv.go index 59405dbc3b8d8..b532b95494e50 100644 --- a/cmd/serv.go +++ b/cmd/serv.go @@ -148,7 +148,7 @@ func runServ(c *cli.Context) error { reponame := strings.ToLower(strings.TrimSuffix(rr[1], ".git")) if setting.EnablePprof || c.Bool("enable-pprof") { - if err := os.MkdirAll(PprofDataPath, os.ModePerm); err != nil { + if err := os.MkdirAll(setting.PprofDataPath, os.ModePerm); err != nil { fail("Error while trying to create PPROF_DATA_PATH", "Error while trying to create PPROF_DATA_PATH: %v", err) }