[bot] AutoMerging: merge all upstream's changes:

* https://github.com/fatedier/frp:
  config: add some validations (fatedier#3610)
  add e2e tests for v1 config (fatedier#3608)

Release.md

@@ -1 +1,3 @@
 ### Features
+
+* Configuration: We now support TOML, YAML, and JSON for configuration. Please note that INI is deprecated and will be removed in future releases. New features will only be available in TOML, YAML, or JSON. Users wanting these new features should switch their configuration format accordingly. #2521

cmd/frpc/sub/root.go

@@ -117,6 +117,7 @@ var rootCmd = &cobra.Command{
 		// Do not show command usage here.
 		err := runClient(cfgFile)
 		if err != nil {
+			fmt.Println(err)
 			os.Exit(1)
 		}
 		return nil
@@ -186,7 +187,7 @@ func parseClientCommonCfgFromCmd() (*v1.ClientCommonConfig, error) {
 
 	cfg.Complete()
 
-	err, warning := validation.ValidateClientCommonConfig(cfg)
+	warning, err := validation.ValidateClientCommonConfig(cfg)
 	if warning != nil {
 		fmt.Printf("WARNING: %v\n", warning)
 	}
@@ -199,7 +200,6 @@ func parseClientCommonCfgFromCmd() (*v1.ClientCommonConfig, error) {
 func runClient(cfgFilePath string) error {
 	cfg, pxyCfgs, visitorCfgs, isLegacyFormat, err := config.LoadClientConfig(cfgFilePath)
 	if err != nil {
-		fmt.Println(err)
 		return err
 	}
 	if isLegacyFormat {

cmd/frps/root.go

@@ -108,15 +108,17 @@ var rootCmd = &cobra.Command{
 		if cfgFile != "" {
 			svrCfg, isLegacyFormat, err = config.LoadServerConfig(cfgFile)
 			if err != nil {
-				return err
+				fmt.Println(err)
+				os.Exit(1)
 			}
 			if isLegacyFormat {
 				fmt.Printf("WARNING: ini format is deprecated and the support will be removed in the future, " +
 					"please use yaml/json/toml format instead!\n")
 			}
 		} else {
 			if svrCfg, err = parseServerConfigFromCmd(); err != nil {
-				return err
+				fmt.Println(err)
+				os.Exit(1)
 			}
 		}
 
@@ -125,7 +127,8 @@ var rootCmd = &cobra.Command{
 			fmt.Printf("WARNING: %v\n", warning)
 		}
 		if err != nil {
-			return err
+			fmt.Println(err)
+			os.Exit(1)
 		}
 
 		if err := runServer(svrCfg); err != nil {
@@ -168,7 +171,7 @@ func parseServerConfigFromCmd() (*v1.ServerConfig, error) {
 	cfg.Log.MaxDays = logMaxDays
 	cfg.Log.DisablePrintColor = disableLogColor
 	cfg.SubDomainHost = subDomainHost
-	cfg.TLS.Force = tlsOnly
+	cfg.Transport.TLS.Force = tlsOnly
 	cfg.MaxPortsPerClient = maxPortsPerClient
 
 	// Only token authentication is supported in cmd mode

go.mod

@@ -3,7 +3,6 @@ module github.com/fatedier/frp
 go 1.20
 
 require (
-	github.com/BurntSushi/toml v0.3.1
 	github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5
 	github.com/coreos/go-oidc/v3 v3.6.0
 	github.com/fatedier/beego v0.0.0-20171024143340-6c6a4f5bd5eb
@@ -15,6 +14,7 @@ require (
 	github.com/hashicorp/yamux v0.1.1
 	github.com/onsi/ginkgo/v2 v2.11.0
 	github.com/onsi/gomega v1.27.8
+	github.com/pelletier/go-toml/v2 v2.1.0
 	github.com/pion/stun v0.6.1
 	github.com/pires/go-proxyproto v0.7.0
 	github.com/prometheus/client_golang v1.16.0

go.sum

@@ -1,7 +1,6 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c h1:/IBSNwUN8+eKzUzbJPqhK839ygXJ82sde8x3ogr6R28=
 github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
-github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
@@ -93,6 +92,8 @@ github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU
 github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM=
 github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc=
 github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ=
+github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
+github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
 github.com/pion/dtls/v2 v2.2.7 h1:cSUBsETxepsCSFSxC3mc/aDo14qQLMSL+O6IjG28yV8=
 github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s=
 github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY=

hack/run-e2e.sh

@@ -6,7 +6,7 @@ ROOT=$(unset CDPATH && cd "$(dirname "$SCRIPT")/.." && pwd)
 ginkgo_command=$(which ginkgo 2>/dev/null)
 if [ -z "$ginkgo_command" ]; then
     echo "ginkgo not found, try to install..."
-    go install github.com/onsi/ginkgo/v2/ginkgo@v2.8.3
+    go install github.com/onsi/ginkgo/v2/ginkgo@v2.11.0
 fi
 
 debug=false

pkg/auth/auth.go

@@ -31,9 +31,9 @@ type Setter interface {
 func NewAuthSetter(cfg v1.AuthClientConfig) (authProvider Setter) {
 	switch cfg.Method {
 	case consts.TokenAuthMethod:
-		authProvider = NewTokenAuth(cfg.AdditionalAuthScopes, cfg.Token)
+		authProvider = NewTokenAuth(cfg.AdditionalScopes, cfg.Token)
 	case consts.OidcAuthMethod:
-		authProvider = NewOidcAuthSetter(cfg.AdditionalAuthScopes, cfg.OIDC)
+		authProvider = NewOidcAuthSetter(cfg.AdditionalScopes, cfg.OIDC)
 	default:
 		panic(fmt.Sprintf("wrong method: '%s'", cfg.Method))
 	}
@@ -49,9 +49,9 @@ type Verifier interface {
 func NewAuthVerifier(cfg v1.AuthServerConfig) (authVerifier Verifier) {
 	switch cfg.Method {
 	case consts.TokenAuthMethod:
-		authVerifier = NewTokenAuth(cfg.AdditionalAuthScopes, cfg.Token)
+		authVerifier = NewTokenAuth(cfg.AdditionalScopes, cfg.Token)
 	case consts.OidcAuthMethod:
-		authVerifier = NewOidcAuthVerifier(cfg.AdditionalAuthScopes, cfg.OIDC)
+		authVerifier = NewOidcAuthVerifier(cfg.AdditionalScopes, cfg.OIDC)
 	}
 	return authVerifier
 }

pkg/config/legacy/conversion.go

@@ -29,10 +29,10 @@ func Convert_ClientCommonConf_To_v1(conf *ClientCommonConf) *v1.ClientCommonConf
 	out.Auth.Method = conf.ClientConfig.AuthenticationMethod
 	out.Auth.Token = conf.ClientConfig.Token
 	if conf.ClientConfig.AuthenticateHeartBeats {
-		out.Auth.AdditionalAuthScopes = append(out.Auth.AdditionalAuthScopes, v1.AuthScopeHeartBeats)
+		out.Auth.AdditionalScopes = append(out.Auth.AdditionalScopes, v1.AuthScopeHeartBeats)
 	}
 	if conf.ClientConfig.AuthenticateNewWorkConns {
-		out.Auth.AdditionalAuthScopes = append(out.Auth.AdditionalAuthScopes, v1.AuthScopeNewWorkConns)
+		out.Auth.AdditionalScopes = append(out.Auth.AdditionalScopes, v1.AuthScopeNewWorkConns)
 	}
 	out.Auth.OIDC.ClientID = conf.ClientConfig.OidcClientID
 	out.Auth.OIDC.ClientSecret = conf.ClientConfig.OidcClientSecret
@@ -89,10 +89,10 @@ func Convert_ServerCommonConf_To_v1(conf *ServerCommonConf) *v1.ServerConfig {
 	out.Auth.Method = conf.ServerConfig.AuthenticationMethod
 	out.Auth.Token = conf.ServerConfig.Token
 	if conf.ServerConfig.AuthenticateHeartBeats {
-		out.Auth.AdditionalAuthScopes = append(out.Auth.AdditionalAuthScopes, v1.AuthScopeHeartBeats)
+		out.Auth.AdditionalScopes = append(out.Auth.AdditionalScopes, v1.AuthScopeHeartBeats)
 	}
 	if conf.ServerConfig.AuthenticateNewWorkConns {
-		out.Auth.AdditionalAuthScopes = append(out.Auth.AdditionalAuthScopes, v1.AuthScopeNewWorkConns)
+		out.Auth.AdditionalScopes = append(out.Auth.AdditionalScopes, v1.AuthScopeNewWorkConns)
 	}
 	out.Auth.OIDC.Audience = conf.ServerConfig.OidcAudience
 	out.Auth.OIDC.Issuer = conf.ServerConfig.OidcIssuer
@@ -146,12 +146,12 @@ func Convert_ServerCommonConf_To_v1(conf *ServerCommonConf) *v1.ServerConfig {
 	out.Transport.MaxPoolCount = conf.MaxPoolCount
 	out.Transport.HeartbeatTimeout = conf.HeartbeatTimeout
 
-	out.MaxPortsPerClient = conf.MaxPortsPerClient
+	out.Transport.TLS.Force = conf.TLSOnly
+	out.Transport.TLS.CertFile = conf.TLSCertFile
+	out.Transport.TLS.KeyFile = conf.TLSKeyFile
+	out.Transport.TLS.TrustedCaFile = conf.TLSTrustedCaFile
 
-	out.TLS.Force = conf.TLSOnly
-	out.TLS.CertFile = conf.TLSCertFile
-	out.TLS.KeyFile = conf.TLSKeyFile
-	out.TLS.TrustedCaFile = conf.TLSTrustedCaFile
+	out.MaxPortsPerClient = conf.MaxPortsPerClient
 
 	for _, v := range conf.HTTPPlugins {
 		out.HTTPPlugins = append(out.HTTPPlugins, v1.HTTPPluginOptions{

pkg/config/load.go

@@ -23,7 +23,7 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/BurntSushi/toml"
+	toml "github.com/pelletier/go-toml/v2"
 	"github.com/samber/lo"
 	"gopkg.in/ini.v1"
 	"k8s.io/apimachinery/pkg/util/sets"
@@ -119,7 +119,6 @@ func LoadConfigure(b []byte, c any) error {
 			return err
 		}
 	}
-
 	decoder := yaml.NewYAMLOrJSONDecoder(bytes.NewBuffer(b), 4096)
 	return decoder.Decode(c)
 }

pkg/config/v1/client.go

@@ -168,7 +168,7 @@ type AuthClientConfig struct {
 	Method string `json:"method,omitempty"`
 	// Specify whether to include auth info in additional scope.
 	// Current supported scopes are: "HeartBeats", "NewWorkConns".
-	AdditionalAuthScopes []AuthScope `json:"additionalAuthScopes,omitempty"`
+	AdditionalScopes []AuthScope `json:"additionalScopes,omitempty"`
 	// Token specifies the authorization token used to create keys to be sent
 	// to the server. The server must have a matching token for authorization
 	// to succeed.  By default, this value is "".

pkg/config/v1/plugin.go

@@ -46,10 +46,11 @@ func (c *TypedClientPluginOptions) UnmarshalJSON(b []byte) error {
 	if !ok {
 		return fmt.Errorf("unknown plugin type: %s", typeStruct.Type)
 	}
-	if err := json.Unmarshal(b, v); err != nil {
+	options := reflect.New(v).Interface().(ClientPluginOptions)
+	if err := json.Unmarshal(b, options); err != nil {
 		return err
 	}
-	c.ClientPluginOptions = v
+	c.ClientPluginOptions = options
 	return nil
 }
 

pkg/config/v1/server.go

@@ -30,32 +30,32 @@ type ServerConfig struct {
 	BindAddr string `json:"bindAddr,omitempty"`
 	// BindPort specifies the port that the server listens on. By default, this
 	// value is 7000.
-	BindPort int `json:"bindPort,omitempty" validate:"gte=0,lte=65535"`
+	BindPort int `json:"bindPort,omitempty"`
 	// KCPBindPort specifies the KCP port that the server listens on. If this
 	// value is 0, the server will not listen for KCP connections.
-	KCPBindPort int `json:"kcpBindPort,omitempty" validate:"gte=0,lte=65535"`
+	KCPBindPort int `json:"kcpBindPort,omitempty"`
 	// QUICBindPort specifies the QUIC port that the server listens on.
 	// Set this value to 0 will disable this feature.
-	QUICBindPort int `json:"quicBindPort,omitempty" validate:"gte=0,lte=65535"`
+	QUICBindPort int `json:"quicBindPort,omitempty"`
 	// ProxyBindAddr specifies the address that the proxy binds to. This value
 	// may be the same as BindAddr.
 	ProxyBindAddr string `json:"proxyBindAddr,omitempty"`
 	// VhostHTTPPort specifies the port that the server listens for HTTP Vhost
 	// requests. If this value is 0, the server will not listen for HTTP
 	// requests.
-	VhostHTTPPort int `json:"vhostHTTPPort,omitempty" validate:"gte=0,lte=65535"`
+	VhostHTTPPort int `json:"vhostHTTPPort,omitempty"`
 	// VhostHTTPTimeout specifies the response header timeout for the Vhost
 	// HTTP server, in seconds. By default, this value is 60.
 	VhostHTTPTimeout int64 `json:"vhostHTTPTimeout,omitempty"`
 	// VhostHTTPSPort specifies the port that the server listens for HTTPS
 	// Vhost requests. If this value is 0, the server will not listen for HTTPS
 	// requests.
-	VhostHTTPSPort int `json:"vhostHTTPSPort,omitempty" validate:"gte=0,lte=65535"`
+	VhostHTTPSPort int `json:"vhostHTTPSPort,omitempty"`
 	// TCPMuxHTTPConnectPort specifies the port that the server listens for TCP
 	// HTTP CONNECT requests. If the value is 0, the server will not multiplex TCP
 	// requests on one single port. If it's not - it will listen on this value for
 	// HTTP CONNECT requests.
-	TCPMuxHTTPConnectPort int `json:"tcpmuxHTTPConnectPort,omitempty" validate:"gte=0,lte=65535"`
+	TCPMuxHTTPConnectPort int `json:"tcpmuxHTTPConnectPort,omitempty"`
 	// If TCPMuxPassthrough is true, frps won't do any update on traffic.
 	TCPMuxPassthrough bool `json:"tcpmuxPassthrough,omitempty"`
 	// SubDomainHost specifies the domain that will be attached to sub-domains
@@ -76,8 +76,6 @@ type ServerConfig struct {
 
 	Transport ServerTransportConfig `json:"transport,omitempty"`
 
-	TLS TLSServerConfig `json:"tls,omitempty"`
-
 	// DetailedErrorsToClient defines whether to send the specific error (with
 	// debug info) to frpc. By default, this value is true.
 	DetailedErrorsToClient *bool `json:"detailedErrorsToClient,omitempty"`
@@ -109,9 +107,6 @@ func (c *ServerConfig) Complete() {
 	if c.ProxyBindAddr == "" {
 		c.ProxyBindAddr = c.BindAddr
 	}
-	if c.TLS.TrustedCaFile != "" {
-		c.TLS.Force = true
-	}
 
 	if c.WebServer.Port > 0 {
 		c.WebServer.Addr = util.EmptyOr(c.WebServer.Addr, "0.0.0.0")
@@ -125,10 +120,10 @@ func (c *ServerConfig) Complete() {
 }
 
 type AuthServerConfig struct {
-	Method               string               `json:"method,omitempty"`
-	AdditionalAuthScopes []AuthScope          `json:"additionalAuthScopes,omitempty"`
-	Token                string               `json:"token,omitempty"`
-	OIDC                 AuthOIDCServerConfig `json:"oidc,omitempty"`
+	Method           string               `json:"method,omitempty"`
+	AdditionalScopes []AuthScope          `json:"additionalScopes,omitempty"`
+	Token            string               `json:"token,omitempty"`
+	OIDC             AuthOIDCServerConfig `json:"oidc,omitempty"`
 }
 
 func (c *AuthServerConfig) Complete() {
@@ -171,6 +166,8 @@ type ServerTransportConfig struct {
 	HeartbeatTimeout int64 `json:"heartbeatTimeout,omitempty"`
 	// QUIC options.
 	QUIC QUICOptions `json:"quic,omitempty"`
+	// TLS specifies TLS settings for the connection from the client.
+	TLS TLSServerConfig `json:"tls,omitempty"`
 }
 
 func (c *ServerTransportConfig) Complete() {
@@ -180,6 +177,9 @@ func (c *ServerTransportConfig) Complete() {
 	c.MaxPoolCount = util.EmptyOr(c.MaxPoolCount, 5)
 	c.HeartbeatTimeout = util.EmptyOr(c.HeartbeatTimeout, 90)
 	c.QUIC.Complete()
+	if c.TLS.TrustedCaFile != "" {
+		c.TLS.Force = true
+	}
 }
 
 type TLSServerConfig struct {

pkg/config/v1/validation/client.go

@@ -29,6 +29,21 @@ func ValidateClientCommonConfig(c *v1.ClientCommonConfig) (Warning, error) {
 		warnings Warning
 		errs     error
 	)
+	if !lo.Contains(supportedAuthMethods, c.Auth.Method) {
+		errs = AppendError(errs, fmt.Errorf("invalid auth method, optional values are %v", supportedAuthMethods))
+	}
+	if !lo.Every(supportedAuthAdditionalScopes, c.Auth.AdditionalScopes) {
+		errs = AppendError(errs, fmt.Errorf("invalid auth additional scopes, optional values are %v", supportedAuthAdditionalScopes))
+	}
+
+	if err := validateLogConfig(&c.Log); err != nil {
+		errs = AppendError(errs, err)
+	}
+
+	if err := validateWebServerConfig(&c.WebServer); err != nil {
+		errs = AppendError(errs, err)
+	}
+
 	if c.Transport.HeartbeatTimeout > 0 && c.Transport.HeartbeatInterval > 0 {
 		if c.Transport.HeartbeatTimeout < c.Transport.HeartbeatInterval {
 			errs = AppendError(errs, fmt.Errorf("invalid transport.heartbeatTimeout, heartbeat timeout should not less than heartbeat interval"))
@@ -48,8 +63,8 @@ func ValidateClientCommonConfig(c *v1.ClientCommonConfig) (Warning, error) {
 		warnings = AppendError(warnings, checkTLSConfig("transport.tls.trustedCaFile", c.Transport.TLS.TrustedCaFile))
 	}
 
-	if !lo.Contains([]string{"tcp", "kcp", "quic", "websocket", "wss"}, c.Transport.Protocol) {
-		errs = AppendError(errs, fmt.Errorf("invalid transport.protocol, only support tcp, kcp, quic, websocket, wss"))
+	if !lo.Contains(supportedTransportProtocols, c.Transport.Protocol) {
+		errs = AppendError(errs, fmt.Errorf("invalid transport.protocol, optional values are %v", supportedTransportProtocols))
 	}
 
 	for _, f := range c.IncludeConfigFiles {
@@ -71,7 +86,7 @@ func ValidateAllClientConfig(c *v1.ClientCommonConfig, pxyCfgs []v1.ProxyConfigu
 		warning, err := ValidateClientCommonConfig(c)
 		warnings = AppendError(warnings, warning)
 		if err != nil {
-			return err, warnings
+			return warnings, err
 		}
 	}
 

pkg/config/v1/validation/common.go

@@ -17,6 +17,8 @@ package validation
 import (
 	"fmt"
 
+	"github.com/samber/lo"
+
 	v1 "github.com/fatedier/frp/pkg/config/v1"
 )
 
@@ -29,13 +31,21 @@ func validateWebServerConfig(c *v1.WebServerConfig) error {
 			return fmt.Errorf("tls.keyFile must be specified when tls is enabled")
 		}
 	}
-	return nil
+
+	return ValidatePort(c.Port, "webServer.port")
 }
 
 // ValidatePort checks that the network port is in range
-func ValidatePort(port int) error {
+func ValidatePort(port int, fieldPath string) error {
 	if 0 <= port && port <= 65535 {
 		return nil
 	}
-	return fmt.Errorf("port number %d must be in the range 0..65535", port)
+	return fmt.Errorf("%s: port number %d must be in the range 0..65535", fieldPath, port)
+}
+
+func validateLogConfig(c *v1.LogConfig) error {
+	if !lo.Contains(supportedLogLevels, c.Level) {
+		return fmt.Errorf("invalid log level, optional values are %v", supportedLogLevels)
+	}
+	return nil
 }