Merge branch 'main' into fix-error-during-agent-shutdown

Makefile

@@ -118,11 +118,9 @@ format: ## Format code
 	go fmt ./... && cd sdk && go fmt ./... && cd ../test/performance && go fmt ./... && cd ../../test/integration && go fmt ./...
 	buf format -w ./sdk/proto/
 
-install-tools: ## Install dependencies in tools.go
-	@echo "Getting Tools"
-	@grep _ ./scripts/tools.go | awk '{print $$2}' | xargs -tI % go get %
+install-tools: ## Install dependencies in tools.go using vendored version see https://www.jvt.me/posts/2023/06/19/go-install-from-mod/
 	@echo "Installing Tools"
-	@grep _ ./scripts/tools.go | awk '{print $$2}' | xargs -tI % go install %
+	@grep _ ./scripts/tools.go | awk '{print $$2}' | xargs -tI % env GOBIN=$$(git rev-parse --show-toplevel)/bin GOWORK=off go install -mod=vendor %
 	@go run github.com/evilmartians/lefthook install pre-push
 
 generate-swagger: ## Generates swagger.json from source code

go.mod

@@ -3,7 +3,6 @@ module github.com/nginx/agent/v2
 go 1.19
 
 require (
-	github.com/alvaroloes/enumer v1.1.2
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/fsnotify/fsnotify v1.6.0
 	github.com/gogo/protobuf v1.3.2
@@ -237,7 +236,6 @@ require (
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
 	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0-rc3 // indirect
-	github.com/pascaldekloe/name v1.0.1 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.6 // indirect
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
 	github.com/pkg/errors v0.9.1 // indirect

go.sum

@@ -94,8 +94,6 @@ github.com/alexkohler/prealloc v1.0.0 h1:Hbq0/3fJPQhNkN0dR95AVrr6R7tou91y0uHG5pO
 github.com/alexkohler/prealloc v1.0.0/go.mod h1:VetnK3dIgFBBKmg0YnD9F9x6Icjd+9cvfHR56wJVlKE=
 github.com/alingse/asasalint v0.0.11 h1:SFwnQXJ49Kx/1GghOFz1XGqHYKp21Kq1nHad/0WQRnw=
 github.com/alingse/asasalint v0.0.11/go.mod h1:nCaoMhw7a9kSJObvQyVzNTPBDbNpdocqrSP7t/cW5+I=
-github.com/alvaroloes/enumer v1.1.2 h1:5khqHB33TZy1GWCO/lZwcroBFh7u+0j40T83VUbfAMY=
-github.com/alvaroloes/enumer v1.1.2/go.mod h1:FxrjvuXoDAx9isTJrv4c+T410zFi0DtXIT0m65DJ+Wo=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
@@ -697,9 +695,6 @@ github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJ
 github.com/otiai10/curr v1.0.0/go.mod h1:LskTG5wDwr8Rs+nNQ+1LlxRjAtTZZjtJW4rMXl6j4vs=
 github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo=
 github.com/otiai10/mint v1.3.1/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
-github.com/pascaldekloe/name v0.0.0-20180628100202-0fd16699aae1/go.mod h1:eD5JxqMiuNYyFNmyY9rkJ/slN8y59oEu4Ei7F8OoKWQ=
-github.com/pascaldekloe/name v1.0.1 h1:9lnXOHeqeHHnWLbKfH6X98+4+ETVqFqxN09UXSjcMb0=
-github.com/pascaldekloe/name v1.0.1/go.mod h1:Z//MfYJnH4jVpQ9wkclwu2I2MkHmXTlT9wR5UZScttM=
 github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE=
 github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU=
 github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek=
@@ -1218,7 +1213,6 @@ golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3
 golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/tools v0.0.0-20190524210228-3d17549cdc6b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
 golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
 golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=

scripts/tools.go

@@ -1,20 +1,21 @@
 //go:build tools
 // +build tools
 
+// https://www.jvt.me/posts/2022/06/15/go-tools-dependency-management/
+
 package tools
 
 import (
-	_ "github.com/alvaroloes/enumer"
+	_ "github.com/bufbuild/buf/cmd/buf"
+	_ "github.com/evilmartians/lefthook"
+	_ "github.com/go-swagger/go-swagger/cmd/swagger"
 	_ "github.com/gogo/protobuf/protoc-gen-gogo"
 	_ "github.com/gogo/protobuf/protoc-gen-gogofast"
 	_ "github.com/golang/mock/mockgen"
+	_ "github.com/golangci/golangci-lint/cmd/golangci-lint"
+	_ "github.com/goreleaser/nfpm/v2/cmd/nfpm"
 	_ "github.com/maxbrunsfeld/counterfeiter/v6"
 	_ "github.com/mwitkow/go-proto-validators/protoc-gen-govalidators"
-	_ "google.golang.org/grpc/cmd/protoc-gen-go-grpc"
 	_ "github.com/pseudomuto/protoc-gen-doc/cmd/protoc-gen-doc"
-	_ "github.com/go-swagger/go-swagger/cmd/swagger"
-	_ "github.com/bufbuild/buf/cmd/buf"
-	_ "github.com/goreleaser/nfpm/v2/cmd/nfpm"
-	_ "github.com/evilmartians/lefthook"
-	_ "github.com/golangci/golangci-lint/cmd/golangci-lint"
+	_ "google.golang.org/grpc/cmd/protoc-gen-go-grpc"
 )

sdk/client/client.go

@@ -5,8 +5,6 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-//go:generate enumer -type=MsgClassification -text -yaml -json -transform=snake -trimprefix=MsgClassification
-
 package client
 
 import (

sdk/config_apply.go

@@ -32,9 +32,10 @@ type ConfigApply struct {
 	notExistDirs map[string]struct{} // set of directories that exists in the config provided payload, but not on disk
 }
 
-func NewConfigApply(
+func NewConfigApplyWithIgnoreDirectives(
 	confFile string,
 	allowedDirectories map[string]struct{},
+	ignoreDirectives []string,
 ) (*ConfigApply, error) {
 	w, err := zip.NewWriter("/")
 	if err != nil {
@@ -47,11 +48,19 @@ func NewConfigApply(
 		notExistDirs: make(map[string]struct{}),
 	}
 	if confFile != "" {
-		return b, b.mapCurrentFiles(confFile, allowedDirectories)
+		return b, b.mapCurrentFiles(confFile, allowedDirectories, ignoreDirectives)
 	}
 	return b, nil
 }
 
+// to ignore directives use NewConfigApplyWithIgnoreDirectives()
+func NewConfigApply(
+	confFile string,
+	allowedDirectories map[string]struct{},
+) (*ConfigApply, error) {
+	return NewConfigApplyWithIgnoreDirectives(confFile, allowedDirectories, []string{})
+}
+
 // Rollback dumps the saved file content, and delete the notExists file. Best effort, will log error and continue
 // if file operation failed during rollback.
 func (b *ConfigApply) Rollback(cause error) error {
@@ -179,10 +188,11 @@ func (b *ConfigApply) RemoveFromNotExists(fullPath string) {
 
 // mapCurrentFiles parse the provided file via cross-plane, generate a list of files, which should be identical to the
 // DirectoryMap, will mark off the files as the config is being applied, any leftovers after complete should be deleted.
-func (b *ConfigApply) mapCurrentFiles(confFile string, allowedDirectories map[string]struct{}) error {
+func (b *ConfigApply) mapCurrentFiles(confFile string, allowedDirectories map[string]struct{}, ignoreDirectives []string) error {
 	log.Debugf("parsing %s", confFile)
 	payload, err := crossplane.Parse(confFile,
 		&crossplane.ParseOptions{
+			IgnoreDirectives:   ignoreDirectives,
 			SingleFile:         false,
 			StopParsingOnError: true,
 		},

sdk/config_apply_test.go

@@ -98,6 +98,7 @@ func TestNewConfigApply(t *testing.T) {
 		name                string
 		confFile            string
 		allowedDirectories  map[string]struct{}
+		ignoreDirectives    []string
 		expectedConfigApply *ConfigApply
 		expectError         bool
 	}{
@@ -107,6 +108,7 @@ func TestNewConfigApply(t *testing.T) {
 			allowedDirectories: map[string]struct{}{
 				tmpDir: {},
 			},
+			ignoreDirectives: []string{},
 			expectedConfigApply: &ConfigApply{
 				existing: map[string]struct{}{
 					defaultConfFile: {},
@@ -124,6 +126,7 @@ func TestNewConfigApply(t *testing.T) {
 			name:               "no config file present",
 			confFile:           "",
 			allowedDirectories: map[string]struct{}{},
+			ignoreDirectives:   []string{},
 			expectedConfigApply: &ConfigApply{
 				existing:     map[string]struct{}{},
 				notExists:    map[string]struct{}{},
@@ -135,6 +138,7 @@ func TestNewConfigApply(t *testing.T) {
 			name:               "empty config file present",
 			confFile:           emptyConfFile,
 			allowedDirectories: map[string]struct{}{},
+			ignoreDirectives:   []string{},
 			expectedConfigApply: &ConfigApply{
 				existing:     map[string]struct{}{},
 				notExists:    map[string]struct{}{},
@@ -146,6 +150,7 @@ func TestNewConfigApply(t *testing.T) {
 			name:               "unknown config file present",
 			confFile:           "/tmp/unknown.conf",
 			allowedDirectories: map[string]struct{}{},
+			ignoreDirectives:   []string{},
 			expectedConfigApply: &ConfigApply{
 				existing:     map[string]struct{}{},
 				notExists:    map[string]struct{}{},
@@ -157,7 +162,7 @@ func TestNewConfigApply(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			configApply, err := NewConfigApply(tc.confFile, tc.allowedDirectories)
+			configApply, err := NewConfigApplyWithIgnoreDirectives(tc.confFile, tc.allowedDirectories, tc.ignoreDirectives)
 			assert.Equal(t, tc.expectedConfigApply.existing, configApply.GetExisting())
 			assert.Equal(t, tc.expectedConfigApply.notExists, configApply.GetNotExists())
 			assert.Equal(t, tc.expectedConfigApply.notExistDirs, configApply.GetNotExistDirs())
@@ -266,8 +271,9 @@ func TestConfigApplyCompleteAndRollback(t *testing.T) {
 	require.NoError(t, os.WriteFile(confFile, []byte(confFileContent), 0644))
 
 	allowedDirectories := map[string]struct{}{tmpDir: {}}
+	ignoreDirectives := []string{}
 
-	configApply, err := NewConfigApply(confFile, allowedDirectories)
+	configApply, err := NewConfigApplyWithIgnoreDirectives(confFile, allowedDirectories, ignoreDirectives)
 	assert.Equal(t, 5, len(configApply.GetExisting()))
 	assert.Nil(t, err)
 

sdk/config_helpers.go

@@ -92,17 +92,19 @@ func (dm DirectoryMap) appendFileWithProto(dir string, fileProto *proto.File) er
 	return nil
 }
 
-// GetNginxConfig parse the configFile into proto.NginxConfig payload, using the provided nginxID, and systemID for
+// GetNginxConfigWithIgnoreDirectives parse the configFile into proto.NginxConfig payload, using the provided nginxID, and systemID for
 // ConfigDescriptor in the NginxConfig. The allowedDirectories is used to allowlist the directories we include
 // in the aux payload.
-func GetNginxConfig(
+func GetNginxConfigWithIgnoreDirectives(
 	confFile,
 	nginxId,
 	systemId string,
 	allowedDirectories map[string]struct{},
+	ignoreDirectives []string,
 ) (*proto.NginxConfig, error) {
 	payload, err := crossplane.Parse(confFile,
 		&crossplane.ParseOptions{
+			IgnoreDirectives:   ignoreDirectives,
 			SingleFile:         false,
 			StopParsingOnError: true,
 		},
@@ -133,6 +135,16 @@ func GetNginxConfig(
 	return nginxConfig, nil
 }
 
+// to ignore directives use GetNginxConfigWithIgnoreDirectives()
+func GetNginxConfig(
+	confFile,
+	nginxId,
+	systemId string,
+	allowedDirectories map[string]struct{},
+) (*proto.NginxConfig, error) {
+	return GetNginxConfigWithIgnoreDirectives(confFile, nginxId, systemId, allowedDirectories, []string{})
+}
+
 // updateNginxConfigFromPayload updates config files from payload.
 func updateNginxConfigFromPayload(
 	confFile string,
@@ -735,9 +747,10 @@ func pingStatusAPIEndpoint(statusAPI string) bool {
 	return true
 }
 
-func GetStatusApiInfo(confFile string) (statusApi string, err error) {
+func GetStatusApiInfoWithIgnoreDirectives(confFile string, ignoreDirectives []string) (statusApi string, err error) {
 	payload, err := crossplane.Parse(confFile,
 		&crossplane.ParseOptions{
+			IgnoreDirectives:   ignoreDirectives,
 			SingleFile:         false,
 			StopParsingOnError: true,
 			CombineConfigs:     true,
@@ -756,7 +769,12 @@ func GetStatusApiInfo(confFile string) (statusApi string, err error) {
 	return "", errors.New("no status api reachable from the agent found")
 }
 
-func GetErrorAndAccessLogs(confFile string) (*proto.ErrorLogs, *proto.AccessLogs, error) {
+// to ignore directives use GetStatusApiInfoWithIgnoreDirectives()
+func GetStatusApiInfo(confFile string) (statusApi string, err error) {
+	return GetStatusApiInfoWithIgnoreDirectives(confFile, []string{})
+}
+
+func GetErrorAndAccessLogsWithIgnoreDirectives(confFile string, ignoreDirectives []string) (*proto.ErrorLogs, *proto.AccessLogs, error) {
 	nginxConfig := &proto.NginxConfig{
 		Action:       proto.NginxConfigAction_RETURN,
 		ConfigData:   nil,
@@ -770,6 +788,7 @@ func GetErrorAndAccessLogs(confFile string) (*proto.ErrorLogs, *proto.AccessLogs
 
 	payload, err := crossplane.Parse(confFile,
 		&crossplane.ParseOptions{
+			IgnoreDirectives:   ignoreDirectives,
 			SingleFile:         false,
 			StopParsingOnError: true,
 		},
@@ -796,6 +815,11 @@ func GetErrorAndAccessLogs(confFile string) (*proto.ErrorLogs, *proto.AccessLogs
 	return nginxConfig.ErrorLogs, nginxConfig.AccessLogs, err
 }
 
+// to ignore directives use GetErrorAndAccessLogsWithIgnoreDirectives()
+func GetErrorAndAccessLogs(confFile string) (*proto.ErrorLogs, *proto.AccessLogs, error) {
+	return GetErrorAndAccessLogsWithIgnoreDirectives(confFile, []string{})
+}
+
 func GetErrorLogs(errorLogs *proto.ErrorLogs) []string {
 	result := []string{}
 	for _, log := range errorLogs.ErrorLog {
@@ -835,7 +859,7 @@ func convertToHexFormat(hexString string) string {
 	return formatted
 }
 
-func GetAppProtectPolicyAndSecurityLogFiles(cfg *proto.NginxConfig) ([]string, []string) {
+func GetAppProtectPolicyAndSecurityLogFilesWithIgnoreDirectives(cfg *proto.NginxConfig, ignoreDirectives []string) ([]string, []string) {
 	policyMap := make(map[string]bool)
 	profileMap := make(map[string]bool)
 
@@ -845,6 +869,7 @@ func GetAppProtectPolicyAndSecurityLogFiles(cfg *proto.NginxConfig) ([]string, [
 
 			payload, err := crossplane.Parse(confFile,
 				&crossplane.ParseOptions{
+					IgnoreDirectives:   ignoreDirectives,
 					SingleFile:         false,
 					StopParsingOnError: true,
 				},
@@ -892,3 +917,8 @@ func GetAppProtectPolicyAndSecurityLogFiles(cfg *proto.NginxConfig) ([]string, [
 
 	return policies, profiles
 }
+
+// to ignore directives use GetAppProtectPolicyAndSecurityLogFilesWithIgnoreDirectives()
+func GetAppProtectPolicyAndSecurityLogFiles(cfg *proto.NginxConfig) ([]string, []string) {
+	return GetAppProtectPolicyAndSecurityLogFilesWithIgnoreDirectives(cfg, []string{})
+}

sdk/config_helpers_test.go

@@ -633,12 +633,13 @@ func TestGetNginxConfig(t *testing.T) {
 		assert.NoError(t, err)
 
 		allowedDirs := map[string]struct{}{}
+		ignoreDirectives := []string{}
 
 		if test.expected.Zaux != nil {
 			allowedDirs[test.expected.Zaux.RootDirectory] = struct{}{}
 			allowedDirs["/tmp/testdata/nginx/"] = struct{}{}
 		}
-		result, err := GetNginxConfig(test.fileName, nginxID, systemID, allowedDirs)
+		result, err := GetNginxConfigWithIgnoreDirectives(test.fileName, nginxID, systemID, allowedDirs, ignoreDirectives)
 		assert.NoError(t, err)
 
 		assert.Equal(t, test.expected.Action, result.Action)
@@ -750,7 +751,8 @@ func TestGetStatusApiInfo(t *testing.T) {
 			output := bytes.Replace(input, []byte("127.0.0.1:80"), []byte(splitUrl), -1)
 			assert.NoError(t, os.WriteFile(test.fileName, output, 0664))
 
-			result, err := GetStatusApiInfo(test.fileName)
+			ignoreDirectives := []string{}
+			result, err := GetStatusApiInfoWithIgnoreDirectives(test.fileName, ignoreDirectives)
 
 			//Update port in expected plusApi with the port of the mock server
 			test.plusApi = strings.Replace(test.plusApi, ":80", ":"+strings.Split(splitUrl, ":")[1], 1)
@@ -1093,8 +1095,9 @@ func TestGetErrorAndAccessLogs(t *testing.T) {
 
 		err = setUpFile(test.fileName, []byte(test.config))
 		assert.NoError(t, err)
+		ignoreDirectives := []string{}
 
-		errorLogs, accessLogs, err := GetErrorAndAccessLogs(test.fileName)
+		errorLogs, accessLogs, err := GetErrorAndAccessLogsWithIgnoreDirectives(test.fileName, ignoreDirectives)
 		assert.NoError(t, err)
 
 		for index, accessLog := range accessLogs.AccessLog {
@@ -1649,11 +1652,12 @@ func TestGetAppProtectPolicyAndSecurityLogFiles(t *testing.T) {
 			assert.NoError(t, err)
 
 			allowedDirs := map[string]struct{}{}
+			ignoreDirectives := []string{}
 
-			cfg, err := GetNginxConfig(tc.file, nginxID, systemID, allowedDirs)
+			cfg, err := GetNginxConfigWithIgnoreDirectives(tc.file, nginxID, systemID, allowedDirs, ignoreDirectives)
 			assert.NoError(t, err)
 
-			policies, profiles := GetAppProtectPolicyAndSecurityLogFiles(cfg)
+			policies, profiles := GetAppProtectPolicyAndSecurityLogFilesWithIgnoreDirectives(cfg, ignoreDirectives)
 			assert.ElementsMatch(t, tc.expPolicies, policies)
 			assert.ElementsMatch(t, tc.expProfiles, profiles)
 		})

src/core/config/config.go

@@ -181,6 +181,7 @@ func GetConfig(clientId string) (*Config, error) {
 		AllowedDirectoriesMap: map[string]struct{}{},
 		DisplayName:           Viper.GetString(DisplayNameKey),
 		InstanceGroup:         Viper.GetString(InstanceGroupKey),
+		IgnoreDirectives:      Viper.GetStringSlice(IgnoreDirectivesKey),
 	}
 
 	for _, dir := range strings.Split(config.ConfigDirs, ":") {