Add support Ignore directives {#229)

* for all crossplane.ParseOptions

sdk/config_apply.go

@@ -35,6 +35,7 @@ type ConfigApply struct {
 func NewConfigApply(
 	confFile string,
 	allowedDirectories map[string]struct{},
+	ignoreDirectives []string,
 ) (*ConfigApply, error) {
 	w, err := zip.NewWriter("/")
 	if err != nil {
@@ -47,7 +48,7 @@ func NewConfigApply(
 		notExistDirs: make(map[string]struct{}),
 	}
 	if confFile != "" {
-		return b, b.mapCurrentFiles(confFile, allowedDirectories)
+		return b, b.mapCurrentFiles(confFile, allowedDirectories, ignoreDirectives)
 	}
 	return b, nil
 }
@@ -179,10 +180,11 @@ func (b *ConfigApply) RemoveFromNotExists(fullPath string) {
 
 // mapCurrentFiles parse the provided file via cross-plane, generate a list of files, which should be identical to the
 // DirectoryMap, will mark off the files as the config is being applied, any leftovers after complete should be deleted.
-func (b *ConfigApply) mapCurrentFiles(confFile string, allowedDirectories map[string]struct{}) error {
+func (b *ConfigApply) mapCurrentFiles(confFile string, allowedDirectories map[string]struct{}, ignoreDirectives []string) error {
 	log.Debugf("parsing %s", confFile)
 	payload, err := crossplane.Parse(confFile,
 		&crossplane.ParseOptions{
+			IgnoreDirectives:   ignoreDirectives,
 			SingleFile:         false,
 			StopParsingOnError: true,
 		},

sdk/config_apply_test.go

@@ -98,6 +98,7 @@ func TestNewConfigApply(t *testing.T) {
 		name                string
 		confFile            string
 		allowedDirectories  map[string]struct{}
+		ignoreDirectives    []string
 		expectedConfigApply *ConfigApply
 		expectError         bool
 	}{
@@ -107,6 +108,7 @@ func TestNewConfigApply(t *testing.T) {
 			allowedDirectories: map[string]struct{}{
 				tmpDir: {},
 			},
+			ignoreDirectives: []string{},
 			expectedConfigApply: &ConfigApply{
 				existing: map[string]struct{}{
 					defaultConfFile: {},
@@ -124,6 +126,7 @@ func TestNewConfigApply(t *testing.T) {
 			name:               "no config file present",
 			confFile:           "",
 			allowedDirectories: map[string]struct{}{},
+			ignoreDirectives:   []string{},
 			expectedConfigApply: &ConfigApply{
 				existing:     map[string]struct{}{},
 				notExists:    map[string]struct{}{},
@@ -135,6 +138,7 @@ func TestNewConfigApply(t *testing.T) {
 			name:               "empty config file present",
 			confFile:           emptyConfFile,
 			allowedDirectories: map[string]struct{}{},
+			ignoreDirectives:   []string{},
 			expectedConfigApply: &ConfigApply{
 				existing:     map[string]struct{}{},
 				notExists:    map[string]struct{}{},
@@ -146,6 +150,7 @@ func TestNewConfigApply(t *testing.T) {
 			name:               "unknown config file present",
 			confFile:           "/tmp/unknown.conf",
 			allowedDirectories: map[string]struct{}{},
+			ignoreDirectives:   []string{},
 			expectedConfigApply: &ConfigApply{
 				existing:     map[string]struct{}{},
 				notExists:    map[string]struct{}{},
@@ -157,7 +162,7 @@ func TestNewConfigApply(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			configApply, err := NewConfigApply(tc.confFile, tc.allowedDirectories)
+			configApply, err := NewConfigApply(tc.confFile, tc.allowedDirectories, tc.ignoreDirectives)
 			assert.Equal(t, tc.expectedConfigApply.existing, configApply.GetExisting())
 			assert.Equal(t, tc.expectedConfigApply.notExists, configApply.GetNotExists())
 			assert.Equal(t, tc.expectedConfigApply.notExistDirs, configApply.GetNotExistDirs())
@@ -266,8 +271,9 @@ func TestConfigApplyCompleteAndRollback(t *testing.T) {
 	require.NoError(t, os.WriteFile(confFile, []byte(confFileContent), 0644))
 
 	allowedDirectories := map[string]struct{}{tmpDir: {}}
+	ignoreDirectives := []string{}
 
-	configApply, err := NewConfigApply(confFile, allowedDirectories)
+	configApply, err := NewConfigApply(confFile, allowedDirectories, ignoreDirectives)
 	assert.Equal(t, 5, len(configApply.GetExisting()))
 	assert.Nil(t, err)
 

sdk/config_helpers.go

@@ -730,9 +730,10 @@ func pingStatusAPIEndpoint(statusAPI string) bool {
 	return true
 }
 
-func GetStatusApiInfo(confFile string) (statusApi string, err error) {
+func GetStatusApiInfo(confFile string, ignoreDirectives []string) (statusApi string, err error) {
 	payload, err := crossplane.Parse(confFile,
 		&crossplane.ParseOptions{
+			IgnoreDirectives:   ignoreDirectives,
 			SingleFile:         false,
 			StopParsingOnError: true,
 			CombineConfigs:     true,
@@ -751,7 +752,7 @@ func GetStatusApiInfo(confFile string) (statusApi string, err error) {
 	return "", errors.New("no status api reachable from the agent found")
 }
 
-func GetErrorAndAccessLogs(confFile string) (*proto.ErrorLogs, *proto.AccessLogs, error) {
+func GetErrorAndAccessLogs(confFile string, ignoreDirectives []string) (*proto.ErrorLogs, *proto.AccessLogs, error) {
 	nginxConfig := &proto.NginxConfig{
 		Action:       proto.NginxConfigAction_RETURN,
 		ConfigData:   nil,
@@ -765,6 +766,7 @@ func GetErrorAndAccessLogs(confFile string) (*proto.ErrorLogs, *proto.AccessLogs
 
 	payload, err := crossplane.Parse(confFile,
 		&crossplane.ParseOptions{
+			IgnoreDirectives:   ignoreDirectives,
 			SingleFile:         false,
 			StopParsingOnError: true,
 		},
@@ -830,7 +832,7 @@ func convertToHexFormat(hexString string) string {
 	return formatted
 }
 
-func GetAppProtectPolicyAndSecurityLogFiles(cfg *proto.NginxConfig) ([]string, []string) {
+func GetAppProtectPolicyAndSecurityLogFiles(cfg *proto.NginxConfig, ignoreDirectives []string) ([]string, []string) {
 	policyMap := make(map[string]bool)
 	profileMap := make(map[string]bool)
 
@@ -840,6 +842,7 @@ func GetAppProtectPolicyAndSecurityLogFiles(cfg *proto.NginxConfig) ([]string, [
 
 			payload, err := crossplane.Parse(confFile,
 				&crossplane.ParseOptions{
+					IgnoreDirectives:   ignoreDirectives,
 					SingleFile:         false,
 					StopParsingOnError: true,
 				},

sdk/config_helpers_test.go

@@ -751,7 +751,8 @@ func TestGetStatusApiInfo(t *testing.T) {
 			output := bytes.Replace(input, []byte("127.0.0.1:80"), []byte(splitUrl), -1)
 			assert.NoError(t, os.WriteFile(test.fileName, output, 0664))
 
-			result, err := GetStatusApiInfo(test.fileName)
+			ignoreDirectives := []string{}
+			result, err := GetStatusApiInfo(test.fileName, ignoreDirectives)
 
 			//Update port in expected plusApi with the port of the mock server
 			test.plusApi = strings.Replace(test.plusApi, ":80", ":"+strings.Split(splitUrl, ":")[1], 1)
@@ -980,8 +981,9 @@ func TestGetErrorAndAccessLogs(t *testing.T) {
 
 		err = setUpFile(test.fileName, []byte(test.config))
 		assert.NoError(t, err)
+		ignoreDirectives := []string{}
 
-		errorLogs, accessLogs, err := GetErrorAndAccessLogs(test.fileName)
+		errorLogs, accessLogs, err := GetErrorAndAccessLogs(test.fileName, ignoreDirectives)
 		assert.NoError(t, err)
 
 		for index, accessLog := range accessLogs.AccessLog {
@@ -1541,7 +1543,7 @@ func TestGetAppProtectPolicyAndSecurityLogFiles(t *testing.T) {
 			cfg, err := GetNginxConfig(tc.file, nginxID, systemID, allowedDirs, ignoreDirectives)
 			assert.NoError(t, err)
 
-			policies, profiles := GetAppProtectPolicyAndSecurityLogFiles(cfg)
+			policies, profiles := GetAppProtectPolicyAndSecurityLogFiles(cfg, ignoreDirectives)
 			assert.ElementsMatch(t, tc.expPolicies, policies)
 			assert.ElementsMatch(t, tc.expProfiles, profiles)
 		})

src/core/environment_test.go

@@ -622,7 +622,7 @@ func TestWriteFiles(t *testing.T) {
 	for _, file := range files {
 		assert.NoFileExists(t, file.GetName())
 	}
-	backup, err := sdk.NewConfigApply("", nil)
+	backup, err := sdk.NewConfigApply("", nil, []string{})
 	assert.NoError(t, err)
 
 	env := EnvironmentType{}
@@ -656,7 +656,7 @@ func TestWriteFilesWhenExists(t *testing.T) {
 
 	AllowedDirectoriesMap := map[string]struct{}{"/tmp": {}}
 
-	backup, err := sdk.NewConfigApply("", nil)
+	backup, err := sdk.NewConfigApply("", nil, []string{})
 	assert.NoError(t, err)
 	for _, file := range files {
 		assert.NoFileExists(t, file.GetName())
@@ -686,7 +686,7 @@ func TestWriteFilesNotAllowed(t *testing.T) {
 			Permissions: "0644",
 		},
 	}
-	backup, err := sdk.NewConfigApply("", nil)
+	backup, err := sdk.NewConfigApply("", nil, []string{})
 	assert.NoError(t, err)
 
 	AllowedDirectoriesMap := map[string]struct{}{"/opt": {}}
@@ -705,7 +705,7 @@ func TestWriteFile(t *testing.T) {
 		Contents:    []byte("contents"),
 		Permissions: "0777",
 	}
-	backup, err := sdk.NewConfigApply("", nil)
+	backup, err := sdk.NewConfigApply("", nil, []string{})
 	assert.NoError(t, err)
 	assert.NoError(t, writeFile(backup, file, "/tmp"))
 	assert.FileExists(t, file.GetName())

src/core/nginx.go

@@ -203,7 +203,7 @@ func (n *NginxBinaryType) GetNginxDetailsFromProcess(nginxProcess Process) *prot
 		nginxDetailsFacade.ConfPath = path
 	}
 
-	url, err := sdk.GetStatusApiInfo(nginxDetailsFacade.ConfPath)
+	url, err := sdk.GetStatusApiInfo(nginxDetailsFacade.ConfPath, n.config.IgnoreDirectives)
 	if err != nil {
 		log.Tracef("Unable to get status api from the configuration: NGINX metrics will be unavailable for this system. please configure a status API to get NGINX metrics: %v", err)
 	}
@@ -388,7 +388,7 @@ func (n *NginxBinaryType) WriteConfig(config *proto.NginxConfig) (*sdk.ConfigApp
 
 	log.Info("Updating NGINX config")
 	var configApply *sdk.ConfigApply
-	configApply, err = sdk.NewConfigApply(details.ConfPath, n.config.AllowedDirectoriesMap)
+	configApply, err = sdk.NewConfigApply(details.ConfPath, n.config.AllowedDirectoriesMap, n.config.IgnoreDirectives)
 	if err != nil {
 		log.Warnf("config_apply error: %s", err)
 		return nil, err
@@ -535,7 +535,7 @@ func (n *NginxBinaryType) writeBackup(config *proto.NginxConfig, confFiles []*pr
 		allowedDirs := map[string]struct{}{"/tmp": {}}
 		path := filepath.Join("/tmp", strconv.FormatInt(time.Now().Unix(), 10))
 
-		configApply, err := sdk.NewConfigApply("/tmp", n.config.AllowedDirectoriesMap)
+		configApply, err := sdk.NewConfigApply("/tmp", n.config.AllowedDirectoriesMap, n.config.IgnoreDirectives)
 		if err != nil {
 			log.Warnf("config_apply error: %s", err)
 			return

src/extensions/nginx-app-protect/nap/nap_metadata.go

@@ -24,6 +24,7 @@ import (
 func UpdateMetadata(
 	cfg *proto.NginxConfig,
 	appProtectWAFDetails *proto.AppProtectWAFDetails,
+	ignoreDirectives []string,
 ) error {
 	previousPrecompiledPublication := false
 	previousMeta := Metadata{}
@@ -49,7 +50,7 @@ func UpdateMetadata(
 		return nil
 	}
 
-	policies, profiles := sdk.GetAppProtectPolicyAndSecurityLogFiles(cfg)
+	policies, profiles := sdk.GetAppProtectPolicyAndSecurityLogFiles(cfg, ignoreDirectives)
 
 	policyBundles := []*BundleMetadata{}
 	profileBundles := []*BundleMetadata{}

src/extensions/nginx-app-protect/nap/nap_metadata_test.go

@@ -171,8 +171,9 @@ func TestUpdateNapMetadata(t *testing.T) {
 				WafLocation:             metadataFile,
 				PrecompiledPublication:  tc.precompPub,
 			}
+			ignoreDirecitves := []string{}
 
-			err = UpdateMetadata(cfg, appProtectWAFDetails)
+			err = UpdateMetadata(cfg, appProtectWAFDetails, ignoreDirecitves)
 			assert.NoError(t, err)
 
 			data, err := os.ReadFile(metadataFile)

src/plugins/agent_api.go

@@ -484,7 +484,7 @@ func (h *NginxHandler) applyNginxConfig(nginxDetail *proto.NginxDetails, buf *by
 		Contents:    buf.Bytes(),
 	}
 
-	configApply, err := sdk.NewConfigApply(protoFile.GetName(), h.config.AllowedDirectoriesMap)
+	configApply, err := sdk.NewConfigApply(protoFile.GetName(), h.config.AllowedDirectoriesMap, h.config.IgnoreDirectives)
 	if err != nil {
 		return fmt.Errorf("unable to write config: %v", err)
 	}

src/plugins/metrics.go

@@ -300,7 +300,7 @@ func createCollectorConfigsMap(config *config.Config, env core.Environment, bina
 			stubStatusApi = detail.StatusUrl
 		}
 
-		errorLogs, accessLogs, err := sdk.GetErrorAndAccessLogs(detail.ConfPath)
+		errorLogs, accessLogs, err := sdk.GetErrorAndAccessLogs(detail.ConfPath, config.IgnoreDirectives)
 		if err != nil {
 			log.Warnf("Error reading access and error logs from config %s %v", detail.ConfPath, err)
 		}

src/plugins/nginx.go

@@ -250,7 +250,7 @@ func (n *Nginx) uploadConfig(config *proto.ConfigDescriptor, messageId string) e
 	}
 
 	if n.isNginxAppProtectEnabled {
-		err = nap.UpdateMetadata(cfg, n.nginxAppProtectSoftwareDetails)
+		err = nap.UpdateMetadata(cfg, n.nginxAppProtectSoftwareDetails, n.config.IgnoreDirectives)
 		if err != nil {
 			log.Errorf("Unable to update NAP metadata: %v", err)
 		}

src/plugins/nginx_test.go

@@ -523,8 +523,9 @@ func TestNginxConfigApply(t *testing.T) {
 
 			env := tutils.GetMockEnvWithProcess()
 			allowedDirectoriesMap := map[string]struct{}{dir: {}}
+			ignoreDirectives := []string{}
 
-			config, err := sdk.NewConfigApply(tempConf.Name(), allowedDirectoriesMap)
+			config, err := sdk.NewConfigApply(tempConf.Name(), allowedDirectoriesMap, ignoreDirectives)
 			assert.NoError(t, err)
 
 			binary := tutils.NewMockNginxBinary()
@@ -895,7 +896,8 @@ func TestNginx_completeConfigApply(t *testing.T) {
 	tempConf, err := os.CreateTemp(dir, "nginx.conf")
 	assert.NoError(t, err)
 	allowedDirectoriesMap := map[string]struct{}{dir: {}}
-	configApply, err := sdk.NewConfigApply(tempConf.Name(), allowedDirectoriesMap)
+	ignoreDirectives := []string{}
+	configApply, err := sdk.NewConfigApply(tempConf.Name(), allowedDirectoriesMap, ignoreDirectives)
 	assert.NoError(t, err)
 
 	response := &NginxConfigValidationResponse{
@@ -987,7 +989,8 @@ func TestNginx_rollbackConfigApply(t *testing.T) {
 	tempConf, err := os.CreateTemp(dir, "nginx.conf")
 	assert.NoError(t, err)
 	allowedDirectoriesMap := map[string]struct{}{dir: {}}
-	configApply, err := sdk.NewConfigApply(tempConf.Name(), allowedDirectoriesMap)
+	ignoreDirectives := []string{}
+	configApply, err := sdk.NewConfigApply(tempConf.Name(), allowedDirectoriesMap, ignoreDirectives)
 	assert.NoError(t, err)
 
 	response := &NginxConfigValidationResponse{