Merge pull request #1853 from pubudu538/analytics-new

Adding analytics for multi environments
wso2 · Dec 14, 2023 · 266b55d · 266b55d
2 parents 5538825 + 9faf4d7
commit 266b55d
Show file tree

Hide file tree

Showing 15 changed files with 161 additions and 12 deletions.
diff --git a/adapter/config/parser.go b/adapter/config/parser.go
@@ -21,6 +21,7 @@
 package config
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"reflect"
@@ -38,7 +39,11 @@ var (
 
 const (
 	// RelativeConfigPath is the relative file path where the configuration file is.
-	relativeConfigPath = "/conf/config.toml"
+	relativeConfigPath   = "/conf/config.toml"
+	gatewayTypeDefault   = "Onprem"
+	gatewayTypeChoreo    = "Choreo"
+	gatewayTypeChoreoPDP = "Choreo-PDP"
+	gatewayTypeValue     = "gatewayType"
 )
 
 // ReadConfigs implements adapter configuration read operation. The read operation will happen only once, hence
@@ -72,6 +77,13 @@ func ReadConfigs() *Config {
 		pkgconf.ResolveConfigEnvValues(reflect.ValueOf(&(adapterConfig.Envoy)).Elem(), "Router", true)
 		pkgconf.ResolveConfigEnvValues(reflect.ValueOf(&(adapterConfig.Enforcer)).Elem(), "Enforcer", false)
 		pkgconf.ResolveConfigEnvValues(reflect.ValueOf(&(adapterConfig.Analytics)).Elem(), "Analytics", false)
+
+		// validate the analytics configuration values
+		validationErr := validateAnalyticsConfigs(adapterConfig)
+		if validationErr != nil {
+			loggerConfig.ErrorC(logging.PrintError(logging.Error1002, logging.BLOCKER, "Error validating the configurations, error: %v", parseErr.Error()))
+			return
+		}
 	})
 	return adapterConfig
 }
@@ -115,3 +127,25 @@ func GetLogConfigPath() (string, error) {
 func GetApkHome() string {
 	return pkgconf.GetApkHome()
 }
+
+func validateAnalyticsConfigs(conf *Config) error {
+
+	gatewayType := gatewayTypeDefault
+	if _, exists := conf.Analytics.Properties[gatewayTypeValue]; exists {
+		gatewayType = conf.Analytics.Properties[gatewayTypeValue]
+	} else {
+		conf.Analytics.Properties[gatewayTypeValue] = gatewayTypeDefault
+	}
+
+	allowedValuesForGatewayType := map[string]bool{
+		gatewayTypeDefault:   true,
+		gatewayTypeChoreo:    true,
+		gatewayTypeChoreoPDP: true,
+	}
+
+	if _, exists := allowedValuesForGatewayType[gatewayType]; !exists {
+		return fmt.Errorf("invalid configuration value for analytics.gatewayType. Allowed values are %s, %s, or %s",
+			gatewayTypeDefault, gatewayTypeChoreo, gatewayTypeChoreoPDP)
+	}
+	return nil
+}
diff --git a/adapter/config/parser_test.go b/adapter/config/parser_test.go
@@ -0,0 +1,73 @@
+/*
+ *  Copyright (c) 2023, WSO2 LLC. (http://www.wso2.org) All Rights Reserved.
+ *
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ *
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ *
+ */
+
+package config
+
+import (
+	"testing"
+)
+
+func TestValidateConfigValuesForValidValues(t *testing.T) {
+
+	validValues := []string{gatewayTypeDefault, gatewayTypeChoreo, gatewayTypeChoreoPDP}
+	conf := ReadConfigs()
+
+	for _, config := range validValues {
+		propertyMap := make(map[string]string)
+		propertyMap[gatewayTypeValue] = config
+		conf.Analytics.Properties = propertyMap
+
+		err := validateAnalyticsConfigs(conf)
+		if err != nil {
+			t.Errorf("Expected validation of '%s' to be successful, but got an error: %v", config, err)
+		}
+	}
+
+	propertyMap := make(map[string]string)
+	conf.Analytics.Properties = propertyMap
+
+	err := validateAnalyticsConfigs(conf)
+	if err != nil {
+		t.Errorf("Expected validation of '%s' to be successful, but got an error: %v", "empty property map", err)
+	}
+
+	propertyMap["notDefinedGatewayTypeProperty"] = "dummy_value"
+	conf.Analytics.Properties = propertyMap
+
+	err = validateAnalyticsConfigs(conf)
+	if err != nil {
+		t.Errorf("Expected validation of '%s' to be successful, but got an error: %v", "not defined gateway type property", err)
+	}
+}
+
+func TestValidateConfigValuesForInvalidValues(t *testing.T) {
+
+	validValues := []string{gatewayTypeDefault + "-test", "choreo", "choreo-pdp"}
+	conf := ReadConfigs()
+
+	for _, config := range validValues {
+		propertyMap := make(map[string]string)
+		propertyMap[gatewayTypeValue] = config
+		conf.Analytics.Properties = propertyMap
+
+		err := validateAnalyticsConfigs(conf)
+		if err == nil {
+			t.Errorf("Expected validation of '%s' to result in an error, but got nil", config)
+		}
+	}
+
+}
diff --git a/.../java/org/wso2/apk/enforcer/analytics/publisher/reporter/cloud/DefaultInputValidator.java b/.../java/org/wso2/apk/enforcer/analytics/publisher/reporter/cloud/DefaultInputValidator.java
@@ -85,6 +85,7 @@ public class DefaultInputValidator {
             new AbstractMap.SimpleImmutableEntry<>(APPLICATION_NAME, String.class),
             new AbstractMap.SimpleImmutableEntry<>(APPLICATION_OWNER, String.class),
             new AbstractMap.SimpleImmutableEntry<>(REGION_ID, String.class),
+            new AbstractMap.SimpleImmutableEntry<>(ENVIRONMENT_ID, String.class),
             new AbstractMap.SimpleImmutableEntry<>(GATEWAY_TYPE, String.class),
             new AbstractMap.SimpleImmutableEntry<>(USER_AGENT_HEADER, String.class),
             new AbstractMap.SimpleImmutableEntry<>(PROXY_RESPONSE_CODE, Integer.class),
@@ -114,6 +115,7 @@ public class DefaultInputValidator {
             new AbstractMap.SimpleImmutableEntry<>(APPLICATION_NAME, String.class),
             new AbstractMap.SimpleImmutableEntry<>(APPLICATION_OWNER, String.class),
             new AbstractMap.SimpleImmutableEntry<>(REGION_ID, String.class),
+            new AbstractMap.SimpleImmutableEntry<>(ENVIRONMENT_ID, String.class),
             new AbstractMap.SimpleImmutableEntry<>(GATEWAY_TYPE, String.class),
             new AbstractMap.SimpleImmutableEntry<>(PROXY_RESPONSE_CODE, Integer.class),
             new AbstractMap.SimpleImmutableEntry<>(TARGET_RESPONSE_CODE, Integer.class))

diff --git a/...est/java/org/wso2/apk/enforcer/analytics/publisher/DefaultFaultMetricBuilderTestCase.java b/...est/java/org/wso2/apk/enforcer/analytics/publisher/DefaultFaultMetricBuilderTestCase.java
@@ -83,6 +83,7 @@ public void testAttributesWithInvalidTypes() throws MetricCreationException, Met
                 .addAttribute(Constants.API_CREATION, "admin")
                 .addAttribute(Constants.API_METHOD, "POST")
                 .addAttribute(Constants.API_CREATOR_TENANT_DOMAIN, "carbon.super")
+                .addAttribute(Constants.ENVIRONMENT_ID, "Development")
                 .addAttribute(Constants.APPLICATION_ID, "3445-6778")
                 .addAttribute(Constants.APPLICATION_NAME, "default")
                 .addAttribute(Constants.APPLICATION_OWNER, "admin")
@@ -110,6 +111,7 @@ public void testMetricBuilder() throws MetricCreationException, MetricReportingE
                 .addAttribute(Constants.API_CREATION, "admin")
                 .addAttribute(Constants.API_METHOD, "POST")
                 .addAttribute(Constants.API_CREATOR_TENANT_DOMAIN, "carbon.super")
+                .addAttribute(Constants.ENVIRONMENT_ID, "Development")
                 .addAttribute(Constants.APPLICATION_ID, "3445-6778")
                 .addAttribute(Constants.APPLICATION_NAME, "default")
                 .addAttribute(Constants.APPLICATION_OWNER, "admin")
@@ -120,7 +122,7 @@ public void testMetricBuilder() throws MetricCreationException, MetricReportingE
                 .build();
 
         Assert.assertFalse(eventMap.isEmpty());
-        Assert.assertEquals(eventMap.size(), 21, "Some attributes are missing from the resulting event map");
+        Assert.assertEquals(eventMap.size(), 22, "Some attributes are missing from the resulting event map");
         Assert.assertEquals(eventMap.get(Constants.EVENT_TYPE), "fault", "Event type should be set to fault");
         Assert.assertEquals(eventMap.get(Constants.API_TYPE), "HTTP", "API type should be set to HTTP");
     }

diff --git a/.../java/org/wso2/apk/enforcer/analytics/publisher/DefaultResponseMetricBuilderTestCase.java b/.../java/org/wso2/apk/enforcer/analytics/publisher/DefaultResponseMetricBuilderTestCase.java
@@ -83,6 +83,7 @@ public void testAttributesWithInvalidTypes() throws MetricCreationException, Met
                 .addAttribute(Constants.API_CONTEXT, "/v1/")
                 .addAttribute(Constants.API_RESOURCE_TEMPLATE, "/resource/{value}")
                 .addAttribute(Constants.API_CREATOR_TENANT_DOMAIN, "carbon.super")
+                .addAttribute(Constants.ENVIRONMENT_ID, "Development")
                 .addAttribute(Constants.DESTINATION, "localhost:8080")
                 .addAttribute(Constants.APPLICATION_ID, "3445-6778")
                 .addAttribute(Constants.APPLICATION_NAME, "default")
@@ -121,6 +122,7 @@ public void testMetricBuilder() throws MetricCreationException, MetricReportingE
                 .addAttribute(Constants.API_CONTEXT, "/v1/")
                 .addAttribute(Constants.API_RESOURCE_TEMPLATE, "/resource/{value}")
                 .addAttribute(Constants.API_CREATOR_TENANT_DOMAIN, "carbon.super")
+                .addAttribute(Constants.ENVIRONMENT_ID, "Development")
                 .addAttribute(Constants.DESTINATION, "localhost:8080")
                 .addAttribute(Constants.APPLICATION_ID, "3445-6778")
                 .addAttribute(Constants.APPLICATION_NAME, "default")
@@ -140,7 +142,7 @@ public void testMetricBuilder() throws MetricCreationException, MetricReportingE
                 .build();
 
         Assert.assertFalse(eventMap.isEmpty());
-        Assert.assertEquals(eventMap.size(), 29, "Some attributes are missing from the resulting event map");
+        Assert.assertEquals(eventMap.size(), 30, "Some attributes are missing from the resulting event map");
         Assert.assertEquals(eventMap.get(Constants.EVENT_TYPE), "response", "Event type should be set to fault");
         Assert.assertEquals(eventMap.get(Constants.USER_AGENT), "Mobile Safari",
                 "User agent should be set to Mobile Safari");

diff --git a/...cs.publishers/src/test/java/org/wso2/apk/enforcer/analytics/publisher/util/TestUtils.java b/...cs.publishers/src/test/java/org/wso2/apk/enforcer/analytics/publisher/util/TestUtils.java
@@ -47,6 +47,7 @@ public static void populateBuilder(MetricEventBuilder builder) throws MetricRepo
                 .addAttribute(Constants.API_CONTEXT, "/v1/")
                 .addAttribute(Constants.API_RESOURCE_TEMPLATE, "/resource/{value}")
                 .addAttribute(Constants.API_CREATOR_TENANT_DOMAIN, "carbon.super")
+                .addAttribute(Constants.ENVIRONMENT_ID, "Development")
                 .addAttribute(Constants.DESTINATION, "localhost:8080")
                 .addAttribute(Constants.APPLICATION_ID, "3445-6778")
                 .addAttribute(Constants.APPLICATION_NAME, "default")

diff --git a/gateway/enforcer/org.wso2.apk.enforcer/build.gradle b/gateway/enforcer/org.wso2.apk.enforcer/build.gradle
@@ -59,6 +59,7 @@ dependencies {
     implementation libs.log4j.core
     implementation libs.log4j.jcl
     implementation libs.log4j.slf4j
+    implementation libs.log4j.slf4j.simple
     implementation libs.minidev.json
     implementation libs.netty.http
     implementation libs.netty.http2

diff --git a/...g.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/analytics/AnalyticsConstants.java b/...g.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/analytics/AnalyticsConstants.java
@@ -36,6 +36,9 @@ public class AnalyticsConstants {
     protected static final String CHOREO_FAULT_SCHEMA = "CHOREO_ERROR";
 
     protected static final String ELK_TYPE = "elk";
-    protected static final String DEFAULT_ELK_PUBLISHER_REPORTER_CLASS 
-            = "org.wso2.apk.enforcer.analytics.publisher.reporter.elk.ELKMetricReporter";
+    protected static final String DEFAULT_ELK_PUBLISHER_REPORTER_CLASS = "org.wso2.apk.enforcer.analytics.publisher.reporter.elk.ELKMetricReporter";
+
+    protected static final String GATEWAY_TYPE_CONFIG_KEY = "gatewayType";
+    protected static final String DEFAULT_GATEWAY_TYPE = "Onprem";
+
 }
diff --git a/...r/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/analytics/AnalyticsUtils.java b/...r/org.wso2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/analytics/AnalyticsUtils.java
@@ -24,11 +24,19 @@
 import org.apache.logging.log4j.Logger;
 import org.wso2.apk.enforcer.commons.model.AuthenticationContext;
 import org.wso2.apk.enforcer.commons.model.RequestContext;
+import org.wso2.apk.enforcer.config.ConfigHolder;
 import org.wso2.apk.enforcer.constants.AnalyticsConstants;
 import org.wso2.apk.enforcer.constants.MetadataConstants;
 import org.wso2.apk.enforcer.models.API;
 import org.wso2.apk.enforcer.subscription.SubscriptionDataHolder;
 
+import java.lang.reflect.Constructor;
+import java.lang.reflect.InvocationTargetException;
+import java.util.Map;
+
+import static org.wso2.apk.enforcer.analytics.AnalyticsConstants.GATEWAY_TYPE_CONFIG_KEY;
+import static org.wso2.apk.enforcer.analytics.AnalyticsConstants.DEFAULT_GATEWAY_TYPE;
+
 /**
  * Common Utility functions
  */
@@ -45,6 +53,15 @@ public static String setDefaultIfNull(String value) {
         return value == null ? AnalyticsConstants.DEFAULT_FOR_UNASSIGNED : value;
     }
 
+    public static String getGatewayType() {
+        Map<String, Object> properties = ConfigHolder.getInstance().getConfig().getAnalyticsConfig().getProperties();
+        String gatewayType = DEFAULT_GATEWAY_TYPE;
+        if (properties != null) {
+            gatewayType = (String) properties.getOrDefault(GATEWAY_TYPE_CONFIG_KEY, DEFAULT_GATEWAY_TYPE);
+        }
+        return gatewayType;
+    }
+
     /**
      * Extracts Authentication Context from the request Context. If Authentication Context is not available,
      * new Authentication Context object will be created with authenticated property is set to false.

diff --git a/....enforcer/src/main/java/org/wso2/apk/enforcer/analytics/ChoreoAnalyticsForWSProvider.java b/....enforcer/src/main/java/org/wso2/apk/enforcer/analytics/ChoreoAnalyticsForWSProvider.java
@@ -164,7 +164,7 @@ public MetaInfo getMetaInfo() {
         MetaInfo metaInfo = new MetaInfo();
         // Correlation ID is as same as X-Request-ID
         metaInfo.setCorrelationId(extAuthMetadata.get(MetadataConstants.CORRELATION_ID_KEY));
-        metaInfo.setGatewayType("ENVOY");
+        metaInfo.setGatewayType(AnalyticsUtils.getGatewayType());
         metaInfo.setRegionId(extAuthMetadata.get(MetadataConstants.REGION_KEY));
         return metaInfo;
     }

diff --git a/...2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/analytics/ChoreoAnalyticsProvider.java b/...2.apk.enforcer/src/main/java/org/wso2/apk/enforcer/analytics/ChoreoAnalyticsProvider.java
@@ -190,7 +190,7 @@ public MetaInfo getMetaInfo() {
         Map<String, Value> fieldsMap = getFieldsMapFromLogEntry();
         MetaInfo metaInfo = new MetaInfo();
         metaInfo.setCorrelationId(getValueAsString(fieldsMap, MetadataConstants.CORRELATION_ID_KEY));
-        metaInfo.setGatewayType(AnalyticsConstants.GATEWAY_LABEL);
+        metaInfo.setGatewayType(AnalyticsUtils.getGatewayType());
         metaInfo.setRegionId(getValueAsString(fieldsMap, MetadataConstants.REGION_KEY));
         return metaInfo;
     }

diff --git a/....enforcer/src/main/java/org/wso2/apk/enforcer/analytics/ChoreoFaultAnalyticsProvider.java b/....enforcer/src/main/java/org/wso2/apk/enforcer/analytics/ChoreoFaultAnalyticsProvider.java
@@ -190,7 +190,7 @@ public MetaInfo getMetaInfo() {
 
         MetaInfo metaInfo = new MetaInfo();
         metaInfo.setRegionId(ConfigHolder.getInstance().getEnvVarConfig().getEnforcerRegionId());
-        metaInfo.setGatewayType(AnalyticsConstants.GATEWAY_LABEL);
+        metaInfo.setGatewayType(AnalyticsUtils.getGatewayType());
         metaInfo.setCorrelationId(requestContext.getRequestID());
         return metaInfo;
     }

diff --git a/helm-charts/templates/data-plane/gateway-components/log-conf.yaml b/helm-charts/templates/data-plane/gateway-components/log-conf.yaml
@@ -153,15 +153,15 @@ data:
     "{{$key}}" = "{{$value}}"
     {{- end}}
     {{- end }}
-    {{- if and .Values.wso2.apk.dp.gatewayRuntime.analytics.publisher}}
+    {{- if and .Values.wso2.apk.dp.gatewayRuntime.analytics.publishers}}
     [analytics.enforcer]
-    {{- range .Values.wso2.apk.dp.gatewayRuntime.analytics.publisher}}
+    {{- range .Values.wso2.apk.dp.gatewayRuntime.analytics.publishers}}
     [[analytics.enforcer.publisher]]
     enabled = {{ .enabled }}
     type = "{{ .type }}"
-    {{- if and .properties}}
+    {{- if and .configProperties}}
     [analytics.enforcer.publisher.configProperties]
-    {{- range $key, $value := .properties}}
+    {{- range $key, $value := .configProperties}}
     "{{$key}}" = "{{$value}}"
     {{- end }}
     {{- end}}

diff --git a/helm-charts/values.yaml.template b/helm-charts/values.yaml.template
@@ -716,6 +716,18 @@ wso2:
           type: "Choreo"
           # -- Choreo analytics secret.
           secretName: "choreo-analytics-secret"
+          # -- Property values for the analytics.
+          properties:
+            property_name : property_value
+          # -- Analytics Publishers
+          publishers:
+          - enabled: true
+            type: "default"
+            configProperties:
+              auth.api.url: "$env{analytics_authURL}"
+              auth.api.token: "$env{analytics_authToken}"
+          - enabled: true
+            type: "elk"
           # -- Optional: File name of the log file.
           logFileName: "logs/enforcer_analytics.log"
           # -- Optional: Log level the analytics data. Can be one of DEBUG, INFO, WARN, ERROR, OFF.

diff --git a/libs.versions.toml b/libs.versions.toml
@@ -61,6 +61,7 @@ log4j-api = {module = "org.apache.logging.log4j:log4j-api", version.ref = "log4j
 log4j-core = {module = "org.apache.logging.log4j:log4j-core", version.ref = "log4j"}
 log4j-jcl = {module = "org.apache.logging.log4j:log4j-jcl", version.ref = "log4j"}
 log4j-slf4j = {module = "org.apache.logging.log4j:log4j-slf4j-impl", version.ref = "log4j"}
+log4j-slf4j-simple = {module = "org.slf4j:slf4j-simple", version.ref = "slf4j-simple"}
 mapstruct = {module = "org.mapstruct:mapstruct", version.ref = "mapstruct"}
 minidev-json = {module = "net.minidev:json-smart", version.ref = "minidev"}
 netty-http = {module = "io.netty:netty-codec-http", version.ref = "netty"}
@@ -196,3 +197,4 @@ jakarta-xml-apis = "4.0.1"
 javax-xml-bind = "2.3.1"
 mock-server-netty = "3.10.8"
 jedis = "4.3.1"
+slf4j-simple = "2.0.9"