(apache#5522) Enhance environment trait to include values from secret…

…s/configmaps

e2e/advanced/environment_test.go

@@ -25,6 +25,7 @@ package advanced
 import (
 	"context"
 	"fmt"
+	"github.com/stretchr/testify/require"
 	"os"
 	"strings"
 	"testing"
@@ -39,7 +40,58 @@ import (
 	"github.com/apache/camel-k/v2/pkg/util/defaults"
 )
 
-func TestEnvironmentTrait(t *testing.T) {
+func TestEnvironmentTraitVars(t *testing.T) {
+	t.Parallel()
+
+	WithNewTestNamespace(t, func(ctx context.Context, g *WithT, ns string) {
+		operatorID := "camel-k-trait-environment"
+		g.Expect(CopyCamelCatalog(t, ctx, ns, operatorID)).To(Succeed())
+		g.Expect(CopyIntegrationKits(t, ctx, ns, operatorID)).To(Succeed())
+		g.Expect(KamelInstallWithID(t, ctx, operatorID, ns)).To(Succeed())
+
+		g.Eventually(SelectedPlatformPhase(t, ctx, ns, operatorID), TestTimeoutMedium).Should(Equal(v1.IntegrationPlatformPhaseReady))
+
+		// Create configmap
+		var cmData = make(map[string]string)
+		cmData["my-cm-key"] = "hello configmap"
+		err := CreatePlainTextConfigmap(t, ctx, ns, "my-cm", cmData)
+		require.NoError(t, err)
+
+		// Create secret
+		var secData = make(map[string]string)
+		secData["my-sec-key"] = "very top secret"
+		err = CreatePlainTextSecret(t, ctx, ns, "my-sec", secData)
+		require.NoError(t, err)
+
+		t.Run("Run simple env-var", func(t *testing.T) {
+			name := RandomizedSuffixName("envvar")
+			g.Expect(KamelRunWithID(t, ctx, operatorID, ns, "--name", name, "-t", "environment.vars=MY_ENV_VAR='hello world'", "files/envvar.yaml").Execute()).To(Succeed())
+			g.Eventually(IntegrationPodPhase(t, ctx, ns, name), TestTimeoutLong).Should(Equal(corev1.PodRunning))
+			g.Eventually(IntegrationConditionStatus(t, ctx, ns, name, v1.IntegrationConditionReady), TestTimeoutShort).Should(Equal(corev1.ConditionTrue))
+			g.Eventually(IntegrationLogs(t, ctx, ns, name), TestTimeoutShort).Should(ContainSubstring("hello world"))
+		})
+
+		t.Run("Run env-var from configmap", func(t *testing.T) {
+			name := RandomizedSuffixName("envvar-configmap")
+			g.Expect(KamelRunWithID(t, ctx, operatorID, ns, "--name", name, "-t", "environment.vars=MY_ENV_VAR=configmap:my-cm/my-cm-key", "files/envvar.yaml").Execute()).To(Succeed())
+			g.Eventually(IntegrationPodPhase(t, ctx, ns, name), TestTimeoutLong).Should(Equal(corev1.PodRunning))
+			g.Eventually(IntegrationConditionStatus(t, ctx, ns, name, v1.IntegrationConditionReady), TestTimeoutShort).Should(Equal(corev1.ConditionTrue))
+			g.Eventually(IntegrationLogs(t, ctx, ns, name), TestTimeoutShort).Should(ContainSubstring("hello configmap"))
+		})
+
+		t.Run("Run env-var from secret", func(t *testing.T) {
+			name := RandomizedSuffixName("envvar-secret")
+			g.Expect(KamelRunWithID(t, ctx, operatorID, ns, "--name", name, "-t", "environment.vars=MY_ENV_VAR=secret:my-sec/my-sec-key", "files/envvar.yaml").Execute()).To(Succeed())
+			g.Eventually(IntegrationPodPhase(t, ctx, ns, name), TestTimeoutLong).Should(Equal(corev1.PodRunning))
+			g.Eventually(IntegrationConditionStatus(t, ctx, ns, name, v1.IntegrationConditionReady), TestTimeoutShort).Should(Equal(corev1.ConditionTrue))
+			g.Eventually(IntegrationLogs(t, ctx, ns, name), TestTimeoutShort).Should(ContainSubstring("very top secret"))
+		})
+
+		g.Expect(Kamel(t, ctx, "delete", "--all", "-n", ns).Execute()).To(Succeed())
+	})
+}
+
+func TestEnvironmentTraitHttpProxy(t *testing.T) {
 	t.Parallel()
 
 	WithNewTestNamespace(t, func(ctx context.Context, g *WithT, ns string) {
@@ -67,7 +119,7 @@ func TestEnvironmentTrait(t *testing.T) {
 		}
 
 		// Install Camel K with the HTTP proxy environment variable
-		operatorID := "camel-k-trait-environment"
+		operatorID := "camel-k-trait-environment-http"
 		g.Expect(CopyCamelCatalog(t, ctx, ns, operatorID)).To(Succeed())
 		g.Expect(CopyIntegrationKits(t, ctx, ns, operatorID)).To(Succeed())
 		g.Expect(KamelInstallWithID(t, ctx, operatorID, ns, "--operator-env-vars", fmt.Sprintf("HTTP_PROXY=%s", httpProxy), "--operator-env-vars", "NO_PROXY="+strings.Join(noProxy, ","))).To(Succeed())

e2e/advanced/files/envvar.yaml

@@ -0,0 +1,25 @@
+# camel-k: language=yaml
+
+# ---------------------------------------------------------------------------
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ---------------------------------------------------------------------------
+
+- from:
+    uri: "timer:tick"
+    steps:
+      - setBody:
+          simple: "${env:MY_ENV_VAR}"
+      - to: "log:info"

pkg/apis/camel/v1/trait/environment.go

@@ -29,6 +29,7 @@ type EnvironmentTrait struct {
 	HTTPProxy *bool `property:"http-proxy" json:"httpProxy,omitempty"`
 	// A list of environment variables to be added to the integration container.
 	// The syntax is KEY=VALUE, e.g., `MY_VAR="my value"`.
+	// The value may also be a reference to a configmap or secret, e.g. `MY_VAR=configmap:my-cm/my-cm-key`
 	// These take precedence over the previously defined environment variables.
 	Vars []string `property:"vars" json:"vars,omitempty"`
 }

pkg/cmd/run.go

@@ -944,11 +944,11 @@ func (o *runCmdOptions) extractProperties(c client.Client, value string) (*prope
 		// we already validated the existence of files during validate()
 		return loadPropertyFile(strings.Replace(value, "file:", "", 1))
 	case strings.HasPrefix(value, "secret:"):
-		return loadPropertiesFromSecret(o.Context, c, o.Namespace, strings.Replace(value, "secret:", "", 1))
+		return property.LoadPropertiesFromSecret(o.Context, c, o.Namespace, strings.TrimPrefix(value, "secret:"))
 	case strings.HasPrefix(value, "configmap:"):
-		return loadPropertiesFromConfigMap(o.Context, c, o.Namespace, strings.Replace(value, "configmap:", "", 1))
+		return property.LoadPropertiesFromConfigMap(o.Context, c, o.Namespace, strings.TrimPrefix(value, "configmap:"))
 	default:
-		return keyValueProps(value)
+		return property.LoadPropertiesFromString(value)
 	}
 }
 

pkg/cmd/run_support.go

@@ -25,7 +25,6 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
-	"reflect"
 	"strings"
 	"time"
 
@@ -34,7 +33,6 @@ import (
 	"github.com/apache/camel-k/v2/pkg/util/camel"
 	"github.com/apache/camel-k/v2/pkg/util/kubernetes"
 	"github.com/apache/camel-k/v2/pkg/util/resource"
-	"github.com/magiconair/properties"
 	"github.com/spf13/cobra"
 )
 
@@ -77,52 +75,6 @@ func filterFileLocation(maybeFileLocations []string) []string {
 	return filteredOptions
 }
 
-func keyValueProps(value string) (*properties.Properties, error) {
-	return properties.Load([]byte(value), properties.UTF8)
-}
-
-func loadPropertiesFromSecret(ctx context.Context, c client.Client, ns string, name string) (*properties.Properties, error) {
-	secret := kubernetes.LookupSecret(ctx, c, ns, name)
-	if secret == nil {
-		return nil, fmt.Errorf("%s secret not found in %s namespace, make sure to provide it before the Integration can run", name, ns)
-	}
-	return fromMapToProperties(secret.Data,
-		func(v reflect.Value) string { return string(v.Bytes()) },
-		func(v reflect.Value) (*properties.Properties, error) {
-			return properties.Load(v.Bytes(), properties.UTF8)
-		})
-}
-
-func loadPropertiesFromConfigMap(ctx context.Context, c client.Client, ns string, name string) (*properties.Properties, error) {
-	cm := kubernetes.LookupConfigmap(ctx, c, ns, name)
-	if cm == nil {
-		return nil, fmt.Errorf("%s configmap not found in %s namespace, make sure to provide it before the Integration can run", name, ns)
-	}
-	return fromMapToProperties(cm.Data,
-		func(v reflect.Value) string { return v.String() },
-		func(v reflect.Value) (*properties.Properties, error) { return keyValueProps(v.String()) })
-}
-
-func fromMapToProperties(data interface{}, toString func(reflect.Value) string, loadProperties func(reflect.Value) (*properties.Properties, error)) (*properties.Properties, error) {
-	result := properties.NewProperties()
-	m := reflect.ValueOf(data)
-	for _, k := range m.MapKeys() {
-		key := k.String()
-		value := m.MapIndex(k)
-		if strings.HasSuffix(key, ".properties") {
-			p, err := loadProperties(value)
-			if err == nil {
-				result.Merge(p)
-			} else if _, _, err = result.Set(key, toString(value)); err != nil {
-				return nil, fmt.Errorf("cannot assign %s to %s", value, key)
-			}
-		} else if _, _, err := result.Set(key, toString(value)); err != nil {
-			return nil, fmt.Errorf("cannot assign %s to %s", value, key)
-		}
-	}
-	return result, nil
-}
-
 // downloadDependency downloads the file located at the given URL into a temporary folder and returns the local path to the generated temporary file.
 func downloadDependency(ctx context.Context, url url.URL) (string, error) {
 	tctx, cancel := context.WithTimeout(ctx, 3*time.Second)

pkg/trait/environment.go

@@ -18,8 +18,11 @@ limitations under the License.
 package trait
 
 import (
+	"fmt"
 	"os"
+	"strings"
 
+	"github.com/magiconair/properties"
 	"k8s.io/utils/pointer"
 
 	traitv1 "github.com/apache/camel-k/v2/pkg/apis/camel/v1/trait"
@@ -46,7 +49,7 @@ const (
 	envVarCamelKRuntimeVersion = "CAMEL_K_RUNTIME_VERSION"
 	envVarMountPathConfigMaps  = "CAMEL_K_MOUNT_PATH_CONFIGMAPS"
 
-	// Disabling gosec linter as it may triggers:
+	// Disabling gosec linter as it may trigger:
 	//
 	//   pkg/trait/environment.go:41: G101: Potential hardcoded credentials (gosec)
 	//	   envVarMountPathSecrets     = "CAMEL_K_MOUNT_PATH_SECRETS"
@@ -99,12 +102,64 @@ func (t *environmentTrait) Apply(e *Environment) error {
 		}
 	}
 
-	if t.Vars != nil {
-		for _, env := range t.Vars {
-			k, v := property.SplitPropertyFileEntry(env)
+	for _, env := range t.Vars {
+		k, v := property.SplitPropertyFileEntry(env)
+		switch {
+		case strings.HasPrefix(v, "configmap:"):
+			nameSpec := strings.TrimPrefix(v, "configmap:")
+			if err := t.loadEnvVarsFromConfigmap(e, k, nameSpec); err != nil {
+				return err
+			}
+		case strings.HasPrefix(v, "secret:"):
+			nameSpec := strings.TrimPrefix(v, "secret:")
+			if err := t.loadEnvVarsFromSecret(e, k, nameSpec); err != nil {
+				return err
+			}
+		default:
 			envvar.SetVal(&e.EnvVars, k, v)
 		}
 	}
+	return nil
+}
+
+func (t *environmentTrait) loadEnvVarsFromConfigmap(e *Environment, envKey, nameSpec string) error {
+	name, srcKey, err := t.splitValueToNameAndKey(nameSpec)
+	if err != nil {
+		return err
+	}
+	props, err := property.LoadPropertiesFromConfigMap(e.Ctx, e.Client, e.Platform.Namespace, name)
+	if err != nil {
+		return err
+	}
+	return t.setEnvVarsFromProperties(e, envKey, srcKey, props)
+}
+
+func (t *environmentTrait) loadEnvVarsFromSecret(e *Environment, envKey, nameSpec string) error {
+	name, srcKey, err := t.splitValueToNameAndKey(nameSpec)
+	if err != nil {
+		return err
+	}
+	props, err := property.LoadPropertiesFromSecret(e.Ctx, e.Client, e.Platform.Namespace, name)
+	if err != nil {
+		return err
+	}
+	return t.setEnvVarsFromProperties(e, envKey, srcKey, props)
+}
 
+func (t *environmentTrait) setEnvVarsFromProperties(e *Environment, envKey, srcKey string, props *properties.Properties) error {
+	v, ok := props.Get(srcKey)
+	if !ok {
+		return fmt.Errorf("cannot find key '%s' in: %+v", srcKey, props.Keys())
+	}
+	envvar.SetVal(&e.EnvVars, envKey, v)
 	return nil
 }
+
+func (t *environmentTrait) splitValueToNameAndKey(nameSpec string) (string, string, error) {
+	toks := strings.SplitN(nameSpec, "/", 2)
+	if len(toks) != 2 {
+		return "", "", fmt.Errorf("invalid environment trait value: %s", nameSpec)
+	}
+	name, key := toks[0], toks[1]
+	return name, key, nil
+}

pkg/util/property/property.go

@@ -19,9 +19,13 @@ package property
 
 import (
 	"bytes"
+	"context"
 	"fmt"
+	"reflect"
 	"strings"
 
+	"github.com/apache/camel-k/v2/pkg/client"
+	"github.com/apache/camel-k/v2/pkg/util/kubernetes"
 	"github.com/magiconair/properties"
 )
 
@@ -53,6 +57,52 @@ func EncodePropertyFile(sourceProperties map[string]string) (string, error) {
 	return buf.String(), nil
 }
 
+func FromMapToProperties(data interface{}, toString func(reflect.Value) string, loadProperties func(reflect.Value) (*properties.Properties, error)) (*properties.Properties, error) {
+	result := properties.NewProperties()
+	m := reflect.ValueOf(data)
+	for _, k := range m.MapKeys() {
+		key := k.String()
+		value := m.MapIndex(k)
+		if strings.HasSuffix(key, ".properties") {
+			p, err := loadProperties(value)
+			if err == nil {
+				result.Merge(p)
+			} else if _, _, err = result.Set(key, toString(value)); err != nil {
+				return nil, fmt.Errorf("cannot assign %s to %s", value, key)
+			}
+		} else if _, _, err := result.Set(key, toString(value)); err != nil {
+			return nil, fmt.Errorf("cannot assign %s to %s", value, key)
+		}
+	}
+	return result, nil
+}
+
+func LoadPropertiesFromString(value string) (*properties.Properties, error) {
+	return properties.Load([]byte(value), properties.UTF8)
+}
+
+func LoadPropertiesFromConfigMap(ctx context.Context, c client.Client, ns string, name string) (*properties.Properties, error) {
+	cm := kubernetes.LookupConfigmap(ctx, c, ns, name)
+	if cm == nil {
+		return nil, fmt.Errorf("%s configmap not found in %s namespace, make sure to provide it before the Integration can run", name, ns)
+	}
+	return FromMapToProperties(cm.Data,
+		func(v reflect.Value) string { return v.String() },
+		func(v reflect.Value) (*properties.Properties, error) { return LoadPropertiesFromString(v.String()) })
+}
+
+func LoadPropertiesFromSecret(ctx context.Context, c client.Client, ns string, name string) (*properties.Properties, error) {
+	secret := kubernetes.LookupSecret(ctx, c, ns, name)
+	if secret == nil {
+		return nil, fmt.Errorf("%s secret not found in %s namespace, make sure to provide it before the Integration can run", name, ns)
+	}
+	return FromMapToProperties(secret.Data,
+		func(v reflect.Value) string { return string(v.Bytes()) },
+		func(v reflect.Value) (*properties.Properties, error) {
+			return properties.Load(v.Bytes(), properties.UTF8)
+		})
+}
+
 // SplitPropertyFileEntry splits an encoded property into key/value pair, without decoding the content.
 func SplitPropertyFileEntry(entry string) (string, string) {
 	pair := strings.SplitN(entry, "=", 2)

script/Makefile

@@ -397,9 +397,13 @@ endif
 	./script/bundle_kamelets.sh $(KAMELET_CATALOG_REPO) $(KAMELET_CATALOG_REPO_TAG)
 
 build-compile-integration-tests:
+ifndef NOTEST
 	@echo "####### Compiling integration tests..."
 	export CAMEL_K_E2E_JUST_COMPILE="true"; \
 	go test -run nope -tags="integration" ./e2e/...
+else
+	@echo "####### Skipping e2e test compile..."
+endif
 
 clean:
 	# disable gomodules when executing go clean: