diff --git a/error_test.go b/error_test.go index 3c1aa7c..41c33ef 100644 --- a/error_test.go +++ b/error_test.go @@ -75,7 +75,7 @@ func TestErrorMessages(t *testing.T) { cNested := New() cNestedMeta := New() - testMeta := &Meta{"test.source"} + testMeta := &Meta{Source: "test.source"} cMeta.metadata = testMeta cNestedMeta.metadata = testMeta diff --git a/merge.go b/merge.go index da04d10..ec02690 100644 --- a/merge.go +++ b/merge.go @@ -62,6 +62,10 @@ import ( // // field is ignored by Merge // Field string `config:",ignore"` // +// // field is marked as redacted; Unpack will return "[REDACTED]" by default +// // (use ShowRedacted option on Unpack to see original value) +// Field string `config:",redact"` +// // Returns an error if merging fails to normalize and validate the from value. // If duplicate setting names are detected in the input, merging fails as well. // @@ -460,33 +464,59 @@ func normalizeValue( ) (value, Error) { v = chaseValue(v) + // Mark metadata for redacted fields + // Redaction applies to string, []byte, and []rune types + // Actual redaction happens during Unpack + meta := opts.meta + if tagOpts.redact { + isRedactableType := v.Kind() == reflect.String || + (v.Kind() == reflect.Slice && (v.Type().Elem().Kind() == reflect.Uint8 || v.Type().Elem().Kind() == reflect.Int32)) + + if isRedactableType { + var metaCopy Meta + if meta != nil { + metaCopy = *meta + } + + metaCopy.Redacted = true + meta = &metaCopy + } + } + switch v.Type() { case tDuration: d := v.Interface().(time.Duration) - return newString(ctx, opts.meta, d.String()), nil + return newString(ctx, meta, d.String()), nil case tRegexp: r := v.Addr().Interface().(*regexp.Regexp) - return newString(ctx, opts.meta, r.String()), nil + return newString(ctx, meta, r.String()), nil } // handle primitives switch v.Kind() { case reflect.Bool: - return newBool(ctx, opts.meta, v.Bool()), nil + return newBool(ctx, meta, v.Bool()), nil case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: i := v.Int() if i > 0 { - return newUint(ctx, opts.meta, uint64(i)), nil + return newUint(ctx, meta, uint64(i)), nil } - return newInt(ctx, opts.meta, i), nil + return newInt(ctx, meta, i), nil case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: - return newUint(ctx, opts.meta, v.Uint()), nil + return newUint(ctx, meta, v.Uint()), nil case reflect.Float32, reflect.Float64: f := v.Float() - return newFloat(ctx, opts.meta, f), nil + return newFloat(ctx, meta, f), nil case reflect.String: - return normalizeString(ctx, opts, v.String()) + return normalizeString(ctx, opts, meta, v.String()) case reflect.Array, reflect.Slice: + // For arrays/slices, we need to pass the updated metadata + if meta != opts.meta { + // Create new options with updated metadata + newOpts := *opts + newOpts.meta = meta + return normalizeArray(&newOpts, tagOpts, ctx, v) + } return normalizeArray(opts, tagOpts, ctx, v) case reflect.Map: return normalizeMapValue(opts, ctx, v) @@ -503,30 +533,30 @@ func normalizeValue( return normalizeStructValue(opts, ctx, v) default: if v.IsNil() { - return &cfgNil{cfgPrimitive{ctx, opts.meta}}, nil + return &cfgNil{cfgPrimitive{ctx, meta}}, nil } - return nil, raiseUnsupportedInputType(ctx, opts.meta, v) + return nil, raiseUnsupportedInputType(ctx, meta, v) } } -func normalizeString(ctx context, opts *options, str string) (value, Error) { +func normalizeString(ctx context, opts *options, meta *Meta, str string) (value, Error) { if !opts.varexp { - return newString(ctx, opts.meta, str), nil + return newString(ctx, meta, str), nil } varexp, err := parseSplice(str, opts.pathSep, opts.maxIdx, opts.enableNumKeys, opts.escapePath) if err != nil { - return nil, raiseParseSplice(ctx, opts.meta, err) + return nil, raiseParseSplice(ctx, meta, err) } switch p := varexp.(type) { case constExp: - return newString(ctx, opts.meta, string(p)), nil + return newString(ctx, meta, string(p)), nil case *reference: - return newRef(ctx, opts.meta, p), nil + return newRef(ctx, meta, p), nil } - return newSplice(ctx, opts.meta, varexp), nil + return newSplice(ctx, meta, varexp), nil } func fieldOptsOverride(opts *options, fieldName string, idx int) (*options, Error) { diff --git a/opts.go b/opts.go index de2e24e..e47bf60 100644 --- a/opts.go +++ b/opts.go @@ -59,6 +59,7 @@ type options struct { configuredFields *fieldSet ignoreCommas bool + showRedacted bool // When true, shows unredacted values for fields marked with redact tag } type valueCache map[string]spliceValue @@ -195,6 +196,16 @@ func doResolveNOOP(o *options) { }) } +// ShowRedacted option disables automatic redaction during Unpack for fields marked with the `redact` tag. +// By default, when unpacking, fields with the `redact` tag are replaced with "[REDACTED]". +// Use this option with Unpack to preserve and return the original unredacted values. +// The redact tag applies to string, []byte, and []rune types only. +var ShowRedacted Option = doShowRedacted + +func doShowRedacted(o *options) { + o.showRedacted = true +} + var ( // ReplaceValues option configures all merging and unpacking operations to // replace old dictionaries and arrays while merging. Value merging can be diff --git a/redact_test.go b/redact_test.go new file mode 100644 index 0000000..7653c1e --- /dev/null +++ b/redact_test.go @@ -0,0 +1,429 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package ucfg + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDefaultUnpackRedactsFields(t *testing.T) { + type testConfig struct { + Username string `config:"username"` + Password string `config:"password,redact"` + APIKey string `config:"api_key,redact"` + Host string `config:"host"` + } + + input := testConfig{ + Username: "admin", + Password: "secret123", + APIKey: "key-abc-123", + Host: "localhost", + } + + // Config stores original values (always) + cfg, err := NewFrom(input) + require.NoError(t, err) + + // Default Unpack behavior: redacted fields are replaced with "[REDACTED]" + result := make(map[string]interface{}) + err = cfg.Unpack(&result) + require.NoError(t, err) + + assert.Equal(t, "admin", result["username"]) + assert.Equal(t, sREDACT, result["password"]) + assert.Equal(t, sREDACT, result["api_key"]) + assert.Equal(t, "localhost", result["host"]) +} + +func TestUnpackWithShowRedactedOption(t *testing.T) { + type testConfig struct { + Username string `config:"username"` + Password string `config:"password,redact"` + APIKey string `config:"api_key,redact"` + Host string `config:"host"` + } + + input := testConfig{ + Username: "admin", + Password: "secret123", + APIKey: "key-abc-123", + Host: "localhost", + } + + // Config stores original values (always) + cfg, err := NewFrom(input) + require.NoError(t, err) + + // Unpack with ShowRedacted option: original values are shown + result := make(map[string]interface{}) + err = cfg.Unpack(&result, ShowRedacted) + require.NoError(t, err) + + assert.Equal(t, "admin", result["username"]) + assert.Equal(t, "secret123", result["password"]) + assert.Equal(t, "key-abc-123", result["api_key"]) + assert.Equal(t, "localhost", result["host"]) +} + +func TestUnpackRedactsNestedStructs(t *testing.T) { + type database struct { + Host string `config:"host"` + Password string `config:"password,redact"` + } + + type testConfig struct { + AppName string `config:"app_name"` + Database database `config:"database"` + APIToken string `config:"api_token,redact"` + } + + input := testConfig{ + AppName: "myapp", + Database: database{ + Host: "db.example.com", + Password: "dbpass123", + }, + APIToken: "token-xyz-789", + } + + // Test default behavior (redacted during Unpack) + cfg, err := NewFrom(input) + require.NoError(t, err) + + var result testConfig + err = cfg.Unpack(&result) + require.NoError(t, err) + + assert.Equal(t, "myapp", result.AppName) + assert.Equal(t, "db.example.com", result.Database.Host) + assert.Equal(t, sREDACT, result.Database.Password) + assert.Equal(t, sREDACT, result.APIToken) + + // Test with ShowRedacted option (unredacted during Unpack) + var resultUnredacted testConfig + err = cfg.Unpack(&resultUnredacted, ShowRedacted) + require.NoError(t, err) + + assert.Equal(t, "myapp", resultUnredacted.AppName) + assert.Equal(t, "db.example.com", resultUnredacted.Database.Host) + assert.Equal(t, "dbpass123", resultUnredacted.Database.Password) + assert.Equal(t, "token-xyz-789", resultUnredacted.APIToken) +} + +func TestUnpackRedactsArrayElements(t *testing.T) { + type credentials struct { + Username string `config:"username"` + Password string `config:"password,redact"` + } + + type testConfig struct { + Name string `config:"name"` + Creds []credentials `config:"credentials"` + } + + input := testConfig{ + Name: "test", + Creds: []credentials{ + {Username: "user1", Password: "pass1"}, + {Username: "user2", Password: "pass2"}, + }, + } + + // Test default behavior (redacted during Unpack) + cfg, err := NewFrom(input) + require.NoError(t, err) + + var result testConfig + err = cfg.Unpack(&result) + require.NoError(t, err) + + assert.Equal(t, "test", result.Name) + require.Len(t, result.Creds, 2) + assert.Equal(t, "user1", result.Creds[0].Username) + assert.Equal(t, sREDACT, result.Creds[0].Password) + assert.Equal(t, "user2", result.Creds[1].Username) + assert.Equal(t, sREDACT, result.Creds[1].Password) + + // Test with ShowRedacted option (unredacted during Unpack) + var resultUnredacted testConfig + err = cfg.Unpack(&resultUnredacted, ShowRedacted) + require.NoError(t, err) + + assert.Equal(t, "test", resultUnredacted.Name) + require.Len(t, resultUnredacted.Creds, 2) + assert.Equal(t, "user1", resultUnredacted.Creds[0].Username) + assert.Equal(t, "pass1", resultUnredacted.Creds[0].Password) + assert.Equal(t, "user2", resultUnredacted.Creds[1].Username) + assert.Equal(t, "pass2", resultUnredacted.Creds[1].Password) +} + +func TestUnpackWithNoRedactedFields(t *testing.T) { + type testConfig struct { + Name string `config:"name"` + Value int `config:"value"` + } + + input := testConfig{ + Name: "test", + Value: 42, + } + + cfg, err := NewFrom(input) + require.NoError(t, err) + + // Unpack to verify nothing changed + var result testConfig + err = cfg.Unpack(&result) + require.NoError(t, err) + + assert.Equal(t, "test", result.Name) + assert.Equal(t, 42, result.Value) +} + +func TestUnpackRedactsMultipleStrings(t *testing.T) { + type testConfig struct { + StringVal1 string `config:"string_val1,redact"` + StringVal2 string `config:"string_val2,redact"` + NormalVal string `config:"normal_val"` + } + + input := testConfig{ + StringVal1: "secret1", + StringVal2: "secret2", + NormalVal: "public", + } + + // Test default behavior (strings redacted during Unpack) + cfg, err := NewFrom(input) + require.NoError(t, err) + + result := make(map[string]interface{}) + err = cfg.Unpack(&result) + require.NoError(t, err) + + // Redacted string fields should be "[REDACTED]" + assert.Equal(t, sREDACT, result["string_val1"]) + assert.Equal(t, sREDACT, result["string_val2"]) + assert.Equal(t, "public", result["normal_val"]) + + // Test with ShowRedacted option (unredacted during Unpack) + resultUnredacted := make(map[string]interface{}) + err = cfg.Unpack(&resultUnredacted, ShowRedacted) + require.NoError(t, err) + + // Original values should be preserved + assert.Equal(t, "secret1", resultUnredacted["string_val1"]) + assert.Equal(t, "secret2", resultUnredacted["string_val2"]) + assert.Equal(t, "public", resultUnredacted["normal_val"]) +} + +func TestUnpackRedactsOnlyStringsByteRune(t *testing.T) { + type testConfig struct { + StringVal string `config:"string_val,redact"` + IntVal int `config:"int_val,redact"` + BoolVal bool `config:"bool_val,redact"` + FloatVal float64 `config:"float_val,redact"` + } + + input := testConfig{ + StringVal: "secret", + IntVal: 12345, + BoolVal: true, + FloatVal: 3.14, + } + + // Test default behavior - only string should be redacted + cfg, err := NewFrom(input) + require.NoError(t, err) + + result := make(map[string]interface{}) + err = cfg.Unpack(&result) + require.NoError(t, err) + + // Only string field should be redacted + assert.Equal(t, sREDACT, result["string_val"]) + // Non-string types should keep their original values (redact tag ignored) + assert.Equal(t, uint64(12345), result["int_val"]) + assert.Equal(t, true, result["bool_val"]) + assert.Equal(t, 3.14, result["float_val"]) +} + +func TestUnpackRedactsAllSupportedTypes(t *testing.T) { + type testConfig struct { + StringVal string `config:"string_val,redact"` + BytesVal []byte `config:"bytes_val,redact"` + RuneVal []rune `config:"rune_val,redact"` + NormalVal string `config:"normal_val"` + } + + input := testConfig{ + StringVal: "secret-string", + BytesVal: []byte("secret-bytes"), + RuneVal: []rune("secret-rune"), + NormalVal: "public", + } + + // Test default behavior (string, []byte, and []rune redacted during Unpack) + cfg, err := NewFrom(input) + require.NoError(t, err) + + var result testConfig + err = cfg.Unpack(&result) + require.NoError(t, err) + + // All redactable types should be redacted + assert.Equal(t, sREDACT, result.StringVal) + assert.Equal(t, []byte(sREDACT), result.BytesVal) + assert.Equal(t, []rune(sREDACT), result.RuneVal) + assert.Equal(t, "public", result.NormalVal) + + // Test with ShowRedacted option (unredacted during Unpack) + var resultUnredacted testConfig + err = cfg.Unpack(&resultUnredacted, ShowRedacted) + require.NoError(t, err) + + // Original values should be preserved + assert.Equal(t, "secret-string", resultUnredacted.StringVal) + assert.Equal(t, []byte("secret-bytes"), resultUnredacted.BytesVal) + assert.Equal(t, []rune("secret-rune"), resultUnredacted.RuneVal) + assert.Equal(t, "public", resultUnredacted.NormalVal) +} + +func TestUnpackRedactsInlineStructs(t *testing.T) { + type inline struct { + Key string `config:"key"` + Secret string `config:"secret,redact"` + } + + type testConfig struct { + Name string `config:"name"` + Inline inline `config:",inline"` + } + + input := testConfig{ + Name: "test", + Inline: inline{ + Key: "public-key", + Secret: "private-secret", + }, + } + + // Test default behavior (redacted during Unpack) + cfg, err := NewFrom(input) + require.NoError(t, err) + + result := make(map[string]interface{}) + err = cfg.Unpack(&result) + require.NoError(t, err) + + assert.Equal(t, "test", result["name"]) + assert.Equal(t, "public-key", result["key"]) + assert.Equal(t, sREDACT, result["secret"]) + + // Test with ShowRedacted option (unredacted during Unpack) + resultUnredacted := make(map[string]interface{}) + err = cfg.Unpack(&resultUnredacted, ShowRedacted) + require.NoError(t, err) + + assert.Equal(t, "test", resultUnredacted["name"]) + assert.Equal(t, "public-key", resultUnredacted["key"]) + assert.Equal(t, "private-secret", resultUnredacted["secret"]) +} + +func TestUnpackRedactsAfterMerge(t *testing.T) { + type testConfig struct { + Username string `config:"username"` + Password string `config:"password,redact"` + } + + input1 := testConfig{ + Username: "admin", + Password: "secret123", + } + + // Create base config and merge (stores original values) + cfg := New() + err := cfg.Merge(input1) + require.NoError(t, err) + + // Default Unpack applies redaction + result := make(map[string]interface{}) + err = cfg.Unpack(&result) + require.NoError(t, err) + + assert.Equal(t, "admin", result["username"]) + assert.Equal(t, sREDACT, result["password"]) + + // Unpack with ShowRedacted shows original values + result2 := make(map[string]interface{}) + err = cfg.Unpack(&result2, ShowRedacted) + require.NoError(t, err) + + assert.Equal(t, "admin", result2["username"]) + assert.Equal(t, "secret123", result2["password"]) +} + +func TestUnpackRedactsCustomTypes(t *testing.T) { + // Define custom types based on string, []byte, and []rune + type CustomByteString []byte + type CustomString string + type CustomRuneString []rune + + type CustomStruct struct { + CustomB CustomByteString `config:"custom_b,redact"` + CustomS CustomString `config:"custom_s,redact"` + CustomR CustomRuneString `config:"custom_r,redact"` + Normal string `config:"normal"` + } + + input := CustomStruct{ + CustomB: CustomByteString("secret-bytes"), + CustomS: CustomString("secret-string"), + CustomR: CustomRuneString("secret-rune"), + Normal: "public", + } + + // Test default behavior (custom types redacted during Unpack) + cfg, err := NewFrom(input) + require.NoError(t, err) + + var result CustomStruct + err = cfg.Unpack(&result) + require.NoError(t, err) + + // All custom redactable types should be redacted + assert.Equal(t, CustomString(sREDACT), result.CustomS) + assert.Equal(t, CustomByteString(sREDACT), result.CustomB) + assert.Equal(t, CustomRuneString(sREDACT), result.CustomR) + assert.Equal(t, "public", result.Normal) + + // Test with ShowRedacted option (unredacted during Unpack) + var resultUnredacted CustomStruct + err = cfg.Unpack(&resultUnredacted, ShowRedacted) + require.NoError(t, err) + + // Original values should be preserved for custom types + assert.Equal(t, CustomString("secret-string"), resultUnredacted.CustomS) + assert.Equal(t, CustomByteString("secret-bytes"), resultUnredacted.CustomB) + assert.Equal(t, CustomRuneString("secret-rune"), resultUnredacted.CustomR) + assert.Equal(t, "public", resultUnredacted.Normal) +} diff --git a/reify.go b/reify.go index aa4c25a..13a98e0 100644 --- a/reify.go +++ b/reify.go @@ -27,7 +27,7 @@ import ( // and pointers as necessary. // // Unpack supports the options: PathSep, StructTag, ValidatorTag, Env, Resolve, -// ResolveEnv, ReplaceValues, AppendValues, PrependValues. +// ResolveEnv, ReplaceValues, AppendValues, PrependValues, ShowRedacted. // // When unpacking into a value, Unpack first will try to call Unpack if the // value implements the Unpacker interface. Otherwise, Unpack tries to convert @@ -76,6 +76,9 @@ import ( // If the tag sets the `,ignore` flag, the field will not be overwritten. // If the tag sets the `,inline` or `,squash` flag, Unpack will apply the current // configuration namespace to the fields. +// If the tag sets the `,redact` flag, the field will be replaced with "[REDACTED]" +// during Unpack (unless the ShowRedacted option is used, which preserves the original value). +// The redact flag only applies to string, []byte, and []rune types. // If the tag option `replace` is configured, arrays and *ucfg.Config // convertible fields are replaced by the new values. // If the tag options `append` or `prepend` is used, arrays will be merged by @@ -395,6 +398,13 @@ func reifyValue( val value, ) (reflect.Value, Error) { if t.Kind() == reflect.Interface && t.NumMethod() == 0 { + // Apply redaction for interface{} targets + meta := val.meta() + if meta != nil && meta.Redacted && !opts.opts.showRedacted { + // Return redacted value + return reflect.ValueOf(sREDACT), nil + } + reified, err := val.reify(opts.opts) if err != nil { ctx := val.Context() @@ -578,6 +588,18 @@ func reifySliceMerge( tTo reflect.Type, val value, ) (reflect.Value, Error) { + // Apply redaction for []byte and []rune if metadata indicates redacted and showRedacted option is not set + meta := val.meta() + if meta != nil && meta.Redacted && !opts.opts.showRedacted { + elemKind := tTo.Elem().Kind() + switch elemKind { + case reflect.Uint8: // []byte + return reflect.ValueOf([]byte(sREDACT)).Convert(tTo), nil + case reflect.Int32: // []rune + return reflect.ValueOf([]rune(sREDACT)).Convert(tTo), nil + } + } + arr, err := castArr(opts.opts, val) if err != nil { return reflect.Value{}, err @@ -746,6 +768,24 @@ func doReifyPrimitive( } opts.opts.activeFields = previous + // Apply redaction if metadata indicates redacted and showRedacted option is not set + meta := val.meta() + if meta != nil && meta.Redacted && !opts.opts.showRedacted { + // Check if target type is string, []byte, or []rune + kind := baseType.Kind() + switch kind { + case reflect.String: + return reflect.ValueOf(sREDACT).Convert(baseType), nil + case reflect.Slice: + switch baseType.Elem().Kind() { + case reflect.Uint8: // []byte + return reflect.ValueOf([]byte(sREDACT)).Convert(baseType), nil + case reflect.Int32: // []rune + return reflect.ValueOf([]rune(sREDACT)).Convert(baseType), nil + } + } + } + // try primitive conversion kind := baseType.Kind() switch { diff --git a/ucfg.go b/ucfg.go index b38f363..ad8d106 100644 --- a/ucfg.go +++ b/ucfg.go @@ -53,7 +53,8 @@ type fields struct { // Meta holds additional meta data per config value. type Meta struct { - Source string + Source string + Redacted bool } var ( @@ -77,6 +78,8 @@ var ( tRegexp = reflect.TypeOf(regexp.Regexp{}) ) +const sREDACT = "[REDACTED]" + // New creates a new empty Config object. func New() *Config { return &Config{ diff --git a/util.go b/util.go index 39f282f..289efdf 100644 --- a/util.go +++ b/util.go @@ -28,6 +28,7 @@ type tagOptions struct { squash bool ignore bool cfgHandling configHandling + redact bool } // configHandling configures the operation to execute if we merge into a struct @@ -62,6 +63,8 @@ func parseTags(tag string) (string, tagOptions) { opts.cfgHandling = cfgArrAppend case "prepend": opts.cfgHandling = cfgArrPrepend + case "redact": + opts.redact = true } } return s[0], opts