Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
0c945d4
Add ForceDecoding Flag To Decoder Config
mahadzaryab1 Aug 24, 2024
045a4ff
Add Test Cases
mahadzaryab1 Aug 24, 2024
31e4de7
Fix Documentation
mahadzaryab1 Aug 24, 2024
6381ebf
Remove Extra Check
mahadzaryab1 Aug 24, 2024
f841385
Do Not Override Zero Fields Check
mahadzaryab1 Aug 24, 2024
7a6796a
Use Map As Input For Tests
mahadzaryab1 Aug 24, 2024
9ffa15f
Rename Flag And Simplify godoc
mahadzaryab1 Aug 24, 2024
57a5bba
Fix Test Name
mahadzaryab1 Aug 24, 2024
022e0dc
Fix Wording of godoc
mahadzaryab1 Aug 24, 2024
d0b44fb
Use interface{} Instead of map[string]interface{}
mahadzaryab1 Aug 24, 2024
ebd5641
Fix Test Case
mahadzaryab1 Aug 24, 2024
3a956c9
Fix Variable Name
mahadzaryab1 Aug 24, 2024
c20b1ab
Fix godoc
mahadzaryab1 Aug 24, 2024
04189fe
Address Feedback From PR Review
mahadzaryab1 Aug 26, 2024
8078296
Change Logic To Set InputVal And Not Erase OutputVal
mahadzaryab1 Aug 27, 2024
5a595df
Add Additional Test With Type Hook
mahadzaryab1 Aug 27, 2024
7072f97
Add Extra Test Case And Extract Bool Expression Into Variable
mahadzaryab1 Aug 27, 2024
1253cd5
Fix Typo
mahadzaryab1 Aug 27, 2024
5d56b9c
Add More Test Cases
mahadzaryab1 Aug 27, 2024
72963fc
Rename Variable
mahadzaryab1 Aug 27, 2024
b0357fa
Fix Test Case
mahadzaryab1 Aug 27, 2024
30f2b22
Address Feedback From PR Review
mahadzaryab1 Aug 27, 2024
817e61e
Simplify Test Strings
mahadzaryab1 Aug 27, 2024
40f8be1
Use More Descriptive Test Name
mahadzaryab1 Aug 27, 2024
39ae1f1
Add Test Cases For Append Hook
mahadzaryab1 Sep 15, 2024
2ff30a1
Run Linter
mahadzaryab1 Sep 19, 2024
d1548f8
Remove Debug Statement
mahadzaryab1 Sep 19, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 23 additions & 7 deletions mapstructure.go
Original file line number Diff line number Diff line change
Expand Up @@ -278,6 +278,10 @@ type DecoderConfig struct {
// field name or tag. Defaults to `strings.EqualFold`. This can be used
// to implement case-sensitive tag values, support snake casing, etc.
MatchName func(mapKey, fieldName string) bool

// DecodeNil, if set to true, will cause the DecodeHook (if present) to run
// even if the input is nil. This can be used to provide default values.
DecodeNil bool
}

// A Decoder takes a raw interface value and turns it into structured
Expand Down Expand Up @@ -451,6 +455,8 @@ func (d *Decoder) decode(name string, input interface{}, outVal reflect.Value) e
}
}

decodeNil := d.config.DecodeNil && d.config.DecodeHook != nil

if input == nil {
// If the data is nil, then we don't set anything, unless ZeroFields is set
// to true.
Expand All @@ -461,17 +467,27 @@ func (d *Decoder) decode(name string, input interface{}, outVal reflect.Value) e
d.config.Metadata.Keys = append(d.config.Metadata.Keys, name)
}
}
return nil

if !decodeNil {
return nil
}
}

if !inputVal.IsValid() {
// If the input value is invalid, then we just set the value
// to be the zero value.
outVal.Set(reflect.Zero(outVal.Type()))
if d.config.Metadata != nil && name != "" {
d.config.Metadata.Keys = append(d.config.Metadata.Keys, name)
if !decodeNil {
// If the input value is invalid, then we just set the value
// to be the zero value.
outVal.Set(reflect.Zero(outVal.Type()))
if d.config.Metadata != nil && name != "" {
d.config.Metadata.Keys = append(d.config.Metadata.Keys, name)
}
return nil
}
return nil

// If we get here, we have an untyped nil so the type of the input is assumed.
// We do this because all subsequent code requires a valid value for inputVal.
var mapVal map[string]interface{}
inputVal = reflect.MakeMap(reflect.TypeOf(mapVal))

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@mahadzaryab1 I was previously concerned with this line since we're setting the input to a map when the destination field could be a primitive field. I noticed a bunch of unit tests in OTEL Collector started failing if I enable DecodeNil=true because of this open-telemetry/opentelemetry-collector#10260

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@yurishkuro I see. The tests that are failing have a different output kind from a map[string]any. One thing that fixes these tests is simply setting outputVal=inputVal which would remove the assumption of what an input should be and assume that it is the type as the output. However, this doesn't work for the new optional config we added since we expect everything going through this hook here to be a map[string]any, so the Unmarshal method never runs. Is there anything we can do in this hook to force the unmarshaller to run? The other option here would be to specify what type the input is when it is nil. Let me know what you think.

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I may have a fix, testing it out

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sounds good! let me know if i can help with anything

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

#45

}

if d.cachedDecodeHook != nil {
Expand Down
165 changes: 165 additions & 0 deletions mapstructure_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3083,6 +3083,171 @@ func TestDecoder_IgnoreUntaggedFieldsWithStruct(t *testing.T) {
}
}

func TestDecoder_CanPerformDecodingForNilInputs(t *testing.T) {
t.Parallel()

type Transformed struct {
Message string
When string
}

helloHook := func(reflect.Type, reflect.Type, interface{}) (interface{}, error) {
return Transformed{Message: "hello"}, nil
}
goodbyeHook := func(reflect.Type, reflect.Type, interface{}) (interface{}, error) {
return Transformed{Message: "goodbye"}, nil
}
appendHook := func(from reflect.Value, to reflect.Value) (interface{}, error) {
if from.Kind() == reflect.Map {
stringMap := from.Interface().(map[string]interface{})
stringMap["when"] = "see you later"
return stringMap, nil
}
return from.Interface(), nil
}

tests := []struct {
name string
decodeNil bool
input interface{}
result Transformed
expectedResult Transformed
decodeHook DecodeHookFunc
}{
{
name: "decodeNil=true for nil input with hook",
decodeNil: true,
input: nil,
decodeHook: helloHook,
expectedResult: Transformed{Message: "hello"},
},
{
name: "decodeNil=true for nil input without hook",
decodeNil: true,
input: nil,
expectedResult: Transformed{Message: ""},
},
{
name: "decodeNil=false for nil input with hook",
decodeNil: false,
input: nil,
decodeHook: helloHook,
expectedResult: Transformed{Message: ""},
},
{
name: "decodeNil=false for nil input without hook",
decodeNil: false,
input: nil,
expectedResult: Transformed{Message: ""},
},
{
name: "decodeNil=true for non-nil input without hook",
decodeNil: true,
input: map[string]interface{}{"message": "bar"},
expectedResult: Transformed{Message: "bar"},
},
{
name: "decodeNil=true for non-nil input with hook",
decodeNil: true,
input: map[string]interface{}{"message": "bar"},
decodeHook: goodbyeHook,
expectedResult: Transformed{Message: "goodbye"},

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

is this the correct result? It looks like the decode hook has the final say, rather than the actual input.

Copy link
Author

@mahadzaryab1 mahadzaryab1 Sep 7, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The way we have it set up right now is if DecodeNil is true then it will force the hook to run regardless of what the input is. I believe this is what the current behaviour is even without DecodeNil.

},
{
name: "decodeNil=false for non-nil input without hook",
decodeNil: false,
input: map[string]interface{}{"message": "bar"},
expectedResult: Transformed{Message: "bar"},
},
{
name: "decodeNil=false for non-nil input with hook",
decodeNil: false,
input: map[string]interface{}{"message": "bar"},
decodeHook: goodbyeHook,
expectedResult: Transformed{Message: "goodbye"},

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same here.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This one seems okay to me since decodeNil is false so we just run the hook on the input as usual. Let me know if I'm misunderstanding something though.

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

it just doesn't match my understanding of the hooks - in OTEL Collector the hooks are used to provide defaults, but then the regular parsing still takes place. Perhaps it's specific to how the OTEL hooks are implemented, but looking at mapstructure.go L493, even after the hook is invoked the execution still continues, so should't the parsing of user input still happen?

Copy link
Author

@mahadzaryab1 mahadzaryab1 Sep 8, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@yurishkuro Correct me if I'm wrong but it looks like the input gets overridden by the return value of the hook and then the processing continues on that instead of the original input?

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok, I think that's a valid use case to completely replace the input, but based on the documentation the intention of the hook function is to alter the input somehow, not completely replace it. Because if you completely replace it, especially with the actual struct, then the decoding becomes trivial - just assign input to output, whereas the value of this library is in decoding complex structures.

I would suggest adding another test with a new hook where instead of returning a struct, the hook adds another value to the map, e.g.

map[string]any{"message": originalvalue, "when": "see you later"},

this way you can say that yes the hook was run, but it didn't just short-circuit everything, it still proceeded with the normal decoding.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@yurishkuro thanks for the suggestion! i added a few more test cases in. let me know what you think!

},
{
name: "decodeNil=true for nil input without hook and non-empty result",
decodeNil: true,
input: nil,
result: Transformed{Message: "foo"},
expectedResult: Transformed{Message: "foo"},
},
{
name: "decodeNil=true for nil input with hook and non-empty result",
decodeNil: true,
input: nil,
result: Transformed{Message: "foo"},
decodeHook: helloHook,
expectedResult: Transformed{Message: "hello"},
},
{
name: "decodeNil=false for nil input without hook and non-empty result",
decodeNil: false,
input: nil,
result: Transformed{Message: "foo"},
expectedResult: Transformed{Message: "foo"},
},
{
name: "decodeNil=false for nil input with hook and non-empty result",
decodeNil: false,
input: nil,
result: Transformed{Message: "foo"},
decodeHook: helloHook,
expectedResult: Transformed{Message: "foo"},
},
{
name: "decodeNil=false for non-nil input with hook that appends a value",
decodeNil: false,
input: map[string]interface{}{"message": "bar"},
decodeHook: appendHook,
expectedResult: Transformed{Message: "bar", When: "see you later"},
},
{
name: "decodeNil=true for non-nil input with hook that appends a value",
decodeNil: true,
input: map[string]interface{}{"message": "bar"},
decodeHook: appendHook,
expectedResult: Transformed{Message: "bar", When: "see you later"},
},
{
name: "decodeNil=true for nil input with hook that appends a value",
decodeNil: true,
decodeHook: appendHook,
expectedResult: Transformed{When: "see you later"},
},
{
name: "decodeNil=false for nil input with hook that appends a value",
decodeNil: false,
decodeHook: appendHook,
expectedResult: Transformed{},
},
}

for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
config := &DecoderConfig{
Result: &test.result,
DecodeNil: test.decodeNil,
DecodeHook: test.decodeHook,
}

decoder, err := NewDecoder(config)
if err != nil {
t.Fatalf("err: %s", err)
}

if err := decoder.Decode(test.input); err != nil {
t.Fatalf("got an err: %s", err)
}

if test.result != test.expectedResult {
t.Errorf("result should be: %#v, got %#v", test.expectedResult, test.result)
}
})
}
}

func testSliceInput(t *testing.T, input map[string]interface{}, expected *Slice) {
var result Slice
err := Decode(input, &result)
Expand Down