proposal: encoding/json: add a type that can distinguish between unset values and null values

[AtlanCI]

I am now using this method to differentiate between unset and null values。 Is it possible to provide the relevant types in the standard library?

package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"unsafe"
)

type JSONString []byte

const _leadOfJSONString = '^'
const _leadOfJSONStringLen = 1

var _emptyJSONString = []byte{'"', '"'}

func (s JSONString) MarshalJSON() ([]byte, error) {
	if !s.IsSet() {
		b := make([]byte, len(_emptyJSONString))
		copy(b, _emptyJSONString)
		return b, nil
	}

	b := make([]byte, 0, len(s)-_leadOfJSONStringLen+len(_emptyJSONString))
	b = append(b, _emptyJSONString[0]) //"
	b = s.AppentString(b)              //aaaaa
	b = append(b, _emptyJSONString[1]) //"
	return b, nil
}

func (s *JSONString) UnmarshalJSON(b []byte) error {
	if len(b) < len(_emptyJSONString) || b[0] != _emptyJSONString[0] || b[len(b)-1] != _emptyJSONString[len(_emptyJSONString)-1] {
		return errors.New("missing quotation mark")
	}

	if len(b) == len(_emptyJSONString) { //empty string
		*s = make([]byte, _leadOfJSONStringLen)
		(*s)[0] = _leadOfJSONString
		return nil
	}

	b = b[1 : len(b)-1]
	bs := *(*string)(unsafe.Pointer(&b))
	s.Set(bs)
	return nil
}

func (s *JSONString) String() string {
	if !s.IsSet() {
		return ""
	}

	return string((*s)[1:])
}

func (s *JSONString) AppentString(buf []byte) []byte {
	if !s.IsSet() {
		return buf
	}

	return append(buf, (*s)[1:]...)
}

func (s *JSONString) Set(v string) {
	if len(*s) == 0 || len(*s)-len(v) > 128 {
		*s = make([]byte, 0, _leadOfJSONStringLen+len(v))
	} else {
		*s = (*s)[:0]
	}

	*s = append([]byte{_leadOfJSONString}, v...)
}

func (s *JSONString) IsSet() bool {
	return len(*s) > 0 && ((*s)[0] == _leadOfJSONString)
}

func (s *JSONString) Clear() {
	*s = nil
}

type StringTestMessage struct {
	ID   int        `json:"id"`
	Name JSONString `json:"name,omitempty"`
}

func StringUnmarshalTest() {
	{
		msg := &StringTestMessage{}
		err := json.Unmarshal([]byte(`{"id":0}`), msg)
		if err != nil {
			panic(err)
		}
		fmt.Printf("%t|%+v\n", msg.Name.IsSet(), msg) //false|&{ID:0 Name:[]}
	}

	{
		msg := &StringTestMessage{}
		err := json.Unmarshal([]byte(`{"id":111}`), msg) //false|&{ID:111 Name:[]}
		if err != nil {
			panic(err)
		}
		fmt.Printf("%t|%+v\n", msg.Name.IsSet(), msg)
	}

	{
		msg := &StringTestMessage{}
		err := json.Unmarshal([]byte(`{"id":111,"name":""}`), msg) //true|&{ID:111 Name:[94]}
		if err != nil {
			panic(err)
		}
		fmt.Printf("%t|%+v\n", msg.Name.IsSet(), msg)
	}

	{
		msg := &StringTestMessage{}
		err := json.Unmarshal([]byte(`{"id":111,"name":"123"}`), msg) //true|&{ID:111 Name:[94 49 50 51]}
		if err != nil {
			panic(err)
		}
		fmt.Printf("%t|%+v\n", msg.Name.IsSet(), msg)
	}
}

func StringMarshalTest() {
	msg := &StringTestMessage{}
	b, err := json.Marshal(msg)
	if err != nil {
		panic(err)
	}
	fmt.Printf("%t|%s\n", msg.Name.IsSet(), b) //false|{"id":0}

	msg.ID = 111
	b, err = json.Marshal(msg)
	if err != nil {
		panic(err)
	}
	fmt.Printf("%t|%s\n", msg.Name.IsSet(), b) //false|{"id":111}

	msg.Name.Set("")
	b, err = json.Marshal(msg)
	if err != nil {
		panic(err)
	}
	fmt.Printf("%t|%s\n", msg.Name.IsSet(), b) //true|{"id":111,"name":""}

	msg.Name.Clear()
	b, err = json.Marshal(msg)
	if err != nil {
		panic(err)
	}
	fmt.Printf("%t|%s\n", msg.Name.IsSet(), b) //false|{"id":111}

	msg.Name.Set("")
	b, err = json.Marshal(msg)
	if err != nil {
		panic(err)
	}
	fmt.Printf("%t|%s\n", msg.Name.IsSet(), b) //true|{"id":111,"name":""}

	msg.Name.Clear()
	b, err = json.Marshal(msg)
	if err != nil {
		panic(err)
	}
	fmt.Printf("%t|%s\n", msg.Name.IsSet(), b) //false|{"id":111}

	msg.Name.Set("")
	b, err = json.Marshal(msg)
	if err != nil {
		panic(err)
	}
	fmt.Printf("%t|%s\n", msg.Name.IsSet(), b) //true|{"id":111,"name":""}

	msg.Name.Set("123")
	b, err = json.Marshal(msg)
	if err != nil {
		panic(err)
	}
	fmt.Printf("%t|%s\n", msg.Name.IsSet(), b) //true|{"id":111,"name":"123"}
}

[mvdan]

Please note that you should fill https://github.com/golang/proposal/blob/master/go2-language-changes.md when proposing a language change.

[AtlanCI]

1. Would this change make Go easier or harder to learn, and why?
   • This would be simpler for the Go language, and you will often encounter similar requirements during development, where you need to distinguish between unset and null values
2. Has this idea, or one like it, been proposed before?
   • I'm not sure if this has been suggested before, the approach I found online is as above
3. Who does this proposal help, and why?
   • I think this change will make the Go language easier to use, although it could be done using pointers. My scenario is that some surveys are to be archived, and these text formatted messages require a distinction between unset and default values. For example, if someone's age is not filled in, a default value is automatically appended when converting to a go struct. If it is an int type the person will be 0 years old.
4. What is the proposed change?
   • Add a type that can distinguish between unset values and null values in the standard library?
5. Does this affect error handling?
   • no
6. Is this about generics?
   • no
7. Can you describe a possible implementation?

package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"unsafe"
)

type JSONString []byte

const _leadOfJSONString = '^'
const _leadOfJSONStringLen = 1

var _emptyJSONString = []byte{'"', '"'}

func (s JSONString) MarshalJSON() ([]byte, error) {
	if !s.IsSet() {
		b := make([]byte, len(_emptyJSONString))
		copy(b, _emptyJSONString)
		return b, nil
	}

	b := make([]byte, 0, len(s)-_leadOfJSONStringLen+len(_emptyJSONString))
	b = append(b, _emptyJSONString[0]) //"
	b = s.AppentString(b)              //aaaaa
	b = append(b, _emptyJSONString[1]) //"
	return b, nil
}

func (s *JSONString) UnmarshalJSON(b []byte) error {
	if len(b) < len(_emptyJSONString) || b[0] != _emptyJSONString[0] || b[len(b)-1] != _emptyJSONString[len(_emptyJSONString)-1] {
		return errors.New("missing quotation mark")
	}

	if len(b) == len(_emptyJSONString) { //empty string
		*s = make([]byte, _leadOfJSONStringLen)
		(*s)[0] = _leadOfJSONString
		return nil
	}

	b = b[1 : len(b)-1]
	bs := *(*string)(unsafe.Pointer(&b))
	s.Set(bs)
	return nil
}

func (s *JSONString) String() string {
	if !s.IsSet() {
		return ""
	}

	return string((*s)[1:])
}

func (s *JSONString) AppentString(buf []byte) []byte {
	if !s.IsSet() {
		return buf
	}

	return append(buf, (*s)[1:]...)
}

func (s *JSONString) Set(v string) {
	if len(*s) == 0 || len(*s)-len(v) > 128 {
		*s = make([]byte, 0, _leadOfJSONStringLen+len(v))
	} else {
		*s = (*s)[:0]
	}

	*s = append([]byte{_leadOfJSONString}, v...)
}

func (s *JSONString) IsSet() bool {
	return len(*s) > 0 && ((*s)[0] == _leadOfJSONString)
}

func (s *JSONString) Clear() {
	*s = nil
}

type StringTestMessage struct {
	ID   int        `json:"id"`
	Name JSONString `json:"name,omitempty"`
}

func StringUnmarshalTest() {
	{
		msg := &StringTestMessage{}
		err := json.Unmarshal([]byte(`{"id":0}`), msg)
		if err != nil {
			panic(err)
		}
		fmt.Printf("%t|%+v\n", msg.Name.IsSet(), msg) //false|&{ID:0 Name:[]}
	}

	{
		msg := &StringTestMessage{}
		err := json.Unmarshal([]byte(`{"id":111}`), msg) //false|&{ID:111 Name:[]}
		if err != nil {
			panic(err)
		}
		fmt.Printf("%t|%+v\n", msg.Name.IsSet(), msg)
	}

	{
		msg := &StringTestMessage{}
		err := json.Unmarshal([]byte(`{"id":111,"name":""}`), msg) //true|&{ID:111 Name:[94]}
		if err != nil {
			panic(err)
		}
		fmt.Printf("%t|%+v\n", msg.Name.IsSet(), msg)
	}

	{
		msg := &StringTestMessage{}
		err := json.Unmarshal([]byte(`{"id":111,"name":"123"}`), msg) //true|&{ID:111 Name:[94 49 50 51]}
		if err != nil {
			panic(err)
		}
		fmt.Printf("%t|%+v\n", msg.Name.IsSet(), msg)
	}
}

func StringMarshalTest() {
	msg := &StringTestMessage{}
	b, err := json.Marshal(msg)
	if err != nil {
		panic(err)
	}
	fmt.Printf("%t|%s\n", msg.Name.IsSet(), b) //false|{"id":0}

	msg.ID = 111
	b, err = json.Marshal(msg)
	if err != nil {
		panic(err)
	}
	fmt.Printf("%t|%s\n", msg.Name.IsSet(), b) //false|{"id":111}

	msg.Name.Set("")
	b, err = json.Marshal(msg)
	if err != nil {
		panic(err)
	}
	fmt.Printf("%t|%s\n", msg.Name.IsSet(), b) //true|{"id":111,"name":""}

	msg.Name.Clear()
	b, err = json.Marshal(msg)
	if err != nil {
		panic(err)
	}
	fmt.Printf("%t|%s\n", msg.Name.IsSet(), b) //false|{"id":111}

	msg.Name.Set("")
	b, err = json.Marshal(msg)
	if err != nil {
		panic(err)
	}
	fmt.Printf("%t|%s\n", msg.Name.IsSet(), b) //true|{"id":111,"name":""}

	msg.Name.Clear()
	b, err = json.Marshal(msg)
	if err != nil {
		panic(err)
	}
	fmt.Printf("%t|%s\n", msg.Name.IsSet(), b) //false|{"id":111}

	msg.Name.Set("")
	b, err = json.Marshal(msg)
	if err != nil {
		panic(err)
	}
	fmt.Printf("%t|%s\n", msg.Name.IsSet(), b) //true|{"id":111,"name":""}

	msg.Name.Set("123")
	b, err = json.Marshal(msg)
	if err != nil {
		panic(err)
	}
	fmt.Printf("%t|%s\n", msg.Name.IsSet(), b) //true|{"id":111,"name":"123"}
}

[seankhliao]

what advantages does this have over using a pointer? (*string) ?

[ianlancetaylor]

This is marked Go 2 but it doesn't seem to be a language change. It seems to be a suggestion for a new type in the encoding/json package. Adjusting accordingly.

[ianlancetaylor]

The encoding/json package already permits distinguishing between unset strings and empty strings by using a pointer. Why should we add a different mechanism? Thanks.

[AtlanCI]

what advantages does this have over using a pointer? (*string) ?

The encoding/json package already permits distinguishing between unset strings and empty strings by using a pointer. Why should we add a different mechanism? Thanks.

I'm not sure if using pointers is an official recommendation and whether there will be memory loss, I will use json. marshal and unmarshal a lot in my scene。

I haven't understood the existing realization of *string in marshal, I will find out later. Then I will test to see if there is a problem with memory or performance. thanks

[michele]

Pointer are almost a solution, but not quite yet.

When working with Web APIs I often have to distinguish between a field in JSON being passed null (unset that value) and a field not being in the JSON at all (don't touch it).

A good solution might be to use a Null[type] built similarly to sql.NullString: if the field is passed null, Valid is false, otherwise Valid is true and inside the struct you can have an Object T field set to the proper value.
Then you could include a pointer to it in a struct so you could distinguish between field not included in JSON (pointer is nil), field being included and null (Valid == false) and field included and set (Valid == true).

However, we miss one bit for it to work properly: if a field in a struct is a pointer and the JSON for it is null, UnmarshalJSON is not called.
My work around has been to unmarshal the JSON to a map[string]json.RawMessage and check if a key is present and it's value is null to manually assign the &Null[T]{Valid: false}, but as you can see it's not quite ideal and it requires to manually do it for each field you're interested in handling this way.

[DeedleFake]

There was a discussion in #48702 about optional types, and some people suggested just putting them in encoding/json instead of having a general one. There are pros and cons to each, and I'm not quite sure what the thinking is about it now. I'm personally of the opinion that an Option[T any] type would be great, but I think that it needs #49085 to work nicely.