diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b92e8f1..3f80711 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,9 +8,9 @@ jobs: matrix: os: [ubuntu-latest, macos-latest, windows-latest] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - run: go version - run: go test -v - run: | cd hjson-cli - go install -i + go install diff --git a/README.md b/README.md index e33af84..a60d6c7 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,8 @@ Options: -j Output as formatted JSON. -omitRootBraces Omit braces at the root. + -preserveKeyOrder + Preserve key order in objects/maps. -quoteAlways Always quote string values. -v @@ -184,7 +186,7 @@ func main() { ## Comments on struct fields -By using key `comment` in struct field tags you can specify comments to be written on one or more lines preceding the struct field in the Hjson output. +By using key `comment` in struct field tags you can specify comments to be written on one or more lines preceding the struct field in the Hjson output. Another way to output comments is to use *hjson.Node* structs, more on than later. ```go @@ -229,6 +231,88 @@ Output: } ``` +## Read and write comments + +The only way to read comments from Hjson input is to use a destination variable of type *hjson.Node* or **hjson.Node*. The *hjson.Node* must be the root destination, it won't work if you create a field of type *hjson.Node* in some other struct and use that struct as destination. An *hjson.Node* struct is simply a wrapper for a value and comments stored in an *hjson.Comments* struct. It also has several convenience functions, for example *AtIndex()* or *SetKey()* that can be used when you know that the node contains a value of type `[]interface{}` or **hjson.OrderedMap*. All of the elements in `[]interface{}` or **hjson.OrderedMap* will be of type **hjson.Node* in trees created by *hjson.Unmarshal*, but the *hjson.Node* convenience functions unpack the actual values from them. + +When *hjson.Node* or **hjson.Node* is used as destination for Hjson unmarshal the output will be a tree of **hjson.Node* where all of the values contained in tree nodes will be of these types: + +* `nil` (no type) +* `float64`   (if *UseJSONNumber* == `false`) +* *json.Number*   (if *UseJSONNumber* == `true`) +* `string` +* `bool` +* `[]interface{}` +* **hjson.OrderedMap* + +These are just the types used by Hjson unmarshal and the convenience functions, you are free to assign any type of values to nodes in your own code. + +The comments will contain all whitespace chars too (including line feeds) so that an Hjson document can be read and written without altering the layout. This can be disabled by setting the decoding option *WhitespaceAsComments* to `false`. + +```go + +package main + +import ( + "fmt" + + "github.com/hjson/hjson-go/v4" +) + +func main() { + // Now let's look at decoding Hjson data into hjson.Node. + sampleText := []byte(` + { + # specify rate in requests/second + rate: 1000 + array: + [ + foo + bar + ] + }`) + + var node hjson.Node + if err := hjson.Unmarshal(sampleText, &node); err != nil { + panic(err) + } + + node.NK("array").Cm.Before = ` # please specify an array + ` + + if _, _, err := node.NKC("subMap").SetKey("subVal", 1); err != nil { + panic(err) + } + + outBytes, err := hjson.Marshal(node) + if err != nil { + panic(err) + } + + fmt.Println(string(outBytes)) +} +``` + +Output: + +``` + + { + # specify rate in requests/second + rate: 1000 + # please specify an array + array: + [ + foo + bar + ] + subMap: { + subVal: 1 + } + } +``` + + ## Type ambiguity Hjson allows quoteless strings. But if a value is a valid number, boolean or `null` then it will be unmarshalled into that type instead of a string when unmarshalling into `interface{}`. This can lead to unintended consequences if the creator of an Hjson file meant to write a string but didn't think of that the quoteless string they wrote also was a valid number. diff --git a/decode.go b/decode.go index 94ba49e..7c871b4 100644 --- a/decode.go +++ b/decode.go @@ -12,6 +12,12 @@ import ( const maxPointerDepth = 512 +type commentInfo struct { + hasComment bool + cmStart int + cmEnd int +} + // If a destination type implements ElemTyper, Unmarshal() will call ElemType() // on the destination when unmarshalling an array or an object, to see if any // array element or leaf node should be of type string even if it can be treated @@ -35,6 +41,18 @@ type DecoderOptions struct { // is a struct and the input contains object keys which do not match any // non-ignored, exported fields in the destination. DisallowUnknownFields bool + // DisallowDuplicateKeys causes an error to be returned if an object (map) in + // the Hjson input contains duplicate keys. If DisallowDuplicateKeys is set + // to false, later values will silently overwrite previous values for the + // same key. + DisallowDuplicateKeys bool + // WhitespaceAsComments only has any effect when an hjson.Node struct (or + // an *hjson.Node pointer) is used as target for Unmarshal. If + // WhitespaceAsComments is set to true, all whitespace and comments are stored + // in the Node structs so that linefeeds and custom indentation is kept. If + // WhitespaceAsComments instead is set to false, only actual comments are + // stored as comments in Node structs. + WhitespaceAsComments bool } // DefaultDecoderOptions returns the default decoding options. @@ -42,6 +60,8 @@ func DefaultDecoderOptions() DecoderOptions { return DecoderOptions{ UseJSONNumber: false, DisallowUnknownFields: false, + DisallowDuplicateKeys: false, + WhitespaceAsComments: true, } } @@ -52,14 +72,30 @@ type hjsonParser struct { ch byte // The current character structTypeCache map[reflect.Type]structFieldMap willMarshalToJSON bool + nodeDestination bool } var unmarshalerText = reflect.TypeOf((*encoding.TextUnmarshaler)(nil)).Elem() var elemTyper = reflect.TypeOf((*ElemTyper)(nil)).Elem() +func (p *hjsonParser) setComment1(pCm *string, ci commentInfo) { + if ci.hasComment { + *pCm = string(p.data[ci.cmStart:ci.cmEnd]) + } +} + +func (p *hjsonParser) setComment2(pCm *string, ciA, ciB commentInfo) { + if ciA.hasComment && ciB.hasComment { + *pCm = string(p.data[ciA.cmStart:ciA.cmEnd]) + string(p.data[ciB.cmStart:ciB.cmEnd]) + } else { + p.setComment1(pCm, ciA) + p.setComment1(pCm, ciB) + } +} + func (p *hjsonParser) resetAt() { p.at = 0 - p.ch = ' ' + p.next() } func isPunctuatorChar(c byte) bool { @@ -67,22 +103,25 @@ func isPunctuatorChar(c byte) bool { } func (p *hjsonParser) errAt(message string) error { - var i int - col := 0 - line := 1 - for i = p.at - 1; i > 0 && p.data[i] != '\n'; i-- { - col++ - } - for ; i > 0; i-- { - if p.data[i] == '\n' { - line++ + if p.at <= len(p.data) { + var i int + col := 0 + line := 1 + for i = p.at - 1; i > 0 && p.data[i] != '\n'; i-- { + col++ } + for ; i > 0; i-- { + if p.data[i] == '\n' { + line++ + } + } + samEnd := p.at - col + 20 + if samEnd > len(p.data) { + samEnd = len(p.data) + } + return fmt.Errorf("%s at line %d,%d >>> %s", message, line, col, string(p.data[p.at-col:samEnd])) } - samEnd := p.at - col + 20 - if samEnd > len(p.data) { - samEnd = len(p.data) - } - return fmt.Errorf("%s at line %d,%d >>> %s", message, line, col, string(p.data[p.at-col:samEnd])) + return errors.New(message) } func (p *hjsonParser) next() bool { @@ -92,10 +131,22 @@ func (p *hjsonParser) next() bool { p.at++ return true } + p.at++ p.ch = 0 return false } +func (p *hjsonParser) prev() bool { + // get the previous character. + if p.at > 1 { + p.ch = p.data[p.at-2] + p.at-- + return true + } + + return false +} + func (p *hjsonParser) peek(offs int) byte { pos := p.at + offs if pos >= 0 && pos < len(p.data) { @@ -297,18 +348,36 @@ func (p *hjsonParser) readKeyname() (string, error) { } } -func (p *hjsonParser) white() { +func (p *hjsonParser) commonWhite(onlyAfter bool) (commentInfo, bool) { + ci := commentInfo{ + false, + p.at - 1, + 0, + } + var hasLineFeed bool + for p.ch > 0 { // Skip whitespace. for p.ch > 0 && p.ch <= ' ' { + if p.ch == '\n' { + hasLineFeed = true + if onlyAfter { + ci.cmEnd = p.at - 1 + // Skip EOL. + p.next() + return ci, hasLineFeed + } + } p.next() } // Hjson allows comments if p.ch == '#' || p.ch == '/' && p.peek(0) == '/' { + ci.hasComment = p.nodeDestination for p.ch > 0 && p.ch != '\n' { p.next() } } else if p.ch == '/' && p.peek(0) == '*' { + ci.hasComment = p.nodeDestination p.next() p.next() for p.ch > 0 && !(p.ch == '*' && p.peek(0) == '/') { @@ -322,23 +391,66 @@ func (p *hjsonParser) white() { break } } + + // cmEnd is the first char after the comment (i.e. not included in the comment). + ci.cmEnd = p.at - 1 + + return ci, hasLineFeed +} + +func (p *hjsonParser) white() commentInfo { + ci, _ := p.commonWhite(false) + + ci.hasComment = (ci.hasComment || (p.WhitespaceAsComments && (ci.cmEnd > ci.cmStart))) + + return ci +} + +func (p *hjsonParser) whiteAfterComma() commentInfo { + ci, hasLineFeed := p.commonWhite(true) + + ci.hasComment = (ci.hasComment || (p.WhitespaceAsComments && + hasLineFeed && (ci.cmEnd > ci.cmStart))) + + return ci +} + +func (p *hjsonParser) getCommentAfter() commentInfo { + ci, _ := p.commonWhite(true) + + ci.hasComment = (ci.hasComment || (p.WhitespaceAsComments && (ci.cmEnd > ci.cmStart))) + + return ci +} + +func (p *hjsonParser) maybeWrapNode(n *Node, v interface{}) (interface{}, error) { + if p.nodeDestination { + n.Value = v + return n, nil + } + return v, nil } func (p *hjsonParser) readTfnns(dest reflect.Value, t reflect.Type) (interface{}, error) { // Hjson strings can be quoteless // returns string, (json.Number or float64), true, false, or null. + // Or wraps the value in a Node. if isPunctuatorChar(p.ch) { return nil, p.errAt("Found a punctuator character '" + string(p.ch) + "' when expecting a quoteless string (check your syntax)") } chf := p.ch + var node Node value := new(bytes.Buffer) value.WriteByte(p.ch) - // Keep the original dest and t, because we need to check if it implements - // encoding.TextUnmarshaler. - _, newT := unravelDestination(dest, t) + var newT reflect.Type + if !p.nodeDestination { + // Keep the original dest and t, because we need to check if it implements + // encoding.TextUnmarshaler. + _, newT = unravelDestination(dest, t) + } for { p.next() @@ -358,15 +470,15 @@ func (p *hjsonParser) readTfnns(dest reflect.Value, t reflect.Type) (interface{} switch chf { case 'f': if strings.TrimSpace(value.String()) == "false" { - return false, nil + return p.maybeWrapNode(&node, false) } case 'n': if strings.TrimSpace(value.String()) == "null" { - return nil, nil + return p.maybeWrapNode(&node, nil) } case 't': if strings.TrimSpace(value.String()) == "true" { - return true, nil + return p.maybeWrapNode(&node, true) } default: if chf == '-' || chf >= '0' && chf <= '9' { @@ -376,7 +488,7 @@ func (p *hjsonParser) readTfnns(dest reflect.Value, t reflect.Type) (interface{} false, p.willMarshalToJSON || p.DecoderOptions.UseJSONNumber, ); err == nil { - return n, nil + return p.maybeWrapNode(&node, n) } } } @@ -384,7 +496,7 @@ func (p *hjsonParser) readTfnns(dest reflect.Value, t reflect.Type) (interface{} if isEol { // remove any whitespace at the end (ignored in quoteless strings) - return strings.TrimSpace(value.String()), nil + return p.maybeWrapNode(&node, strings.TrimSpace(value.String())) } } value.WriteByte(p.ch) @@ -429,47 +541,68 @@ func getElemTyperType(rv reflect.Value, t reflect.Type) reflect.Type { } func (p *hjsonParser) readArray(dest reflect.Value, t reflect.Type) (value interface{}, err error) { - - // Parse an array value. - // assuming ch == '[' - + var node Node array := make([]interface{}, 0, 1) + // Skip '['. p.next() - p.white() + ciBefore := p.getCommentAfter() + p.setComment1(&node.Cm.InsideFirst, ciBefore) + ciBefore = p.white() if p.ch == ']' { + p.setComment1(&node.Cm.InsideLast, ciBefore) p.next() - return array, nil // empty array + return p.maybeWrapNode(&node, array) // empty array } - elemType := getElemTyperType(dest, t) + var elemType reflect.Type + if !p.nodeDestination { + elemType = getElemTyperType(dest, t) - dest, t = unravelDestination(dest, t) + dest, t = unravelDestination(dest, t) - // All elements in any existing slice/array will be removed, so we only care - // about the type of the new elements that will be created. - if elemType == nil && t != nil && (t.Kind() == reflect.Slice || t.Kind() == reflect.Array) { - elemType = t.Elem() + // All elements in any existing slice/array will be removed, so we only care + // about the type of the new elements that will be created. + if elemType == nil && t != nil && (t.Kind() == reflect.Slice || t.Kind() == reflect.Array) { + elemType = t.Elem() + } } for p.ch > 0 { + var elemNode *Node var val interface{} if val, err = p.readValue(reflect.Value{}, elemType); err != nil { return nil, err } - array = append(array, val) - p.white() + if p.nodeDestination { + var ok bool + if elemNode, ok = val.(*Node); ok { + p.setComment1(&elemNode.Cm.Before, ciBefore) + } + } + // Check white before comma because comma might be on other line. + ciAfter := p.white() // in Hjson the comma is optional and trailing commas are allowed if p.ch == ',' { p.next() - p.white() + ciAfterComma := p.whiteAfterComma() + if elemNode != nil { + existingAfter := elemNode.Cm.After + p.setComment2(&elemNode.Cm.After, ciAfter, ciAfterComma) + elemNode.Cm.After = existingAfter + elemNode.Cm.After + } + // Any comments starting on the line after the comma. + ciAfter = p.white() } if p.ch == ']' { + p.setComment1(&node.Cm.InsideLast, ciAfter) + array = append(array, val) p.next() - return array, nil + return p.maybeWrapNode(&node, array) } - p.white() + array = append(array, val) + ciBefore = ciAfter } return nil, p.errAt("End of input while parsing an array (did you forget a closing ']'?)") @@ -479,44 +612,57 @@ func (p *hjsonParser) readObject( withoutBraces bool, dest reflect.Value, t reflect.Type, + ciBefore commentInfo, ) (value interface{}, err error) { // Parse an object value. - + var node Node + var elemNode *Node object := NewOrderedMap() + // If withoutBraces == true we use the input argument ciBefore as + // Before-comment on the first element of this obj, or as InnerLast-comment + // on this obj if it doesn't contain any elements. If withoutBraces == false + // we ignore the input ciBefore. + if !withoutBraces { // assuming ch == '{' p.next() - } - - p.white() - if p.ch == '}' && !withoutBraces { - p.next() - return object, nil // empty object + ciInsideFirst := p.getCommentAfter() + p.setComment1(&node.Cm.InsideFirst, ciInsideFirst) + ciBefore = p.white() + if p.ch == '}' { + p.setComment1(&node.Cm.InsideLast, ciBefore) + p.next() + return p.maybeWrapNode(&node, object) // empty object + } } var stm structFieldMap - elemType := getElemTyperType(dest, t) - dest, t = unravelDestination(dest, t) + var elemType reflect.Type + if !p.nodeDestination { + elemType = getElemTyperType(dest, t) + + dest, t = unravelDestination(dest, t) + + if elemType == nil && t != nil { + switch t.Kind() { + case reflect.Struct: + var ok bool + stm, ok = p.structTypeCache[t] + if !ok { + stm = getStructFieldInfoMap(t) + p.structTypeCache[t] = stm + } - if elemType == nil && t != nil { - switch t.Kind() { - case reflect.Struct: - var ok bool - stm, ok = p.structTypeCache[t] - if !ok { - stm = getStructFieldInfoMap(t) - p.structTypeCache[t] = stm + case reflect.Map: + // For any key that we find in our loop here below, the new value fully + // replaces any old value. So no need for us to dig down into a tree. + // (This is because we are decoding into a map. If we were decoding into + // a struct we would need to dig down into a tree, to match the behavior + // of Golang's JSON decoder.) + elemType = t.Elem() } - - case reflect.Map: - // For any key that we find in our loop here below, the new value fully - // replaces any old value. So no need for us to dig down into a tree. - // (This is because we are decoding into a map. If we were decoding into - // a struct we would need to dig down into a tree, to match the behavior - // of Golang's JSON decoder.) - elemType = t.Elem() } } @@ -525,7 +671,7 @@ func (p *hjsonParser) readObject( if key, err = p.readKeyname(); err != nil { return nil, err } - p.white() + ciKey := p.white() if p.ch != ':' { return nil, p.errAt("Expected ':' instead of '" + string(p.ch) + "'") } @@ -567,22 +713,49 @@ func (p *hjsonParser) readObject( if val, err = p.readValue(newDest, elemType); err != nil { return nil, err } - object.Set(key, val) - p.white() + if p.nodeDestination { + var ok bool + if elemNode, ok = val.(*Node); ok { + p.setComment1(&elemNode.Cm.Key, ciKey) + elemNode.Cm.Key += elemNode.Cm.Before + elemNode.Cm.Before = "" + p.setComment1(&elemNode.Cm.Before, ciBefore) + } + } + // Check white before comma because comma might be on other line. + ciAfter := p.white() // in Hjson the comma is optional and trailing commas are allowed if p.ch == ',' { p.next() - p.white() + ciAfterComma := p.whiteAfterComma() + if elemNode != nil { + existingAfter := elemNode.Cm.After + p.setComment2(&elemNode.Cm.After, ciAfter, ciAfterComma) + elemNode.Cm.After = existingAfter + elemNode.Cm.After + } + ciAfter = p.white() } if p.ch == '}' && !withoutBraces { + p.setComment1(&node.Cm.InsideLast, ciAfter) + oldValue, isDuplicate := object.Set(key, val) + if isDuplicate && p.DisallowDuplicateKeys { + return nil, p.errAt(fmt.Sprintf("Found duplicate values ('%#v' and '%#v') for key '%v'", + oldValue, val, key)) + } p.next() - return object, nil + return p.maybeWrapNode(&node, object) } - p.white() + oldValue, isDuplicate := object.Set(key, val) + if isDuplicate && p.DisallowDuplicateKeys { + return nil, p.errAt(fmt.Sprintf("Found duplicate values ('%#v' and '%#v') for key '%v'", + oldValue, val, key)) + } + ciBefore = ciAfter } if withoutBraces { - return object, nil + p.setComment1(&node.Cm.InsideLast, ciBefore) + return p.maybeWrapNode(&node, object) } return nil, p.errAt("End of input while parsing an object (did you forget a closing '}'?)") } @@ -590,24 +763,42 @@ func (p *hjsonParser) readObject( // dest and t must not have been unraveled yet here. In readTfnns we need // to check if the original type (or a pointer to it) implements // encoding.TextUnmarshaler. -func (p *hjsonParser) readValue(dest reflect.Value, t reflect.Type) (interface{}, error) { - - // Parse a Hjson value. It could be an object, an array, a string, a number or a word. - - p.white() +func (p *hjsonParser) readValue(dest reflect.Value, t reflect.Type) (ret interface{}, err error) { + ciBefore := p.white() + // Parse an Hjson value. It could be an object, an array, a string, a number or a word. switch p.ch { case '{': - return p.readObject(false, dest, t) + ret, err = p.readObject(false, dest, t, ciBefore) case '[': - return p.readArray(dest, t) + ret, err = p.readArray(dest, t) case '"', '\'': - return p.readString(true) + s, err := p.readString(true) + if err != nil { + return nil, err + } + ret, err = p.maybeWrapNode(&Node{}, s) default: - return p.readTfnns(dest, t) + ret, err = p.readTfnns(dest, t) + // Make sure that any comment will include preceding whitespace. + if p.ch == '#' || p.ch == '/' { + for p.prev() && p.ch <= ' ' { + } + p.next() + } + } + + ciAfter := p.getCommentAfter() + if p.nodeDestination { + if node, ok := ret.(*Node); ok { + p.setComment1(&node.Cm.Before, ciBefore) + p.setComment1(&node.Cm.After, ciAfter) + } } + + return } -func (p *hjsonParser) rootValue(dest reflect.Value) (interface{}, error) { +func (p *hjsonParser) rootValue(dest reflect.Value) (ret interface{}, err error) { // Braces for the root object are optional // We have checked that dest is a pointer before calling rootValue(). @@ -616,37 +807,101 @@ func (p *hjsonParser) rootValue(dest reflect.Value) (interface{}, error) { dest = dest.Elem() t := dest.Type() - p.white() + var errSyntax error + var ciAfter commentInfo + ciBefore := p.white() + switch p.ch { case '{': - return p.checkTrailing(p.readObject(false, dest, t)) + ret, err = p.readObject(false, dest, t, ciBefore) + if err != nil { + return + } + ciAfter, err = p.checkTrailing() + if err != nil { + return + } + if p.nodeDestination { + if node, ok := ret.(*Node); ok { + p.setComment1(&node.Cm.Before, ciBefore) + p.setComment1(&node.Cm.After, ciAfter) + } + } + return case '[': - return p.checkTrailing(p.readArray(dest, t)) + ret, err = p.readArray(dest, t) + if err != nil { + return + } + ciAfter, err = p.checkTrailing() + if err != nil { + return + } + if p.nodeDestination { + if node, ok := ret.(*Node); ok { + p.setComment1(&node.Cm.Before, ciBefore) + p.setComment1(&node.Cm.After, ciAfter) + } + } + return } - // assume we have a root object without braces - res, err := p.checkTrailing(p.readObject(true, dest, t)) - if err == nil { - return res, nil + if ret == nil { + // Assume we have a root object without braces. + ret, errSyntax = p.readObject(true, dest, t, ciBefore) + ciAfter, err = p.checkTrailing() + if errSyntax != nil || err != nil { + // Syntax error, or maybe a single JSON value. + ret = nil + err = nil + } else { + if p.nodeDestination { + if node, ok := ret.(*Node); ok { + p.setComment1(&node.Cm.After, ciAfter) + } + } + return + } } - // test if we are dealing with a single JSON value instead (true/false/null/num/"") - p.resetAt() - if res2, err2 := p.checkTrailing(p.readValue(dest, t)); err2 == nil { - return res2, nil + if ret == nil { + // test if we are dealing with a single JSON value instead (true/false/null/num/"") + p.resetAt() + ret, err = p.readValue(dest, t) + if err == nil { + ciAfter, err = p.checkTrailing() + } + if err == nil { + if p.nodeDestination { + if node, ok := ret.(*Node); ok { + // ciBefore has been read again and set on the node inside the + // function p.readValue(). + existingAfter := node.Cm.After + p.setComment1(&node.Cm.After, ciAfter) + if node.Cm.After != "" { + existingAfter += "\n" + } + node.Cm.After = existingAfter + node.Cm.After + } + } + + return + } } - return res, err -} -func (p *hjsonParser) checkTrailing(v interface{}, err error) (interface{}, error) { - if err != nil { - return nil, err + if errSyntax != nil { + return nil, errSyntax } - p.white() + + return +} + +func (p *hjsonParser) checkTrailing() (commentInfo, error) { + ci := p.white() if p.ch > 0 { - return nil, p.errAt("Syntax error, found trailing characters") + return ci, p.errAt("Syntax error, found trailing characters") } - return v, nil + return ci, nil } // Unmarshal parses the Hjson-encoded data using default options and stores the @@ -662,6 +917,7 @@ func orderedUnmarshal( v interface{}, options DecoderOptions, willMarshalToJSON bool, + nodeDestination bool, ) ( interface{}, error, @@ -678,6 +934,7 @@ func orderedUnmarshal( ch: ' ', structTypeCache: map[reflect.Type]structFieldMap{}, willMarshalToJSON: willMarshalToJSON, + nodeDestination: nodeDestination, } parser.resetAt() value, err := parser.rootValue(rv) @@ -691,9 +948,17 @@ func orderedUnmarshal( // UnmarshalWithOptions parses the Hjson-encoded data and stores the result // in the value pointed to by v. // -// Unless v is of type *hjson.OrderedMap, the Hjson input is internally -// converted to JSON, which is then used as input to the function -// json.Unmarshal(). +// The Hjson input is internally converted to JSON, which is then used as input +// to the function json.Unmarshal(). Unless the input argument v is of any of +// these types: +// +// *hjson.OrderedMap +// **hjson.OrderedMap +// *hjson.Node +// **hjson.Node +// +// Comments can be read from the Hjson-encoded data, but only if the input +// argument v is of type *hjson.Node or **hjson.Node. // // For more details about the output from this function, see the documentation // for json.Unmarshal(). @@ -708,7 +973,18 @@ func UnmarshalWithOptions(data []byte, v interface{}, options DecoderOptions) er } } - value, err := orderedUnmarshal(data, v, options, !destinationIsOrderedMap) + inNode, destinationIsNode := v.(*Node) + if !destinationIsNode { + pInNode, ok := v.(**Node) + if ok { + destinationIsNode = true + inNode = &Node{} + *pInNode = inNode + } + } + + value, err := orderedUnmarshal(data, v, options, !(destinationIsOrderedMap || + destinationIsNode), destinationIsNode) if err != nil { return err } @@ -722,6 +998,13 @@ func UnmarshalWithOptions(data []byte, v interface{}, options DecoderOptions) er reflect.TypeOf(v)) } + if destinationIsNode { + if outNode, ok := value.(*Node); ok { + *inNode = *outNode + return nil + } + } + // Convert to JSON so we can let json.Unmarshal() handle all destination // types (including interfaces json.Unmarshaler and encoding.TextUnmarshaler) // and merging. diff --git a/encode.go b/encode.go index aec6c72..cfbd81f 100644 --- a/encode.go +++ b/encode.go @@ -31,6 +31,9 @@ type EncoderOptions struct { IndentBy string // Base indentation string BaseIndentation string + // Write comments, if any are found in hjson.Node structs or as tags on + // other structs. + Comments bool } // DefaultOptions returns the default encoding options. @@ -41,6 +44,7 @@ type EncoderOptions struct { // QuoteAmbiguousStrings = true // IndentBy = " " // BaseIndentation = "" +// Comments = true func DefaultOptions() EncoderOptions { return EncoderOptions{ Eol: "\n", @@ -50,6 +54,7 @@ func DefaultOptions() EncoderOptions { QuoteAmbiguousStrings: true, IndentBy: " ", BaseIndentation: "", + Comments: true, } } @@ -104,7 +109,22 @@ func (e *hjsonEncoder) quoteReplace(text string) string { })) } -func (e *hjsonEncoder) quote(value string, separator string, isRootObject bool) { +func (e *hjsonEncoder) quoteForComment(cmStr string) bool { + chars := []rune(cmStr) + for _, r := range chars { + switch r { + case '\r', '\n': + return false + case '/', '#': + return true + } + } + + return false +} + +func (e *hjsonEncoder) quote(value string, separator string, isRootObject bool, + keyComment string, hasCommentAfter bool) { // Check if we can insert this string without quotes // see hjson syntax (must not parse as true, false, null or number) @@ -112,8 +132,10 @@ func (e *hjsonEncoder) quote(value string, separator string, isRootObject bool) if len(value) == 0 { e.WriteString(separator + `""`) } else if e.QuoteAlways || - needsQuotes.MatchString(value) || (e.QuoteAmbiguousStrings && (startsWithNumber([]byte(value)) || - startsWithKeyword.MatchString(value))) { + hasCommentAfter || + needsQuotes.MatchString(value) || + (e.QuoteAmbiguousStrings && (startsWithNumber([]byte(value)) || + startsWithKeyword.MatchString(value))) { // If the string contains no control characters, no quote characters, and no // backslash characters, then we can safely slap some quotes around it. @@ -124,7 +146,7 @@ func (e *hjsonEncoder) quote(value string, separator string, isRootObject bool) if !needsEscape.MatchString(value) { e.WriteString(separator + `"` + value + `"`) } else if !needsEscapeML.MatchString(value) && !isRootObject { - e.mlString(value, separator) + e.mlString(value, separator, keyComment) } else { e.WriteString(separator + `"` + e.quoteReplace(value) + `"`) } @@ -134,7 +156,7 @@ func (e *hjsonEncoder) quote(value string, separator string, isRootObject bool) } } -func (e *hjsonEncoder) mlString(value string, separator string) { +func (e *hjsonEncoder) mlString(value string, separator string, keyComment string) { a := strings.Split(value, "\n") if len(a) == 1 { @@ -144,7 +166,9 @@ func (e *hjsonEncoder) mlString(value string, separator string) { e.WriteString(separator + "'''") e.WriteString(a[0]) } else { - e.writeIndent(e.indent + 1) + if !strings.Contains(keyComment, "\n") { + e.writeIndent(e.indent + 1) + } e.WriteString("'''") for _, v := range a { indent := e.indent + 1 @@ -176,6 +200,18 @@ func (e *hjsonEncoder) quoteName(name string) string { return name } +func (e *hjsonEncoder) bracesIndent(isObjElement, isEmpty bool, cm Comments, + separator string) { + + if !isObjElement || cm.Key == "" { + if isObjElement && !e.BracesSameLine && (!isEmpty || cm.InsideFirst != "") { + e.writeIndent(e.indent) + } else { + e.WriteString(separator) + } + } +} + type sortAlpha []reflect.Value func (s sortAlpha) Len() int { @@ -188,19 +224,24 @@ func (s sortAlpha) Less(i, j int) bool { return fmt.Sprintf("%v", s[i]) < fmt.Sprintf("%v", s[j]) } -func (e *hjsonEncoder) writeIndent(indent int) { - e.WriteString(e.Eol) +func (e *hjsonEncoder) writeIndentNoEOL(indent int) { e.WriteString(e.BaseIndentation) for i := 0; i < indent; i++ { e.WriteString(e.IndentBy) } } +func (e *hjsonEncoder) writeIndent(indent int) { + e.WriteString(e.Eol) + e.writeIndentNoEOL(indent) +} + func (e *hjsonEncoder) useMarshalerJSON( value reflect.Value, noIndent bool, separator string, - isRootObject bool, + isRootObject, + isObjElement bool, ) error { b, err := value.Interface().(json.Marshaler).MarshalJSON() if err != nil { @@ -210,22 +251,57 @@ func (e *hjsonEncoder) useMarshalerJSON( decOpt := DefaultDecoderOptions() decOpt.UseJSONNumber = true var dummyDest interface{} - jsonRoot, err := orderedUnmarshal(b, &dummyDest, decOpt, false) + jsonRoot, err := orderedUnmarshal(b, &dummyDest, decOpt, false, false) if err != nil { return err } // Output Hjson with our current options, instead of JSON. - return e.str(reflect.ValueOf(jsonRoot), noIndent, separator, isRootObject) + return e.str(reflect.ValueOf(jsonRoot), noIndent, separator, isRootObject, + isObjElement, Comments{}) } var marshalerJSON = reflect.TypeOf((*json.Marshaler)(nil)).Elem() var marshalerText = reflect.TypeOf((*encoding.TextMarshaler)(nil)).Elem() -func (e *hjsonEncoder) str(value reflect.Value, noIndent bool, separator string, isRootObject bool) error { +func (e *hjsonEncoder) unpackNode(value reflect.Value, cm Comments) (reflect.Value, Comments) { + if value.IsValid() { + if node, ok := value.Interface().(Node); ok { + value = reflect.ValueOf(node.Value) + if e.Comments { + cm = node.Cm + } + } else if pNode, ok := value.Interface().(*Node); ok { + value = reflect.ValueOf(pNode.Value) + if e.Comments { + cm = pNode.Cm + } + } + } + + return value, cm +} + +// This function can often be called from within itself, so do not output +// anything from the upper half of it. +func (e *hjsonEncoder) str( + value reflect.Value, + noIndent bool, + separator string, + isRootObject, + isObjElement bool, + cm Comments, +) error { // Produce a string from value. + // Unpack *Node, possibly overwrite cm. + value, cm = e.unpackNode(value, cm) + + if cm.Key != "" { + separator = "" + } + kind := value.Kind() switch kind { @@ -256,7 +332,7 @@ func (e *hjsonEncoder) str(value reflect.Value, noIndent bool, separator string, e.WriteString("null") return nil } - return e.str(value.Elem(), noIndent, separator, isRootObject) + return e.str(value.Elem(), noIndent, separator, isRootObject, isObjElement, cm) } // Our internal orderedMap implements marshalerJSON. We must therefore place @@ -270,11 +346,11 @@ func (e *hjsonEncoder) str(value reflect.Value, noIndent bool, separator string, name: key, }) } - return e.writeFields(fis, noIndent, separator, isRootObject) + return e.writeFields(fis, noIndent, separator, isRootObject, isObjElement, cm) } if value.Type().Implements(marshalerJSON) { - return e.useMarshalerJSON(value, noIndent, separator, isRootObject) + return e.useMarshalerJSON(value, noIndent, separator, isRootObject, isObjElement) } if value.Type().Implements(marshalerText) { @@ -283,7 +359,8 @@ func (e *hjsonEncoder) str(value reflect.Value, noIndent bool, separator string, return err } - return e.str(reflect.ValueOf(string(b)), noIndent, separator, isRootObject) + return e.str(reflect.ValueOf(string(b)), noIndent, separator, isRootObject, + isObjElement, cm) } switch kind { @@ -296,7 +373,8 @@ func (e *hjsonEncoder) str(value reflect.Value, noIndent bool, separator string, // without quotes e.WriteString(separator + n) } else { - e.quote(value.String(), separator, isRootObject) + e.quote(value.String(), separator, isRootObject, cm.Key, + e.quoteForComment(cm.After)) } case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: @@ -335,33 +413,45 @@ func (e *hjsonEncoder) str(value reflect.Value, noIndent bool, separator string, } case reflect.Slice, reflect.Array: - - len := value.Len() - if len == 0 { - e.WriteString(separator) - e.WriteString("[]") - break + e.bracesIndent(isObjElement, value.Len() == 0, cm, separator) + e.WriteString("[" + cm.InsideFirst) + + if value.Len() == 0 { + if cm.InsideFirst != "" || cm.InsideLast != "" { + e.WriteString(e.Eol) + if cm.InsideLast == "" { + e.writeIndentNoEOL(e.indent) + } + } + e.WriteString(cm.InsideLast + "]") + return nil } indent1 := e.indent e.indent++ - if !noIndent && !e.BracesSameLine { - e.writeIndent(indent1) - } else { - e.WriteString(separator) - } - e.WriteString("[") - // Join all of the element texts together, separated with newlines - for i := 0; i < len; i++ { - e.writeIndent(e.indent) - if err := e.str(value.Index(i), true, "", false); err != nil { + for i := 0; i < value.Len(); i++ { + elem, elemCm := e.unpackNode(value.Index(i), Comments{}) + + if elemCm.Before == "" && elemCm.Key == "" { + e.writeIndent(e.indent) + } else { + e.WriteString(e.Eol + elemCm.Before + elemCm.Key) + } + + if err := e.str(elem, true, "", false, false, elemCm); err != nil { return err } + + e.WriteString(elemCm.After) } - e.writeIndent(indent1) + if cm.InsideLast != "" { + e.WriteString(e.Eol + cm.InsideLast) + } else { + e.writeIndent(indent1) + } e.WriteString("]") e.indent = indent1 @@ -387,7 +477,7 @@ func (e *hjsonEncoder) str(value reflect.Value, noIndent bool, separator string, name: name, }) } - return e.writeFields(fis, noIndent, separator, isRootObject) + return e.writeFields(fis, noIndent, separator, isRootObject, isObjElement, cm) case reflect.Struct: // Struct field info is identical for all instances of the same type. @@ -420,13 +510,16 @@ func (e *hjsonEncoder) str(value reflect.Value, noIndent bool, separator string, continue } - fis = append(fis, fieldInfo{ - field: fv, - name: sfi.name, - comment: sfi.comment, - }) + fi := fieldInfo{ + field: fv, + name: sfi.name, + } + if e.Comments { + fi.comment = sfi.comment + } + fis = append(fis, fi) } - return e.writeFields(fis, noIndent, separator, isRootObject) + return e.writeFields(fis, noIndent, separator, isRootObject, isObjElement, cm) default: return errors.New("Unsupported type " + value.Type().String()) @@ -454,6 +547,28 @@ func isEmptyValue(v reflect.Value) bool { } } +func investigateComment(txt string) ( + endsInsideComment, + endsWithLineFeed bool, +) { + var prev rune + for _, rn := range txt { + switch rn { + case '\n': + endsInsideComment = false + case '#': + endsInsideComment = true + case '/': + if prev == '/' { + endsInsideComment = true + } + } + endsWithLineFeed = (rn == '\n') + prev = rn + } + return +} + // Marshal returns the Hjson encoding of v using // default options. // @@ -582,9 +697,16 @@ func MarshalWithOptions(v interface{}, options EncoderOptions) ([]byte, error) { structTypeCache: map[reflect.Type][]structFieldInfo{}, } - err := e.str(reflect.ValueOf(v), true, e.BaseIndentation, true) + value := reflect.ValueOf(v) + _, cm := e.unpackNode(value, Comments{}) + e.WriteString(cm.Before + cm.Key) + + err := e.str(value, true, e.BaseIndentation, true, false, cm) if err != nil { return nil, err } + + e.WriteString(cm.After) + return e.Bytes(), nil } diff --git a/encode_test.go b/encode_test.go index 8b23b4a..9e1480f 100644 --- a/encode_test.go +++ b/encode_test.go @@ -852,6 +852,22 @@ func TestStructComment(t *testing.T) { # Look ma, new lines B: 3 + C: some text + D: 5 +}` + if string(h) != expected { + t.Errorf("Expected:\n%s\nGot:\n%s\n\n", expected, string(h)) + } + + opt := DefaultOptions() + opt.Comments = false + h, err = MarshalWithOptions(a, opt) + if err != nil { + t.Error(err) + } + expected = `{ + x: hi! + B: 3 C: some text D: 5 }` diff --git a/hjson-cli/main.go b/hjson-cli/main.go index 9176fc8..7f8bb6a 100644 --- a/hjson-cli/main.go +++ b/hjson-cli/main.go @@ -46,6 +46,7 @@ func main() { var omitRootBraces = flag.Bool("omitRootBraces", false, "Omit braces at the root.") var quoteAlways = flag.Bool("quoteAlways", false, "Always quote string values.") var showVersion = flag.Bool("v", false, "Show version.") + var preserveKeyOrder = flag.Bool("preserveKeyOrder", false, "Preserve key order in objects/maps.") flag.Parse() if *help || flag.NArg() > 1 { @@ -77,7 +78,14 @@ func main() { var value interface{} - if err := hjson.Unmarshal(data, &value); err != nil { + if *preserveKeyOrder { + var node *hjson.Node + err = hjson.Unmarshal(data, &node) + value = node + } else { + err = hjson.Unmarshal(data, &value) + } + if err != nil { panic(err) } @@ -100,6 +108,7 @@ func main() { opt.BracesSameLine = *bracesSameLine opt.EmitRootBraces = !*omitRootBraces opt.QuoteAlways = *quoteAlways + opt.Comments = false out, err = hjson.MarshalWithOptions(value, opt) if err != nil { panic(err) diff --git a/node.go b/node.go new file mode 100644 index 0000000..909c275 --- /dev/null +++ b/node.go @@ -0,0 +1,415 @@ +package hjson + +import ( + "encoding/json" + "fmt" + "reflect" +) + +type Comments struct { + // Comment/whitespace on line(s) before the value, and before the value on + // the same line. If not empty, is expected to end with a line feed + + // indentation for the value. + Before string + // Comment/whitespace between the key and this value, if this value is an + // element in a map/object. + Key string + // Comment/whitespace after (but still on the same line as) the leading + // bracket ({ or [) for this value, if this value is a slice/array or + // map/object. Is not expected to contain any line feed. + InsideFirst string + // Comment/whitespace from the beginning of the first line after all child + // values belonging to this value, until the closing bracket (} or ]), if + // this value is a slice/array or map/object. If not empty, is expected to + // end with a line feed + indentation for the closing bracket. + InsideLast string + // Comment/whitespace after (but still on the same line as) the value. Is not + // expected to contain any line feed. hjson.Unmarshal() will try to assign + // comments/whitespace from lines between values to `Before` on the value + // after those lines, or to `InsideLast` on the slice/map if the lines + // containing comments appear after the last element inside a slice/map. + After string +} + +// Node must be used as destination for Unmarshal() or UnmarshalWithOptions() +// whenever comments should be read from the input. The struct is simply a +// wrapper for the actual values and a helper struct containing any comments. +// The Value in the destination Node will be overwritten in the call to +// Unmarshal() or UnmarshalWithOptions(), i.e. node trees are not merged. +// After the unmarshal, Node.Value will contain any of these types: +// +// nil (no type) +// float64 (if UseJSONNumber == false) +// json.Number (if UseJSONNumber == true) +// string +// bool +// []interface{} +// *hjson.OrderedMap +// +// All elements in an []interface{} or *hjson.OrderedMap will be of the type +// *hjson.Node, so that they can contain comments. +// +// This example shows unmarshalling input with comments, changing the value on +// a single key (the input is assumed to have an object/map as root) and then +// marshalling the node tree again, including comments and with preserved key +// order in the object/map. +// +// var node hjson.Node +// err := hjson.Unmarshal(input, &node) +// if err != nil { +// return err +// } +// _, err = node.SetKey("setting1", 3) +// if err != nil { +// return err +// } +// output, err := hjson.Marshal(node) +// if err != nil { +// return err +// } +type Node struct { + Value interface{} + Cm Comments +} + +// Len returns the length of the value wrapped by this Node, if the value is of +// type *hjson.OrderedMap, []interface{} or string. Otherwise 0 is returned. +func (c *Node) Len() int { + if c == nil { + return 0 + } + switch cont := c.Value.(type) { + case *OrderedMap: + return cont.Len() + case []interface{}: + return len(cont) + case string: + return len(cont) + } + return 0 +} + +// AtIndex returns the key (if any) and value (unwrapped from its Node) found +// at the specified index, if this Node contains a value of type +// *hjson.OrderedMap or []interface{}. Returns an error for unexpected types. +// Panics if index < 0 or index >= Len(). +func (c *Node) AtIndex(index int) (string, interface{}, error) { + if c == nil { + return "", nil, fmt.Errorf("Node is nil") + } + var key string + var elem interface{} + switch cont := c.Value.(type) { + case *OrderedMap: + key = cont.Keys[index] + elem = cont.Map[key] + case []interface{}: + elem = cont[index] + default: + return "", nil, fmt.Errorf("Unexpected value type: %v", reflect.TypeOf(c.Value)) + } + node, ok := elem.(*Node) + if !ok { + return "", nil, fmt.Errorf("Unexpected element type: %v", reflect.TypeOf(elem)) + } + return key, node.Value, nil +} + +// AtKey returns the value (unwrapped from its Node) found for the specified +// key, if this Node contains a value of type *hjson.OrderedMap. An error is +// returned for unexpected types. The second returned value is true if the key +// was found, false otherwise. +func (c *Node) AtKey(key string) (interface{}, bool, error) { + if c == nil { + return nil, false, nil + } + om, ok := c.Value.(*OrderedMap) + if !ok { + return nil, false, fmt.Errorf("Unexpected value type: %v", reflect.TypeOf(c.Value)) + } + elem, ok := om.Map[key] + if !ok { + return nil, false, nil + } + node, ok := elem.(*Node) + if !ok { + return nil, false, fmt.Errorf("Unexpected element type: %v", reflect.TypeOf(elem)) + } + return node.Value, true, nil +} + +// Append adds the input value to the end of the []interface{} wrapped by this +// Node. If this Node contains nil without a type, an empty []interface{} is +// first created. If this Node contains a value of any other type, an error is +// returned. +func (c *Node) Append(value interface{}) error { + if c == nil { + return fmt.Errorf("Node is nil") + } + var arr []interface{} + if c.Value == nil { + arr = []interface{}{} + } else { + var ok bool + arr, ok = c.Value.([]interface{}) + if !ok { + return fmt.Errorf("Unexpected value type: %v", reflect.TypeOf(c.Value)) + } + } + c.Value = append(arr, &Node{Value: value}) + return nil +} + +// Insert inserts a new key/value pair at the specified index, if this Node +// contains a value of type *hjson.OrderedMap or []interface{}. Returns an error +// for unexpected types. Panics if index < 0 or index > c.Len(). If the key +// already exists in the OrderedMap, the new value is set but the position of +// the key is not changed. Otherwise the value to insert is wrapped in a new +// Node. If this Node contains []interface{}, the key is ignored. Returns the +// old value and true if the key already exists in the/ OrderedMap, nil and +// false otherwise. +func (c *Node) Insert(index int, key string, value interface{}) (interface{}, bool, error) { + if c == nil { + return nil, false, fmt.Errorf("Node is nil") + } + var oldVal interface{} + var found bool + switch cont := c.Value.(type) { + case *OrderedMap: + oldVal, found := cont.Map[key] + if found { + if node, ok := oldVal.(*Node); ok { + oldVal = node.Value + node.Value = value + } else { + cont.Map[key] = &Node{Value: value} + } + } else { + oldVal, found = cont.Insert(index, key, &Node{Value: value}) + } + case []interface{}: + value = &Node{Value: value} + if index == len(cont) { + c.Value = append(cont, value) + } else { + cont = append(cont[:index+1], cont[index:]...) + cont[index] = value + c.Value = cont + } + default: + return nil, false, fmt.Errorf("Unexpected value type: %v", reflect.TypeOf(c.Value)) + } + return oldVal, found, nil +} + +// SetIndex assigns the specified value to the child Node found at the specified +// index, if this Node contains a value of type *hjson.OrderedMap or +// []interface{}. Returns an error for unexpected types. Returns the key (if +// any) and value previously found at the specified index. Panics if index < 0 +// or index >= Len(). +func (c *Node) SetIndex(index int, value interface{}) (string, interface{}, error) { + if c == nil { + return "", nil, fmt.Errorf("Node is nil") + } + var key string + var elem interface{} + switch cont := c.Value.(type) { + case *OrderedMap: + key = cont.Keys[index] + elem = cont.Map[key] + case []interface{}: + elem = cont[index] + default: + return "", nil, fmt.Errorf("Unexpected value type: %v", reflect.TypeOf(c.Value)) + } + var oldVal interface{} + node, ok := elem.(*Node) + if ok { + oldVal = node.Value + node.Value = value + } else { + oldVal = elem + switch cont := c.Value.(type) { + case *OrderedMap: + cont.Map[key] = &Node{Value: value} + case []interface{}: + cont[index] = &Node{Value: value} + } + } + return key, oldVal, nil +} + +// SetKey assigns the specified value to the child Node identified by the +// specified key, if this Node contains a value of the type *hjson.OrderedMap. +// If this Node contains nil without a type, an empty *hjson.OrderedMap is +// first created. If this Node contains a value of any other type an error is +// returned. If the key cannot be found in the OrderedMap, a new Node is +// created, wrapping the specified value, and appended to the end of the +// OrderedMap. Returns the old value and true if the key already existed in +// the OrderedMap, nil and false otherwise. +func (c *Node) SetKey(key string, value interface{}) (interface{}, bool, error) { + if c == nil { + return nil, false, fmt.Errorf("Node is nil") + } + var om *OrderedMap + if c.Value == nil { + om = NewOrderedMap() + c.Value = om + } else { + var ok bool + om, ok = c.Value.(*OrderedMap) + if !ok { + return nil, false, fmt.Errorf("Unexpected value type: %v", reflect.TypeOf(c.Value)) + } + } + var oldVal interface{} + elem, ok := om.Map[key] + if ok { + var node *Node + node, ok = elem.(*Node) + if ok { + oldVal = node.Value + node.Value = value + } + } + foundKey := true + if !ok { + oldVal, foundKey = om.Set(key, &Node{Value: value}) + } + return oldVal, foundKey, nil +} + +// DeleteIndex deletes the value or key/value pair found at the specified index, +// if this Node contains a value of type *hjson.OrderedMap or []interface{}. +// Returns an error for unexpected types. Panics if index < 0 or +// index >= c.Len(). Returns the deleted key (if any) and value. +func (c *Node) DeleteIndex(index int) (string, interface{}, error) { + if c == nil { + return "", nil, fmt.Errorf("Node is nil") + } + var key string + var value interface{} + switch cont := c.Value.(type) { + case *OrderedMap: + key, value = cont.DeleteIndex(index) + case []interface{}: + value = cont[index] + cont = append(cont[:index], cont[index+1:]...) + c.Value = cont + default: + return "", nil, fmt.Errorf("Unexpected value type: %v", reflect.TypeOf(c.Value)) + } + if node, ok := value.(*Node); ok { + value = node.Value + } + return key, value, nil +} + +// DeleteKey deletes the key/value pair with the specified key, if found and if +// this Node contains a value of type *hjson.OrderedMap. Returns an error for +// unexpected types. Returns the deleted value and true if the key was found, +// nil and false otherwise. +func (c *Node) DeleteKey(key string) (interface{}, bool, error) { + if c == nil { + return nil, false, fmt.Errorf("Node is nil") + } + if om, ok := c.Value.(*OrderedMap); ok { + oldValue, found := om.DeleteKey(key) + if node, ok := oldValue.(*Node); ok { + oldValue = node.Value + } + return oldValue, found, nil + } + return nil, false, fmt.Errorf("Unexpected value type: %v", reflect.TypeOf(c.Value)) +} + +// NI is an acronym formed from "get Node pointer by Index". Returns the *Node +// element found at the specified index, if this Node contains a value of type +// *hjson.OrderedMap or []interface{}. Returns nil otherwise. Panics if +// index < 0 or index >= Len(). Does not create or alter any value. +func (c *Node) NI(index int) *Node { + if c == nil { + return nil + } + var elem interface{} + switch cont := c.Value.(type) { + case *OrderedMap: + elem = cont.AtIndex(index) + case []interface{}: + elem = cont[index] + default: + return nil + } + if node, ok := elem.(*Node); ok { + return node + } + return nil +} + +// NK is an acronym formed from "get Node pointer by Key". Returns the *Node +// element found for the specified key, if this Node contains a value of type +// *hjson.OrderedMap. Returns nil otherwise. Does not create or alter anything. +func (c *Node) NK(key string) *Node { + if c == nil { + return nil + } + om, ok := c.Value.(*OrderedMap) + if !ok { + return nil + } + if elem, ok := om.Map[key]; ok { + if node, ok := elem.(*Node); ok { + return node + } + } + return nil +} + +// NKC is an acronym formed from "get Node pointer by Key, Create if not found". +// Returns the *Node element found for the specified key, if this Node contains +// a value of type *hjson.OrderedMap. If this Node contains nil without a type, +// an empty *hjson.OrderedMap is first created. If this Node contains a value of +// any other type or if the element idendified by the specified key is not of +// type *Node, an error is returned. If the key cannot be found in the +// OrderedMap, a new Node is created for the specified key. Example usage: +// +// var node hjson.Node +// node.NKC("rootKey1").NKC("subKey1").SetKey("valKey1", "my value") +func (c *Node) NKC(key string) *Node { + if c == nil { + return nil + } + var om *OrderedMap + if c.Value == nil { + om = NewOrderedMap() + c.Value = om + } else { + var ok bool + om, ok = c.Value.(*OrderedMap) + if !ok { + return nil + } + } + if elem, ok := om.Map[key]; ok { + if node, ok := elem.(*Node); ok { + return node + } + } else { + node := &Node{} + om.Set(key, node) + return node + } + return nil +} + +// MarshalJSON is an implementation of the json.Marshaler interface, enabling +// hjson.Node trees to be used as input for json.Marshal(). +func (c Node) MarshalJSON() ([]byte, error) { + return json.Marshal(c.Value) +} + +// UnmarshalJSON is an implementation of the json.Unmarshaler interface, +// enabling hjson.Node to be used as destination for json.Unmarshal(). +func (c *Node) UnmarshalJSON(b []byte) error { + return Unmarshal(b, c) +} diff --git a/node_test.go b/node_test.go new file mode 100644 index 0000000..83cd21d --- /dev/null +++ b/node_test.go @@ -0,0 +1,588 @@ +package hjson + +import ( + "encoding/json" + "testing" +) + +func compareStrings(t *testing.T, bOut []byte, txtExpected string) { + if string(bOut) != txtExpected { + t.Errorf("Expected:\n%s\n\nGot:\n%s\n\n", txtExpected, string(bOut)) + } +} + +func verifyNodeContent(t *testing.T, node *Node, txtExpected string) { + opt := DefaultOptions() + opt.EmitRootBraces = false + bOut, err := MarshalWithOptions(node, opt) + if err != nil { + t.Error(err) + } + + compareStrings(t, bOut, txtExpected) +} + +func TestNode1(t *testing.T) { + txt := `b: 1 +a: 2` + + var node *Node + err := Unmarshal([]byte(txt), &node) + if err != nil { + t.Error(err) + } + + verifyNodeContent(t, node, txt) +} + +func TestNode2(t *testing.T) { + txt := `# comment before +b: 1 # comment after +// Comment B4 +a: 2 +/* Last comment */` + + var node *Node + err := Unmarshal([]byte(txt), &node) + if err != nil { + t.Error(err) + } + + if node.Len() != 2 { + t.Errorf("Unexpected map length: %v", node.Len()) + } + + aVal, ok, err := node.AtKey("a") + if err != nil { + t.Error(err) + } + // The value will be a float64 even though it was written without decimals. + if aVal != 2.0 { + t.Errorf("Unexpected value for key 'a': %v", aVal) + } else if !ok { + t.Errorf("node.AtKey('a') returned false") + } + + bKey, bVal, err := node.AtIndex(0) + if err != nil { + t.Error(err) + } + if bKey != "b" { + t.Errorf("Expected key 'b', got: %v", bKey) + } + // The value will be a float64 even though it was written without decimals. + if bVal != 1.0 { + t.Errorf("Unexpected value for key 'b': %v", bVal) + } + + verifyNodeContent(t, node, txt) + + opt := DefaultOptions() + opt.Comments = false + bOut, err := MarshalWithOptions(node, opt) + if err != nil { + t.Error(err) + } + + compareStrings(t, bOut, `{ + b: 1 + a: 2 +}`) + + bOut, err = json.Marshal(node) + if err != nil { + t.Error(err) + } + + compareStrings(t, bOut, `{"b":1,"a":2}`) + + intLen := node.Value.(*OrderedMap).Map["b"].(*Node).Len() + if intLen != 0 { + t.Errorf("Unexpected int length: %v", intLen) + } + + node.SetIndex(0, 3) + + verifyNodeContent(t, node, `# comment before +b: 3 # comment after +// Comment B4 +a: 2 +/* Last comment */`) + + oldVal, found, err := node.SetKey("b", "abcdef") + if err != nil { + t.Error(err) + } + if !found { + t.Errorf("Should have returned true, the key should already exist.") + } + if oldVal != 3 { + t.Errorf("Expected old value 3, got: '%v'", oldVal) + } + + verifyNodeContent(t, node, `# comment before +b: "abcdef" # comment after +// Comment B4 +a: 2 +/* Last comment */`) + + strLen := node.Value.(*OrderedMap).Map["b"].(*Node).Len() + if strLen != 6 { + t.Errorf("Unexpected string length: %v", strLen) + } + + node.Value.(*OrderedMap).Map["b"] = "xyz" + + verifyNodeContent(t, node, `b: xyz +// Comment B4 +a: 2 +/* Last comment */`) +} + +func TestNode3(t *testing.T) { + txt := `# comment before +[ +# after [ + 1 # comment after + // Comment B4 + 2 + # COmment After +] +/* Last comment */` + + var node *Node + Unmarshal([]byte(txt), &node) + + if node.Len() != 2 { + t.Errorf("Unexpected slice length: %v", node.Len()) + } + + firstKey, firstVal, err := node.AtIndex(0) + if err != nil { + t.Error(err) + } + if firstKey != "" { + t.Errorf("Expected empty key, got: %v", firstKey) + } + // The value will be a float64 even though it was written without decimals. + if firstVal != 1.0 { + t.Errorf("Unexpected value for index 0: %v", firstVal) + } + + verifyNodeContent(t, node, txt) + + opt := DefaultOptions() + opt.Comments = false + bOut, err := MarshalWithOptions(node, opt) + if err != nil { + t.Error(err) + } + + compareStrings(t, bOut, `[ + 1 + 2 +]`) + + bOut, err = json.Marshal(node) + + compareStrings(t, bOut, `[1,2]`) + + intLen := node.Value.([]interface{})[1].(*Node).Len() + if intLen != 0 { + t.Errorf("Unexpected int length: %v", intLen) + } + + key, oldVal, err := node.SetIndex(1, "abcdef") + if err != nil { + t.Error(err) + } + if key != "" { + t.Errorf("Expected empty key, got: '%v'", key) + } + if oldVal != 2.0 { + t.Errorf("Expected old value 2.0, got: '%v'", oldVal) + } + + verifyNodeContent(t, node, `# comment before +[ +# after [ + 1 # comment after + // Comment B4 + abcdef + # COmment After +] +/* Last comment */`) + + strLen := node.Value.([]interface{})[1].(*Node).Len() + if strLen != 6 { + t.Errorf("Unexpected string length: %v", strLen) + } + + node.Value.([]interface{})[0] = "xyz" + + verifyNodeContent(t, node, `# comment before +[ + xyz + // Comment B4 + abcdef + # COmment After +] +/* Last comment */`) +} + +func TestNode4(t *testing.T) { + txt := `# comment before +b: /* key comment */ { + sub1: 1 # comment after +} # cm after obj +// Comment B4 +a: 2 +/* Last comment */` + + var node *Node + err := Unmarshal([]byte(txt), &node) + if err != nil { + t.Error(err) + } + + verifyNodeContent(t, node, txt) + + sub1Val, ok, err := node.NK("b").AtKey("sub1") + if err != nil { + t.Error(err) + } + // The value will be a float64 even though it was written without decimals. + if sub1Val != 1.0 { + t.Errorf("Unexpected value for sub1: %v", sub1Val) + } else if !ok { + t.Errorf("AtKey('sub1') returned false") + } + + sub1Val, ok, err = node.NK("Z").AtKey("sub2") + if err != nil { + t.Error(err) + } + if ok { + t.Errorf("Should have returned false when calling AtKey() on nil") + } + + oldVal, found, err := node.NK("Z").SetKey("sub2", 3) + if err == nil { + t.Errorf("Should have returned an error calling SetKey() on nil") + } + + oldVal, found, err = node.NKC("Z").SetKey("sub2", 3) + if err != nil { + t.Error(err) + } + if found { + t.Errorf("Should have returned false, the key should not already exist.") + } + + oldVal, found, err = node.NKC("Z").SetKey("sub2", 4) + if err != nil { + t.Error(err) + } + if !found { + t.Errorf("Should have returned true, the key should already exist.") + } + if oldVal != 3 { + t.Errorf("Expected old value 3, got: '%v'", oldVal) + } + + oldVal, found, err = node.NKC("X").NKC("Y").SetKey("sub3", 5) + if err != nil { + t.Error(err) + } + if found { + t.Errorf("Should have returned false, the key should not already exist.") + } + + verifyNodeContent(t, node, `# comment before +b: /* key comment */ { + sub1: 1 # comment after +} # cm after obj +// Comment B4 +a: 2 +Z: { + sub2: 4 +} +X: { + Y: { + sub3: 5 + } +} +/* Last comment */`) +} + +func TestDisallowDuplicateKeys(t *testing.T) { + txt := `a: 1 +a: 2 +b: 3 +c: 4 +b: 5` + + var node *Node + err := Unmarshal([]byte(txt), &node) + if err != nil { + t.Error(err) + } + + verifyNodeContent(t, node, `a: 2 +b: 5 +c: 4`) + + decOpt := DefaultDecoderOptions() + decOpt.DisallowDuplicateKeys = true + err = UnmarshalWithOptions([]byte(txt), &node, decOpt) + if err == nil { + t.Errorf("Should have returned error because of duplicate keys.") + } +} + +func TestWhitespaceAsComments(t *testing.T) { + txt := ` + +a: 2 + b: 3 + +` + + var node *Node + err := Unmarshal([]byte(txt), &node) + if err != nil { + t.Error(err) + } + + verifyNodeContent(t, node, txt) + + decOpt := DefaultDecoderOptions() + decOpt.WhitespaceAsComments = false + err = UnmarshalWithOptions([]byte(txt), &node, decOpt) + if err != nil { + t.Error(err) + } + + verifyNodeContent(t, node, `a: 2 +b: 3`) +} + +func TestDeclareNodeMap(t *testing.T) { + var node Node + + node2 := node.NK("a") + if node2 != nil { + t.Errorf("node.NK() created a node") + } + + oldVal, found, err := node.NKC("a").NKC("aa").NKC("aaa").SetKey("aaaa", "a string") + if err != nil { + t.Error(err) + } + if found { + t.Errorf("Should have returned false, the key should not already exist.") + } + oldVal, found, err = node.SetKey("b", 2) + if err != nil { + t.Error(err) + } + if found { + t.Errorf("Should have returned false, the key should not already exist.") + } + key, oldVal, err := node.SetIndex(1, 3.0) + if err != nil { + t.Error(err) + } + if key != "b" { + t.Errorf("Expected key 'b', got: '%v'", key) + } + if oldVal != 2 { + t.Errorf("Expected old value 2, got: '%v'", oldVal) + } + + verifyNodeContent(t, &node, `a: { + aa: { + aaa: { + aaaa: a string + } + } +} +b: 3`) + + err = node.Append(4) + if err == nil { + t.Errorf("Should have returned error when trying to append to a map") + } +} + +func TestDeclareNodeSlice(t *testing.T) { + var node Node + + node2 := node.NI(0) + if node2 != nil { + t.Errorf("node.NI() created a node") + } + + err := node.Append(13) + if err != nil { + t.Error(err) + } + err = node.Append("b") + if err != nil { + t.Error(err) + } + key, oldVal, err := node.SetIndex(1, false) + if err != nil { + t.Error(err) + } + if key != "" { + t.Errorf("Expected empty key, got: '%v'", key) + } + if oldVal != "b" { + t.Errorf("Expected old value 'b', got: '%v'", oldVal) + } + + verifyNodeContent(t, &node, `[ + 13 + false +]`) + + node2 = node.NKC("sub") + if node2 != nil { + t.Errorf("Should not have been able to create a node by key in a slice") + } + + _, _, err = node.SetKey("a", 4) + if err == nil { + t.Errorf("Should have returned error when trying to set by key on a slice") + } +} + +func TestNodeNoPointer(t *testing.T) { + txt := `setting1: null # nada +setting2: true // yes` + + var node Node + err := Unmarshal([]byte(txt), &node) + if err != nil { + t.Error(err) + } + oldVal, found, err := node.SetKey("setting1", 3) + if err != nil { + t.Error(err) + } + if !found { + t.Errorf("Should have returned true, the key should already exist") + } + if oldVal != nil { + t.Errorf("Expected old value nil, got: '%v'", oldVal) + } + output, err := Marshal(node) + if err != nil { + t.Error(err) + } + + compareStrings(t, output, `{ + setting1: 3 # nada + setting2: true // yes +}`) +} + +func TestNodeOrderedMapInsertDelete(t *testing.T) { + txt := `a: 1 +b: 2` + + var node Node + err := Unmarshal([]byte(txt), &node) + if err != nil { + t.Error(err) + } + + key, val, err := node.DeleteIndex(0) + if err != nil { + t.Error(err) + } + if key != "a" { + t.Errorf("Expected key 'a', got: '%v'", key) + } + if val != 1.0 { + t.Errorf("Expected old value 1.0, got: '%#v'", val) + } + + val, found, err := node.Insert(1, key, val) + if err != nil { + t.Error(err) + } + if found { + t.Errorf("Found key '%v' that should not have been found", key) + } + + val, found, err = node.DeleteKey("a") + if err != nil { + t.Error(err) + } + if !found { + t.Errorf("Expected to find key 'a' but did not") + } + if val != 1.0 { + t.Errorf("Expected deleted value 1.0, got: '%#v'", val) + } + + val, found, err = node.Insert(0, "c", 3) + if err != nil { + t.Error(err) + } + if found { + t.Error("Found key c that should not have been found") + } + + verifyNodeContent(t, &node, `c: 3 +b: 2`) +} + +func TestNodeSliceInsertDelete(t *testing.T) { + txt := `[ + 1 + 2 +]` + + var node Node + err := Unmarshal([]byte(txt), &node) + if err != nil { + t.Error(err) + } + + key, val, err := node.DeleteIndex(0) + if err != nil { + t.Error(err) + } + if key != "" { + t.Errorf("Expected empty key, got: '%v'", key) + } + if val != 1.0 { + t.Errorf("Expected old value 1.0, got: '%#v'", val) + } + + val, found, err := node.Insert(1, key, val) + if err != nil { + t.Error(err) + } + if found { + t.Errorf("Found key '%v' that should not have been found", key) + } + + val, found, err = node.Insert(0, "c", 3) + if err != nil { + t.Error(err) + } + if found { + t.Error("Found key c that should not have been found") + } + + // The value '2' has a line break as Cm.After + verifyNodeContent(t, &node, `[ + 3 + 2 + 1 +]`) +} diff --git a/orderedmap.go b/orderedmap.go index 7e02e36..95888b6 100644 --- a/orderedmap.go +++ b/orderedmap.go @@ -27,6 +27,7 @@ type OrderedMap struct { Map map[string]interface{} } +// KeyValue is only used as input to NewOrderedMapFromSlice(). type KeyValue struct { Key string Value interface{} diff --git a/structs.go b/structs.go index a98e575..981858a 100644 --- a/structs.go +++ b/structs.go @@ -255,50 +255,88 @@ func (e *hjsonEncoder) writeFields( noIndent bool, separator string, isRootObject bool, + isObjElement bool, + cm Comments, ) error { - if len(fis) == 0 { - e.WriteString(separator) - e.WriteString("{}") - return nil - } - indent1 := e.indent - if !isRootObject || e.EmitRootBraces { - if !noIndent && !e.BracesSameLine { - e.writeIndent(e.indent) - } else { - e.WriteString(separator) + if !isRootObject || e.EmitRootBraces || len(fis) == 0 { + e.bracesIndent(isObjElement, len(fis) == 0, cm, separator) + e.WriteString("{" + cm.InsideFirst) + + if len(fis) == 0 { + if cm.InsideFirst != "" || cm.InsideLast != "" { + e.WriteString(e.Eol) + } + e.WriteString(cm.InsideLast) + if cm.InsideLast != "" { + endsInsideComment, endsWithLineFeed := investigateComment(cm.InsideLast) + if endsInsideComment { + e.writeIndent(e.indent) + } + if endsWithLineFeed { + e.writeIndentNoEOL(e.indent) + } + } else if cm.InsideFirst != "" { + e.writeIndentNoEOL(e.indent) + } + e.WriteString("}") + return nil } e.indent++ - e.WriteString("{") + } else { + e.WriteString(cm.InsideFirst) } // Join all of the member texts together, separated with newlines + var elemCm Comments for i, fi := range fis { + var elem reflect.Value + elem, elemCm = e.unpackNode(fi.field, elemCm) + if i > 0 || !isRootObject || e.EmitRootBraces { + e.WriteString(e.Eol) + } if len(fi.comment) > 0 { for _, line := range strings.Split(fi.comment, e.Eol) { - if i > 0 || !isRootObject || e.EmitRootBraces { - e.writeIndent(e.indent) - } - e.WriteString(fmt.Sprintf("# %s", line)) + e.writeIndentNoEOL(e.indent) + e.WriteString(fmt.Sprintf("# %s\n", line)) } } - if i > 0 || !isRootObject || e.EmitRootBraces { - e.writeIndent(e.indent) + if elemCm.Before == "" { + e.writeIndentNoEOL(e.indent) + } else { + e.WriteString(elemCm.Before) } e.WriteString(e.quoteName(fi.name)) e.WriteString(":") - if err := e.str(fi.field, false, " ", false); err != nil { + e.WriteString(elemCm.Key) + + if err := e.str(elem, false, " ", false, true, elemCm); err != nil { return err } + if len(fi.comment) > 0 && i < len(fis)-1 { e.WriteString(e.Eol) } + + e.WriteString(elemCm.After) + } + + if cm.InsideLast != "" { + e.WriteString(e.Eol + cm.InsideLast) } if !isRootObject || e.EmitRootBraces { - e.writeIndent(indent1) + if cm.InsideLast == "" { + e.writeIndent(indent1) + } else { + endsInsideComment, endsWithLineFeed := investigateComment(cm.InsideLast) + if endsInsideComment { + e.writeIndent(indent1) + } else if endsWithLineFeed { + e.writeIndentNoEOL(indent1) + } + } e.WriteString("}") }