From d5d40e7fe4dcd0365ed9c1b230319d5be83f88cf Mon Sep 17 00:00:00 2001 From: Christian Zangl Date: Sun, 16 Jul 2017 00:27:44 +0200 Subject: [PATCH] add support for single quoted strings see #5 --- LICENSE | 2 +- assets/charset_test.hjson | 14 ++++++----- assets/keys_result.hjson | 3 +++ assets/keys_result.json | 3 +++ assets/keys_test.hjson | 3 +++ assets/sorted/keys_result.hjson | 3 +++ assets/sorted/keys_result.json | 3 +++ assets/sorted/strings2_result.hjson | 29 +++++++++++++++++++++++ assets/sorted/strings2_result.json | 28 ++++++++++++++++++++++ assets/strings2_result.hjson | 29 +++++++++++++++++++++++ assets/strings2_result.json | 28 ++++++++++++++++++++++ assets/strings2_test.hjson | 36 +++++++++++++++++++++++++++++ assets/testlist.txt | 4 ++-- assets/trail_test.hjson | 6 +++-- decode.go | 33 +++++++++++++------------- encode.go | 4 ++-- 16 files changed, 198 insertions(+), 30 deletions(-) create mode 100644 assets/sorted/strings2_result.hjson create mode 100644 assets/sorted/strings2_result.json create mode 100644 assets/strings2_result.hjson create mode 100644 assets/strings2_result.json create mode 100644 assets/strings2_test.hjson diff --git a/LICENSE b/LICENSE index 2b3d0b1..3049b05 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2016 Christian Zangl +Copyright (c) 2016, 2017 Christian Zangl Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/assets/charset_test.hjson b/assets/charset_test.hjson index 7527b1e..9533e4b 100644 --- a/assets/charset_test.hjson +++ b/assets/charset_test.hjson @@ -1,6 +1,8 @@ -ql-ascii: ! "#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~ -js-ascii: "! \"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~" -ml-ascii: - ''' - ! "#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~ - ''' +{ + ql-ascii: ! "#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~ + js-ascii: "! \"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~" + ml-ascii: + ''' + ! "#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~ + ''' +} diff --git a/assets/keys_result.hjson b/assets/keys_result.hjson index 876e6c3..06833c4 100644 --- a/assets/keys_result.hjson +++ b/assets/keys_result.hjson @@ -18,6 +18,9 @@ "foo\"bar": test "'''": test "foo'''bar": test + "'": test + "'foo": test + "foo'bar": test ":": test "foo:bar": test "{": test diff --git a/assets/keys_result.json b/assets/keys_result.json index 81fa480..ead27f1 100644 --- a/assets/keys_result.json +++ b/assets/keys_result.json @@ -18,6 +18,9 @@ "foo\"bar": "test", "'''": "test", "foo'''bar": "test", + "'": "test", + "'foo": "test", + "foo'bar": "test", ":": "test", "foo:bar": "test", "{": "test", diff --git a/assets/keys_test.hjson b/assets/keys_test.hjson index 38f5603..6cfd5ca 100644 --- a/assets/keys_test.hjson +++ b/assets/keys_test.hjson @@ -22,6 +22,9 @@ "foo\"bar": test "'''": test "foo'''bar": test + "'": test + "'foo": test + "foo'bar": test # control char in key name ":": test "foo:bar": test diff --git a/assets/sorted/keys_result.hjson b/assets/sorted/keys_result.hjson index 371764a..0e467b0 100644 --- a/assets/sorted/keys_result.hjson +++ b/assets/sorted/keys_result.hjson @@ -1,7 +1,9 @@ { "\"": test "#c1": test + "'": test "'''": test + "'foo": test -test: test .key: test "/*": test @@ -14,6 +16,7 @@ "foo\"bar": test "foo#bar": test "foo'''bar": test + "foo'bar": test "foo/*bar": test "foo/*foo*/bar": test "foo//bar": test diff --git a/assets/sorted/keys_result.json b/assets/sorted/keys_result.json index 95cc93f..d42daff 100644 --- a/assets/sorted/keys_result.json +++ b/assets/sorted/keys_result.json @@ -1,7 +1,9 @@ { "\"": "test", "#c1": "test", + "'": "test", "'''": "test", + "'foo": "test", "-test": "test", ".key": "test", "/*": "test", @@ -14,6 +16,7 @@ "foo\"bar": "test", "foo#bar": "test", "foo'''bar": "test", + "foo'bar": "test", "foo/*bar": "test", "foo/*foo*/bar": "test", "foo//bar": "test", diff --git a/assets/sorted/strings2_result.hjson b/assets/sorted/strings2_result.hjson new file mode 100644 index 0000000..d953929 --- /dev/null +++ b/assets/sorted/strings2_result.hjson @@ -0,0 +1,29 @@ +{ + foo3a: asdf''' + foo3b: "'''asdf" + foo4a: "asdf'''\nasdf" + foo4b: "asdf\n'''asdf" + "key \"": a key in single quotes + "key 2": a key in single quotes + key1: a key in single quotes + text: + [ + single quoted string + '''You need quotes for escapes''' + " untrimmed " + "untrimmed " + containing " double quotes + containing " double quotes + containing " double quotes + '''"containing more " double quotes"''' + containing ' single quotes + containing ' single quotes + containing ' single quotes + "'containing more ' single quotes'" + "'containing more ' single quotes'" + "\n" + " \n" + "\n \n \n \n" + "\t\n" + ] +} \ No newline at end of file diff --git a/assets/sorted/strings2_result.json b/assets/sorted/strings2_result.json new file mode 100644 index 0000000..5ada1b6 --- /dev/null +++ b/assets/sorted/strings2_result.json @@ -0,0 +1,28 @@ +{ + "foo3a": "asdf'''", + "foo3b": "'''asdf", + "foo4a": "asdf'''\nasdf", + "foo4b": "asdf\n'''asdf", + "key \"": "a key in single quotes", + "key 2": "a key in single quotes", + "key1": "a key in single quotes", + "text": [ + "single quoted string", + "You need quotes\tfor escapes", + " untrimmed ", + "untrimmed ", + "containing \" double quotes", + "containing \" double quotes", + "containing \" double quotes", + "\"containing more \" double quotes\"", + "containing ' single quotes", + "containing ' single quotes", + "containing ' single quotes", + "'containing more ' single quotes'", + "'containing more ' single quotes'", + "\n", + " \n", + "\n \n \n \n", + "\t\n" + ] +} \ No newline at end of file diff --git a/assets/strings2_result.hjson b/assets/strings2_result.hjson new file mode 100644 index 0000000..c89327f --- /dev/null +++ b/assets/strings2_result.hjson @@ -0,0 +1,29 @@ +{ + key1: a key in single quotes + "key 2": a key in single quotes + "key \"": a key in single quotes + text: + [ + single quoted string + '''You need quotes for escapes''' + " untrimmed " + "untrimmed " + containing " double quotes + containing " double quotes + containing " double quotes + '''"containing more " double quotes"''' + containing ' single quotes + containing ' single quotes + containing ' single quotes + "'containing more ' single quotes'" + "'containing more ' single quotes'" + "\n" + " \n" + "\n \n \n \n" + "\t\n" + ] + foo3a: asdf''' + foo3b: "'''asdf" + foo4a: "asdf'''\nasdf" + foo4b: "asdf\n'''asdf" +} \ No newline at end of file diff --git a/assets/strings2_result.json b/assets/strings2_result.json new file mode 100644 index 0000000..88b4ef2 --- /dev/null +++ b/assets/strings2_result.json @@ -0,0 +1,28 @@ +{ + "key1": "a key in single quotes", + "key 2": "a key in single quotes", + "key \"": "a key in single quotes", + "text": [ + "single quoted string", + "You need quotes\tfor escapes", + " untrimmed ", + "untrimmed ", + "containing \" double quotes", + "containing \" double quotes", + "containing \" double quotes", + "\"containing more \" double quotes\"", + "containing ' single quotes", + "containing ' single quotes", + "containing ' single quotes", + "'containing more ' single quotes'", + "'containing more ' single quotes'", + "\n", + " \n", + "\n \n \n \n", + "\t\n" + ], + "foo3a": "asdf'''", + "foo3b": "'''asdf", + "foo4a": "asdf'''\nasdf", + "foo4b": "asdf\n'''asdf" +} \ No newline at end of file diff --git a/assets/strings2_test.hjson b/assets/strings2_test.hjson new file mode 100644 index 0000000..875551f --- /dev/null +++ b/assets/strings2_test.hjson @@ -0,0 +1,36 @@ +{ + # Hjson 3 allows the use of single quotes + + 'key1': a key in single quotes + 'key 2': a key in single quotes + 'key "': a key in single quotes + + text: [ + 'single quoted string' + 'You need quotes\tfor escapes' + ' untrimmed ' + 'untrimmed ' + 'containing " double quotes' + 'containing \" double quotes' + "containing \" double quotes" + '"containing more " double quotes"' + 'containing \' single quotes' + "containing ' single quotes" + "containing \' single quotes" + "'containing more ' single quotes'" + "\'containing more \' single quotes\'" + + '\n' + ' \n' + '\n \n \n \n' + '\t\n' + ] + + # escapes/no escape + + foo3a: 'asdf\'\'\'' + foo3b: '\'\'\'asdf' + + foo4a: 'asdf\'\'\'\nasdf' + foo4b: 'asdf\n\'\'\'asdf' +} diff --git a/assets/testlist.txt b/assets/testlist.txt index f117c95..61de810 100644 --- a/assets/testlist.txt +++ b/assets/testlist.txt @@ -20,7 +20,6 @@ failJSON20_test.json failJSON21_test.json failJSON22_test.json failJSON23_test.json -failJSON24_test.json failJSON26_test.json failJSON28_test.json failJSON29_test.json @@ -71,8 +70,8 @@ pass2_test.json pass3_test.json pass4_test.json passSingle_test.hjson -root_test.hjson stringify1_test.hjson +strings2_test.hjson strings_test.hjson trail_test.hjson stringify/quotes_all_test.hjson @@ -81,4 +80,5 @@ stringify/quotes_keys_test.hjson stringify/quotes_strings_ml_test.json stringify/quotes_strings_test.hjson extra/notabs_test.json +extra/root_test.hjson extra/separator_test.json \ No newline at end of file diff --git a/assets/trail_test.hjson b/assets/trail_test.hjson index 62d98e9..28258c5 100644 --- a/assets/trail_test.hjson +++ b/assets/trail_test.hjson @@ -1,2 +1,4 @@ -// the following line contains trailing whitespace: -foo: 0 -- this string starts at 0 and ends at 1, preceding and trailing whitespace is ignored -- 1 +{ + // the following line contains trailing whitespace: + foo: 0 -- this string starts at 0 and ends at 1, preceding and trailing whitespace is ignored -- 1 +} diff --git a/decode.go b/decode.go index b1442b6..69ed10b 100644 --- a/decode.go +++ b/decode.go @@ -63,6 +63,7 @@ func (p *hjsonParser) peek(offs int) byte { var escapee = map[byte]byte{ '"': '"', + '\'': '\'', '\\': '\\', '/': '/', 'b': '\b', @@ -72,19 +73,24 @@ var escapee = map[byte]byte{ 't': '\t', } -func (p *hjsonParser) readString() (string, error) { +func (p *hjsonParser) readString(allowML bool) (string, error) { // Parse a string value. res := new(bytes.Buffer) + // callers make sure that (ch === '"' || ch === "'") // When parsing for string values, we must look for " and \ characters. - if p.ch != '"' { - return "", p.errAt("Bad string") - } + exitCh := p.ch for p.next() { - if p.ch == '"' { + if p.ch == exitCh { p.next() - return res.String(), nil + if allowML && p.ch == '\'' && res.Len() == 0 { + // ''' indicates a multiline string + p.next() + return p.readMLString() + } else { + return res.String(), nil + } } if p.ch == '\\' { p.next() @@ -195,8 +201,8 @@ func (p *hjsonParser) readKeyname() (string, error) { // quotes for keys are optional in Hjson // unless they include {}[],: or whitespace. - if p.ch == '"' { - return p.readString() + if p.ch == '"' || p.ch == '\'' { + return p.readString(false) } name := new(bytes.Buffer) @@ -264,13 +270,6 @@ func (p *hjsonParser) readTfnns() (interface{}, error) { return nil, p.errAt("Found a punctuator character '" + string(p.ch) + "' when expecting a quoteless string (check your syntax)") } chf := p.ch - if chf == '\'' && p.peek(0) == '\'' && p.peek(1) == '\'' { - p.next() - p.next() - p.next() - return p.readMLString() - } - value := new(bytes.Buffer) value.WriteByte(p.ch) @@ -407,8 +406,8 @@ func (p *hjsonParser) readValue() (interface{}, error) { return p.readObject(false) case '[': return p.readArray() - case '"': - return p.readString() + case '"', '\'': + return p.readString(true) default: return p.readTfnns() } diff --git a/encode.go b/encode.go index 0b80c8c..39fbb68 100644 --- a/encode.go +++ b/encode.go @@ -56,13 +56,13 @@ func init() { // needsEscape tests if the string can be written without escapes needsEscape = regexp.MustCompile(`[\\\"\x00-\x1f` + commonRange + `]`) // needsQuotes tests if the string can be written as a quoteless string (includes needsEscape but without \\ and \") - needsQuotes = regexp.MustCompile(`^\s|^"|^'''|^#|^/\*|^//|^\{|^\}|^\[|^\]|^:|^,|\s$|[\x00-\x1f\x7f-\x9f\x{00ad}\x{0600}-\x{0604}\x{070f}\x{17b4}\x{17b5}\x{200c}-\x{200f}\x{2028}-\x{202f}\x{2060}-\x{206f}\x{feff}\x{fff0}-\x{ffff}]`) + needsQuotes = regexp.MustCompile(`^\s|^"|^'|^#|^/\*|^//|^\{|^\}|^\[|^\]|^:|^,|\s$|[\x00-\x1f\x7f-\x9f\x{00ad}\x{0600}-\x{0604}\x{070f}\x{17b4}\x{17b5}\x{200c}-\x{200f}\x{2028}-\x{202f}\x{2060}-\x{206f}\x{feff}\x{fff0}-\x{ffff}]`) // needsEscapeML tests if the string can be written as a multiline string (like needsEscape but without \n, \r, \\, \", \t) var x08Or9 = `\x08` // `\x09` for the old behavior needsEscapeML = regexp.MustCompile(`'''|^[\s]+$|[\x00-` + x08Or9 + `\x0b\x0c\x0e-\x1f` + commonRange + `]`) // starts with a keyword and optionally is followed by a comment startsWithKeyword = regexp.MustCompile(`^(true|false|null)\s*((,|\]|\}|#|//|/\*).*)?$`) - needsEscapeName = regexp.MustCompile(`[,\{\[\}\]\s:#"]|//|/\*|'''`) + needsEscapeName = regexp.MustCompile(`[,\{\[\}\]\s:#"']|//|/\*`) } var meta = map[byte][]byte{