Escape all control chars

assets/charset2_result.hjson

@@ -0,0 +1,5 @@
+{
+  uescape: "\u0000,\u0001,\uffff"
+  "um\u000blaut": äöüßÄÖÜ
+  hex: ģ䕧覫췯ꯍ
+}

assets/charset2_result.json

@@ -0,0 +1,5 @@
+{
+  "uescape": "\u0000,\u0001,\uffff",
+  "um\u000blaut": "äöüßÄÖÜ",
+  "hex": "ģ䕧覫췯ꯍ"
+}

assets/charset2_test.hjson

@@ -0,0 +1,5 @@
+{
+  uescape: "\u0000,\u0001,\uffff"
+  "um\u000blaut": äöüßÄÖÜ
+  hex: "\u0123\u4567\u89AB\uCDEF\uabcd\uef4A"
+}

assets/testlist.txt

@@ -1,4 +1,5 @@
 charset_test.hjson
+charset2_test.hjson
 comments_test.hjson
 empty_test.hjson
 failCharset1_test.hjson
@@ -83,4 +84,4 @@ stringify/quotes_strings_ml_test.json
 stringify/quotes_strings_test.hjson
 extra/notabs_test.json
 extra/root_test.hjson
-extra/separator_test.json
+extra/separator_test.json

src/main/org/hjson/HjsonWriter.java

@@ -30,6 +30,13 @@ class HjsonWriter {
 
   private IHjsonDsfProvider[] dsfProviders;
 
+  static String commonRange = "\\x7f-\\x9f\\x{00ad}\\x{0600}-\\x{0604}\\x{070f}\\x{17b4}\\x{17b5}\\x{200c}-\\x{200f}\\x{2028}-\\x{202f}\\x{2060}-\\x{206f}\\x{feff}\\x{fff0}-\\x{ffff}";
+  // needsEscape tests if the string can be written without escapes
+  static Pattern needsEscape = Pattern.compile("[\\\\\\\"\\x00-\\x1f" + commonRange + "]");
+  // needsQuotes tests if the string can be written as a quoteless string (includes needsEscape but without \\\\ and \\")
+  static Pattern needsQuotes = Pattern.compile("^\\s|^\"|^'|^#|^/\\*|^//|^\\{|^\\}|^\\[|^\\]|^:|^,|\\s$|[\\x00-\\x1f\\x7f-\\x9f\\x{00ad}\\x{0600}-\\x{0604}\\x{070f}\\x{17b4}\\x{17b5}\\x{200c}-\\x{200f}\\x{2028}-\\x{202f}\\x{2060}-\\x{206f}\\x{feff}\\x{fff0}-\\x{ffff}]");
+  // needsEscapeML tests if the string can be written as a multiline string (like needsEscape but without \\n, \\\\, \\", \\t)
+  static Pattern needsEscapeML = Pattern.compile("'''|^[\\s]+$|[\\x00-\\x08\\x0b-\\x1f" + commonRange + "]");
   static Pattern needsEscapeName=Pattern.compile("[,\\{\\[\\}\\]\\s:#\"']|//|/\\*");
 
   public HjsonWriter(HjsonOptions options) {
@@ -104,22 +111,21 @@ public void save(JsonValue value, Writer tw, int level, String separator, boolea
   }
 
   static String escapeName(String name) {
-    if (name.length()==0 || needsEscapeName.matcher(name).find())
+    if (name.length()==0 || needsEscapeName.matcher(name).find() ||
+      needsEscape.matcher(name).find())
+    {
       return "\""+JsonWriter.escapeString(name)+"\"";
-    else
+    } else {
       return name;
+    }
   }
 
   void writeString(String value, Writer tw, int level, String separator) throws IOException {
     if (value.length()==0) { tw.write(separator+"\"\""); return; }
 
     char left=value.charAt(0), right=value.charAt(value.length()-1);
     char left1=value.length()>1?value.charAt(1):'\0', left2=value.length()>2?value.charAt(2):'\0';
-    boolean doEscape=false;
-    char[] valuec=value.toCharArray();
-    for(char ch : valuec) {
-      if (needsQuotes(ch)) { doEscape=true; break; }
-    }
+    boolean doEscape=needsQuotes.matcher(value).find();
 
     if (doEscape ||
       HjsonParser.isWhiteSpace(left) || HjsonParser.isWhiteSpace(right) ||
@@ -136,17 +142,13 @@ void writeString(String value, Writer tw, int level, String separator) throws IO
       // format or we must replace the offending characters with safe escape
       // sequences.
 
-      boolean noEscape=true;
-      for(char ch : valuec) { if (needsEscape(ch)) { noEscape=false; break; } }
-      if (noEscape) { tw.write(separator+"\""+value+"\""); return; }
-
-      boolean noEscapeML=true, allWhite=true;
-      for(char ch : valuec) {
-        if (needsEscapeML(ch)) { noEscapeML=false; break; }
-        else if (!HjsonParser.isWhiteSpace(ch)) allWhite=false;
+      if (!needsEscape.matcher(value).find()) {
+        tw.write(separator+"\""+value+"\"");
+      } else if (!needsEscapeML.matcher(value).find()) {
+        writeMLString(value, tw, level, separator);
+      } else {
+        tw.write(separator+"\""+JsonWriter.escapeString(value)+"\"");
       }
-      if (noEscapeML && !allWhite && !value.contains("'''")) writeMLString(value, tw, level, separator);
-      else tw.write(separator+"\""+JsonWriter.escapeString(value)+"\"");
     }
     else tw.write(separator+value);
   }
@@ -183,38 +185,4 @@ static boolean startsWithKeyword(String text) {
     char ch=text.charAt(p);
     return ch==',' || ch=='}' || ch==']' || ch=='#' || ch=='/' && (text.length()>p+1 && (text.charAt(p+1)=='/' || text.charAt(p+1)=='*'));
   }
-
-  static boolean needsQuotes(char c) {
-    switch (c) {
-      case '\t':
-      case '\f':
-      case '\b':
-      case '\n':
-      case '\r':
-        return true;
-      default:
-        return false;
-    }
-  }
-
-  static boolean needsEscape(char c) {
-    switch (c) {
-      case '\"':
-      case '\\':
-        return true;
-      default:
-        return needsQuotes(c);
-    }
-  }
-
-  static boolean needsEscapeML(char c) {
-    switch (c) {
-      case '\n':
-      case '\r':
-      case '\t':
-        return false;
-      default:
-        return needsQuotes(c);
-    }
-  }
 }

src/main/org/hjson/JsonWriter.java

@@ -24,6 +24,7 @@
 
 import java.io.IOException;
 import java.io.Writer;
+import java.util.regex.Matcher;
 
 
 class JsonWriter {
@@ -46,7 +47,6 @@ public void save(JsonValue value, Writer tw, int level) throws IOException {
     switch (value.getType()) {
       case OBJECT:
         JsonObject obj=value.asObject();
-        if (obj.size()>0) nl(tw, level);
         tw.write('{');
         for (JsonObject.Member pair : obj) {
           if (following) tw.write(",");
@@ -56,8 +56,7 @@ public void save(JsonValue value, Writer tw, int level) throws IOException {
           tw.write("\":");
           //save(, tw, level+1, " ", false);
           JsonValue v=pair.getValue();
-          JsonType vType=v.getType();
-          if (format && vType!=JsonType.ARRAY && vType!=JsonType.OBJECT) tw.write(" ");
+          if (format) tw.write(" ");
           if (v==null) tw.write("null");
           else save(v, tw, level+1);
           following=true;
@@ -68,17 +67,14 @@ public void save(JsonValue value, Writer tw, int level) throws IOException {
       case ARRAY:
         JsonArray arr=value.asArray();
         int n=arr.size();
-        if (n>0) nl(tw, level);
         tw.write('[');
         for (int i=0; i<n; i++) {
-          if (following) tw.write(",");
+          if (i > 0) tw.write(",");
           JsonValue v=arr.get(i);
-          JsonType vType=v.getType();
-          if (vType!=JsonType.ARRAY && vType!=JsonType.OBJECT) nl(tw, level+1);
-          save(v, tw, level+1);
-          following=true;
+          nl(tw, level+1);
+          save(arr.get(i), tw, level+1);
         }
-        if (following) nl(tw, level);
+        if (n > 0) nl(tw, level);
         tw.write(']');
         break;
       case BOOLEAN:
@@ -95,42 +91,25 @@ public void save(JsonValue value, Writer tw, int level) throws IOException {
     }
   }
 
-  static String escapeName(String name) {
-    boolean needsEscape=name.length()==0;
-    for(char ch : name.toCharArray()) {
-      if (HjsonParser.isWhiteSpace(ch) || ch=='{' || ch=='}' || ch=='[' || ch==']' || ch==',' || ch==':') {
-        needsEscape=true;
-        break;
-      }
-    }
-    if (needsEscape) return "\""+JsonWriter.escapeString(name)+"\"";
-    else return name;
-  }
-
   static String escapeString(String src) {
     if (src==null) return null;
 
-    for (int i=0; i<src.length(); i++) {
-      if (getEscapedChar(src.charAt(i))!=null) {
-        StringBuilder sb=new StringBuilder();
-        if (i>0) sb.append(src, 0, i);
-        return doEscapeString(sb, src, i);
-      }
+    int i = 0;
+    StringBuilder sb=new StringBuilder();
+    Matcher m = HjsonWriter.needsEscape.matcher(src);
+
+    while (m.find()) {
+      // Assume all matches are single chars.
+      sb.append(src, i, m.start()).append(getEscapedChar(m.group().charAt(0)));
+      i = m.end();
     }
-    return src;
-  }
 
-  private static String doEscapeString(StringBuilder sb, String src, int cur) {
-    int start=cur;
-    for (int i=cur; i<src.length(); i++) {
-      String escaped=getEscapedChar(src.charAt(i));
-      if (escaped!=null) {
-        sb.append(src, start, i);
-        sb.append(escaped);
-        start=i+1;
-      }
+    if (i < 1) {
+      return src;
     }
-    sb.append(src, start, src.length());
+
+    sb.append(src, i, src.length());
+
     return sb.toString();
   }
 
@@ -143,7 +122,7 @@ private static String getEscapedChar(char c) {
       case '\f': return "\\f";
       case '\b': return "\\b";
       case '\\': return "\\\\";
-      default: return null;
+      default: return "\\u" + String.format("%04x", (int) c);
     }
   }
 }

src/test/org/hjson/test/Main.java

@@ -28,8 +28,6 @@ private static String load(String file, boolean cr) throws Exception {
   }
 
   private static boolean test(String name, String file, boolean inputCr, boolean outputCr) throws Exception {
-    int extIdx=file.lastIndexOf('.');
-    boolean isJson=extIdx>=0 && file.substring(extIdx).equals(".json");
     boolean shouldFail=name.startsWith("fail");
 
     JsonValue.setEol(outputCr?"\r\n":"\n");
@@ -43,16 +41,13 @@ private static boolean test(String name, String file, boolean inputCr, boolean o
       String data1=data.toString(Stringify.FORMATTED);
       String hjson1=data.toString(Stringify.HJSON);
       if (!shouldFail) {
-        JsonValue result=JsonValue.readJSON(load(name+"_result.json", inputCr));
+        String json2 = load(name+"_result.json", outputCr);
+        JsonValue result=JsonValue.readJSON(json2);
         String data2=result.toString(Stringify.FORMATTED);
         String hjson2=load(name+"_result.hjson", outputCr);
         if (!data1.equals(data2)) return failErr(name, "parse", data1, data2);
         if (!hjson1.equals(hjson2)) return failErr(name, "stringify", hjson1, hjson2);
-
-        if (isJson) {
-          String json1=data.toString(), json2=JsonValue.readHjson(text, opt).toString();
-          if (!json1.equals(json2)) return failErr(name, "json chk", json1, json2);
-        }
+        if (!data1.equals(json2)) return failErr(name, "JSON stringify", data1, json2);
       }
       else return failErr(name, "should fail", null, null);
     }