Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Escape all control chars #36

Merged
merged 3 commits into from
Oct 14, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions assets/charset2_result.hjson
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
uescape: "\u0000,\u0001,\uffff"
"um\u000blaut": äöüßÄÖÜ
hex: ģ䕧覫췯ꯍ
}
5 changes: 5 additions & 0 deletions assets/charset2_result.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"uescape": "\u0000,\u0001,\uffff",
"um\u000blaut": "äöüßÄÖÜ",
"hex": "ģ䕧覫췯ꯍ"
}
5 changes: 5 additions & 0 deletions assets/charset2_test.hjson
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
uescape: "\u0000,\u0001,\uffff"
"um\u000blaut": äöüßÄÖÜ
hex: "\u0123\u4567\u89AB\uCDEF\uabcd\uef4A"
}
3 changes: 2 additions & 1 deletion assets/testlist.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
charset_test.hjson
charset2_test.hjson
comments_test.hjson
empty_test.hjson
failCharset1_test.hjson
Expand Down Expand Up @@ -83,4 +84,4 @@ stringify/quotes_strings_ml_test.json
stringify/quotes_strings_test.hjson
extra/notabs_test.json
extra/root_test.hjson
extra/separator_test.json
extra/separator_test.json
70 changes: 19 additions & 51 deletions src/main/org/hjson/HjsonWriter.java
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,13 @@ class HjsonWriter {

private IHjsonDsfProvider[] dsfProviders;

static String commonRange = "\\x7f-\\x9f\\x{00ad}\\x{0600}-\\x{0604}\\x{070f}\\x{17b4}\\x{17b5}\\x{200c}-\\x{200f}\\x{2028}-\\x{202f}\\x{2060}-\\x{206f}\\x{feff}\\x{fff0}-\\x{ffff}";
// needsEscape tests if the string can be written without escapes
static Pattern needsEscape = Pattern.compile("[\\\\\\\"\\x00-\\x1f" + commonRange + "]");
// needsQuotes tests if the string can be written as a quoteless string (includes needsEscape but without \\\\ and \\")
static Pattern needsQuotes = Pattern.compile("^\\s|^\"|^'|^#|^/\\*|^//|^\\{|^\\}|^\\[|^\\]|^:|^,|\\s$|[\\x00-\\x1f\\x7f-\\x9f\\x{00ad}\\x{0600}-\\x{0604}\\x{070f}\\x{17b4}\\x{17b5}\\x{200c}-\\x{200f}\\x{2028}-\\x{202f}\\x{2060}-\\x{206f}\\x{feff}\\x{fff0}-\\x{ffff}]");
// needsEscapeML tests if the string can be written as a multiline string (like needsEscape but without \\n, \\\\, \\", \\t)
static Pattern needsEscapeML = Pattern.compile("'''|^[\\s]+$|[\\x00-\\x08\\x0b-\\x1f" + commonRange + "]");
static Pattern needsEscapeName=Pattern.compile("[,\\{\\[\\}\\]\\s:#\"']|//|/\\*");

public HjsonWriter(HjsonOptions options) {
Expand Down Expand Up @@ -104,22 +111,21 @@ public void save(JsonValue value, Writer tw, int level, String separator, boolea
}

static String escapeName(String name) {
if (name.length()==0 || needsEscapeName.matcher(name).find())
if (name.length()==0 || needsEscapeName.matcher(name).find() ||
needsEscape.matcher(name).find())
{
return "\""+JsonWriter.escapeString(name)+"\"";
else
} else {
return name;
}
}

void writeString(String value, Writer tw, int level, String separator) throws IOException {
if (value.length()==0) { tw.write(separator+"\"\""); return; }

char left=value.charAt(0), right=value.charAt(value.length()-1);
char left1=value.length()>1?value.charAt(1):'\0', left2=value.length()>2?value.charAt(2):'\0';
boolean doEscape=false;
char[] valuec=value.toCharArray();
for(char ch : valuec) {
if (needsQuotes(ch)) { doEscape=true; break; }
}
boolean doEscape=needsQuotes.matcher(value).find();

if (doEscape ||
HjsonParser.isWhiteSpace(left) || HjsonParser.isWhiteSpace(right) ||
Expand All @@ -136,17 +142,13 @@ void writeString(String value, Writer tw, int level, String separator) throws IO
// format or we must replace the offending characters with safe escape
// sequences.

boolean noEscape=true;
for(char ch : valuec) { if (needsEscape(ch)) { noEscape=false; break; } }
if (noEscape) { tw.write(separator+"\""+value+"\""); return; }

boolean noEscapeML=true, allWhite=true;
for(char ch : valuec) {
if (needsEscapeML(ch)) { noEscapeML=false; break; }
else if (!HjsonParser.isWhiteSpace(ch)) allWhite=false;
if (!needsEscape.matcher(value).find()) {
tw.write(separator+"\""+value+"\"");
} else if (!needsEscapeML.matcher(value).find()) {
writeMLString(value, tw, level, separator);
} else {
tw.write(separator+"\""+JsonWriter.escapeString(value)+"\"");
}
if (noEscapeML && !allWhite && !value.contains("'''")) writeMLString(value, tw, level, separator);
else tw.write(separator+"\""+JsonWriter.escapeString(value)+"\"");
}
else tw.write(separator+value);
}
Expand Down Expand Up @@ -183,38 +185,4 @@ static boolean startsWithKeyword(String text) {
char ch=text.charAt(p);
return ch==',' || ch=='}' || ch==']' || ch=='#' || ch=='/' && (text.length()>p+1 && (text.charAt(p+1)=='/' || text.charAt(p+1)=='*'));
}

static boolean needsQuotes(char c) {
switch (c) {
case '\t':
case '\f':
case '\b':
case '\n':
case '\r':
return true;
default:
return false;
}
}

static boolean needsEscape(char c) {
switch (c) {
case '\"':
case '\\':
return true;
default:
return needsQuotes(c);
}
}

static boolean needsEscapeML(char c) {
switch (c) {
case '\n':
case '\r':
case '\t':
return false;
default:
return needsQuotes(c);
}
}
}
61 changes: 20 additions & 41 deletions src/main/org/hjson/JsonWriter.java
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@

import java.io.IOException;
import java.io.Writer;
import java.util.regex.Matcher;


class JsonWriter {
Expand All @@ -46,7 +47,6 @@ public void save(JsonValue value, Writer tw, int level) throws IOException {
switch (value.getType()) {
case OBJECT:
JsonObject obj=value.asObject();
if (obj.size()>0) nl(tw, level);
tw.write('{');
for (JsonObject.Member pair : obj) {
if (following) tw.write(",");
Expand All @@ -56,8 +56,7 @@ public void save(JsonValue value, Writer tw, int level) throws IOException {
tw.write("\":");
//save(, tw, level+1, " ", false);
JsonValue v=pair.getValue();
JsonType vType=v.getType();
if (format && vType!=JsonType.ARRAY && vType!=JsonType.OBJECT) tw.write(" ");
if (format) tw.write(" ");
if (v==null) tw.write("null");
else save(v, tw, level+1);
following=true;
Expand All @@ -68,17 +67,14 @@ public void save(JsonValue value, Writer tw, int level) throws IOException {
case ARRAY:
JsonArray arr=value.asArray();
int n=arr.size();
if (n>0) nl(tw, level);
tw.write('[');
for (int i=0; i<n; i++) {
if (following) tw.write(",");
if (i > 0) tw.write(",");
JsonValue v=arr.get(i);
JsonType vType=v.getType();
if (vType!=JsonType.ARRAY && vType!=JsonType.OBJECT) nl(tw, level+1);
save(v, tw, level+1);
following=true;
nl(tw, level+1);
save(arr.get(i), tw, level+1);
}
if (following) nl(tw, level);
if (n > 0) nl(tw, level);
tw.write(']');
break;
case BOOLEAN:
Expand All @@ -95,42 +91,25 @@ public void save(JsonValue value, Writer tw, int level) throws IOException {
}
}

static String escapeName(String name) {
boolean needsEscape=name.length()==0;
for(char ch : name.toCharArray()) {
if (HjsonParser.isWhiteSpace(ch) || ch=='{' || ch=='}' || ch=='[' || ch==']' || ch==',' || ch==':') {
needsEscape=true;
break;
}
}
if (needsEscape) return "\""+JsonWriter.escapeString(name)+"\"";
else return name;
}

static String escapeString(String src) {
if (src==null) return null;

for (int i=0; i<src.length(); i++) {
if (getEscapedChar(src.charAt(i))!=null) {
StringBuilder sb=new StringBuilder();
if (i>0) sb.append(src, 0, i);
return doEscapeString(sb, src, i);
}
int i = 0;
StringBuilder sb=new StringBuilder();
Matcher m = HjsonWriter.needsEscape.matcher(src);

while (m.find()) {
// Assume all matches are single chars.
sb.append(src, i, m.start()).append(getEscapedChar(m.group().charAt(0)));
i = m.end();
}
return src;
}

private static String doEscapeString(StringBuilder sb, String src, int cur) {
int start=cur;
for (int i=cur; i<src.length(); i++) {
String escaped=getEscapedChar(src.charAt(i));
if (escaped!=null) {
sb.append(src, start, i);
sb.append(escaped);
start=i+1;
}
if (i < 1) {
return src;
}
sb.append(src, start, src.length());

sb.append(src, i, src.length());

return sb.toString();
}

Expand All @@ -143,7 +122,7 @@ private static String getEscapedChar(char c) {
case '\f': return "\\f";
case '\b': return "\\b";
case '\\': return "\\\\";
default: return null;
default: return "\\u" + String.format("%04x", (int) c);
}
}
}
11 changes: 3 additions & 8 deletions src/test/org/hjson/test/Main.java
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,6 @@ private static String load(String file, boolean cr) throws Exception {
}

private static boolean test(String name, String file, boolean inputCr, boolean outputCr) throws Exception {
int extIdx=file.lastIndexOf('.');
boolean isJson=extIdx>=0 && file.substring(extIdx).equals(".json");
boolean shouldFail=name.startsWith("fail");

JsonValue.setEol(outputCr?"\r\n":"\n");
Expand All @@ -43,16 +41,13 @@ private static boolean test(String name, String file, boolean inputCr, boolean o
String data1=data.toString(Stringify.FORMATTED);
String hjson1=data.toString(Stringify.HJSON);
if (!shouldFail) {
JsonValue result=JsonValue.readJSON(load(name+"_result.json", inputCr));
String json2 = load(name+"_result.json", outputCr);
JsonValue result=JsonValue.readJSON(json2);
String data2=result.toString(Stringify.FORMATTED);
String hjson2=load(name+"_result.hjson", outputCr);
if (!data1.equals(data2)) return failErr(name, "parse", data1, data2);
if (!hjson1.equals(hjson2)) return failErr(name, "stringify", hjson1, hjson2);

if (isJson) {
String json1=data.toString(), json2=JsonValue.readHjson(text, opt).toString();
if (!json1.equals(json2)) return failErr(name, "json chk", json1, json2);
}
if (!data1.equals(json2)) return failErr(name, "JSON stringify", data1, json2);
}
else return failErr(name, "should fail", null, null);
}
Expand Down