BREAKING CHANGE: emit eof only if options.concrete=true

README.md

@@ -58,6 +58,8 @@ In the browser:
 </script>
 ```
 
+`parse()` optionally takes an option bag with a boolean field `concrete`. Currently all it does is emitting [EOF](#end-of-file) node.
+
 `write()` optionally takes a "templates" object, whose properties are functions that process input in different ways (depending on what is needed for output). Every property is optional. Each property is documented below:
 
 ```js
@@ -675,6 +677,9 @@ The fields are as follows:
 }
 ```
 
+This type only appears as the last item of parser results, only if options.concrete is `true`.
+This is needed for the writer to keep any comments or whitespaces at the end of file.
+
 The fields are as follows:
 
 * `type`: Always "eof"

lib/webidl2.js

@@ -11,8 +11,10 @@ import { Includes } from "./productions/includes.js";
 
 /**
  * @param {Tokeniser} tokeniser
+ * @param {object} options
+ * @param {boolean} [options.concrete]
  */
-function parseByTokens(tokeniser) {
+function parseByTokens(tokeniser, options) {
   const source = tokeniser.source;
 
   const DECIMAL = "decimal";
@@ -805,15 +807,18 @@ function parseByTokens(tokeniser) {
       def.extAttrs = ea;
       defs.push(def);
     }
-    defs.push(consume("eof"));
+    const eof = consume("eof");
+    if (options.concrete) {
+      defs.push(eof);
+    }
     return defs;
   }
   const res = definitions();
   if (tokeniser.position < source.length) error("Unrecognised tokens");
   return res;
 }
 
-export function parse(str) {
+export function parse(str, options = {}) {
   const tokeniser = new Tokeniser(str);
-  return parseByTokens(tokeniser);
+  return parseByTokens(tokeniser, options);
 }

test/syntax.js

@@ -2,6 +2,7 @@
 
 const { collect } = require("./util/collect");
 const expect = require("expect");
+const webidl2 = require("../dist/webidl2");
 
 describe("Parses all of the IDLs to produce the correct ASTs", () => {
   for (const test of collect("syntax")) {
@@ -10,3 +11,16 @@ describe("Parses all of the IDLs to produce the correct ASTs", () => {
     });
   }
 });
+
+describe("Options", () => {
+  it("should emit EOF if concrete", () => {
+    const parsed = webidl2.parse("", { concrete: true });
+    expect(parsed.length).toBe(1);
+    expect(parsed[0].type).toBe("eof");
+  });
+
+  it("should not emit EOF if not concrete", () => {
+    const parsed = webidl2.parse("");
+    expect(parsed.length).toBe(0);
+  });
+});

test/util/acquire.js

@@ -1,6 +1,6 @@
 "use strict";
 
-const { collect } = require("./collect");
+const { collect } = require("./collect.js");
 const fs = require("fs");
 
 for (const test of collect("syntax")) {

test/util/collect.js

@@ -1,6 +1,6 @@
 "use strict";
 
-const wp = require("../../dist/webidl2");
+const wp = require("../../dist/webidl2.js");
 const pth = require("path");
 const fs = require("fs");
 const jdp = require("jsondiffpatch");
@@ -19,7 +19,7 @@ function* collect(base, { expectError, raw } = {}) {
   for (const path of idls) {
     try {
       const text = fs.readFileSync(path, "utf8");
-      const ast = wp.parse(text);
+      const ast = wp.parse(text, { concrete: true });
       const validation = wp.validate(ast);
       if (validation) {
         yield new TestItem({ text, ast, path, validation, raw });

test/writer.js

@@ -9,15 +9,16 @@ describe("Rewrite and parses all of the IDLs to produce the same ASTs", () => {
     it(`should produce the same AST for ${test.path}`, () => {
       const rewritten = webidl2.write(test.ast);
       expect(rewritten).toEqual(test.text);
-      const diff = test.diff(webidl2.parse(rewritten, test.opt));
+      const diff = test.diff(webidl2.parse(rewritten, { concrete: true }));
       expect(diff).toBe(undefined);
     });
   }
 });
 
 describe("Writer template functions", () => {
   function rewrite(text, templates) {
-    return webidl2.write(webidl2.parse(text), { templates });
+    const parsed = webidl2.parse(text, { concrete: true });
+    return webidl2.write(parsed, { templates });
   }
   function bracket(str) {
     return `<${str}>`;