Merge pull request #206 from Mingun/debug-helpers

Debug helpers - new output type `ast` / CLI option `-a/--ast`

Author: hildjj

CHANGELOG.md

@@ -29,6 +29,9 @@ Released: TBD
 - [#285](https://github.com/peggyjs/peggy/issues/285) Require that a non-empty
   string be given as a grammarSource if you are generating a source map, from
   @hildjj
+- [#206](https://github.com/peggyjs/peggy/pull/206): New output type `ast` and
+  an `--ast` flag for the CLI to get an internal grammar AST for investigation
+  (can be useful for plugin writers), from @Mingun
 
 ### Bug Fixes
 

bin/peggy-cli.js

@@ -14,7 +14,7 @@ exports.CommanderError = CommanderError;
 exports.InvalidArgumentError = InvalidArgumentError;
 
 // Options that aren't for the API directly:
-const PROG_OPTIONS = ["input", "output", "sourceMap", "startRule", "test", "testFile", "verbose"];
+const PROG_OPTIONS = ["ast", "input", "output", "sourceMap", "startRule", "test", "testFile", "verbose"];
 const MODULE_FORMATS = ["amd", "bare", "commonjs", "es", "globals", "umd"];
 const MODULE_FORMATS_WITH_DEPS = ["amd", "commonjs", "es", "umd"];
 const MODULE_FORMATS_WITH_GLOBAL = ["globals", "umd"];
@@ -173,14 +173,22 @@ class PeggyCLI extends Command {
         "-m, --source-map [mapfile]",
         "Generate a source map. If name is not specified, the source map will be named \"<input_file>.map\" if input is a file and \"source.map\" if input is a standard input. If the special filename `inline` is given, the sourcemap will be embedded in the output file as a data URI.  If the filename is prefixed with `hidden:`, no mapping URL will be included so that the mapping can be specified with an HTTP SourceMap: header.  This option conflicts with the `-t/--test` and `-T/--test-file` options unless `-o/--output` is also specified"
       )
+      .addOption(
+        new Option(
+          "--ast",
+          "Output a grammar AST instead of a parser code"
+        )
+          .default(false)
+          .conflicts(["test", "testFile", "sourceMap"])
+      )
       .option(
         "-S, --start-rule <rule>",
         "When testing, use the given rule as the start rule.  If this rule is not in the allowed start rules, it will be added."
       )
-      .addOption(new Option(
+      .option(
         "-t, --test <text>",
         "Test the parser with the given text, outputting the result of running the parser instead of the parser itself. If the input to be tested is not parsed, the CLI will exit with code 2"
-      ).conflicts("test-file"))
+      )
       .addOption(new Option(
         "-T, --test-file <filename>",
         "Test the parser with the contents of the given file, outputting the result of running the parser instead of the parser itself. If the input to be tested is not parsed, the CLI will exit with code 2"
@@ -313,6 +321,10 @@ class PeggyCLI extends Command {
           }
         }
 
+        if (this.progOptions.ast) {
+          this.argv.output = "ast";
+        }
+
         // Empty string is a valid test input.  Don't just test for falsy.
         if (typeof this.progOptions.test === "string") {
           this.testText = this.progOptions.test;
@@ -511,7 +523,7 @@ class PeggyCLI extends Command {
     });
   }
 
-  writeParser(outputStream, source) {
+  writeOutput(outputStream, source) {
     return new Promise((resolve, reject) => {
       if (!outputStream) {
         resolve();
@@ -626,18 +638,24 @@ class PeggyCLI extends Command {
       this.verbose("CLI", errorText = "parsing grammar");
       const source = peggy.generate(input, this.argv); // All of the real work.
 
-      this.verbose("CLI", errorText = "writing to output file");
+      this.verbose("CLI", errorText = "open output stream");
       const outputStream = await this.openOutputStream();
 
-      this.verbose("CLI", errorText = "writing sourceMap");
-      const mappedSource = await this.writeSourceMap(source);
+      // If option `--ast` is specified, `generate()` returns an AST object
+      if (this.progOptions.ast) {
+        this.verbose("CLI", errorText = "writing AST");
+        await this.writeOutput(outputStream, JSON.stringify(source, null, 2));
+      } else {
+        this.verbose("CLI", errorText = "writing sourceMap");
+        const mappedSource = await this.writeSourceMap(source);
 
-      this.verbose("CLI", errorText = "writing parser");
-      await this.writeParser(outputStream, mappedSource);
+        this.verbose("CLI", errorText = "writing parser");
+        await this.writeOutput(outputStream, mappedSource);
 
-      exitCode = 2;
-      this.verbose("CLI", errorText = "running test");
-      this.test(mappedSource);
+        exitCode = 2;
+        this.verbose("CLI", errorText = "running test");
+        this.test(mappedSource);
+      }
     } catch (error) {
       // Will either exit or throw.
       this.error(errorText, {

docs/documentation.html

@@ -171,6 +171,13 @@ <h3 id="generating-a-parser-command-line">Command Line</h3>
   <dd>Comma-separated list of rules the parser will be allowed to start parsing
   from (default: only the first rule in the grammar).</dd>
 
+  <dt><code>--ast</code></dt>
+  <dd>Outputting an internal AST representation of the grammar after
+  all optimizations instead of the parser source code. Useful for plugin authors
+  to see how their plugin changes the AST. This option cannot be mixed with the
+  <code>-t/--test</code>, <code>-T/--test-file</code> and <code>-m/--source-map</code>
+  options.</dd>
+
   <dt><code>--cache</code></dt>
   <dd>Makes the parser cache results, avoiding exponential parsing time in
   pathological cases but making the parser slower.</dd>
@@ -227,21 +234,21 @@ <h3 id="generating-a-parser-command-line">Command Line</h3>
   <dt><code>-t</code>, <code>--test &lt;text&gt;</code></dt>
   <dd>Test the parser with the given text, outputting the result of running
   the parser against this input.
-  If the input to be tested is not parsed, the CLI will exit with code 2</dd>
+  If the input to be tested is not parsed, the CLI will exit with code 2.</dd>
 
   <dt><code>-T</code>, <code>--test-file &lt;text&gt;</code></dt>
   <dd>Test the parser with the contents of the given file, outputting the
   result of running the parser against this input.
-  If the input to be tested is not parsed, the CLI will exit with code 2</dd>
+  If the input to be tested is not parsed, the CLI will exit with code 2.</dd>
 
   <dt><code>--trace</code></dt>
   <dd>Makes the parser trace its progress.</dd>
 
   <dt><code>-v</code>, <code>--version</code></dt>
-  <dd>Output the version number</dd>
+  <dd>Output the version number.</dd>
 
   <dt><code>-h</code>, <code>--help</code></dt>
-  <dd>Display help for the command</dd>
+  <dd>Display help for the command.</dd>
 
 </dl>
 
@@ -270,7 +277,7 @@ <h3 id="generating-a-parser-command-line">Command Line</h3>
 <p>
 You can test generated parser immediately if you specify the <code>-t/--test</code> or <code>-T/--test-file</code>
 option. This option conflicts with the option <code>-m/--source-map</code> unless <code>-o/--output</code> is
-also specified.
+also specified. This option conflicts with the <code>--ast</code> option.
 </p>
 
 <p>The CLI will exit with the code:</p>
@@ -411,6 +418,9 @@ <h3 id="generating-a-parser-javascript-api">JavaScript API</h3>
         with an embedded source map as a <code>data:</code> URI.  This option
         leads to a larger output string, but is the easiest to integrate with
         developer tooling.</li>
+        <li><code>"ast"</code> - return the internal AST of the grammar as a JSON
+          string. Useful for plugin authors to explore internals of Peggy and
+          for automation.</li>
     </ul>
     <p>(default: <code>"parser"</code>)</p>
     <blockquote>

lib/compiler/index.js

@@ -141,6 +141,9 @@ const compiler = {
 `;
       }
 
+      case "ast":
+        return ast;
+
       default:
         throw new Error("Invalid output format: " + options.output + ".");
     }

lib/compiler/passes/generate-js.js

@@ -258,7 +258,7 @@ function generateJS(ast, options) {
 
   function generateRuleFunction(rule) {
     const parts = [];
-    const stack = new Stack(rule.name, "s", "var");
+    const stack = new Stack(rule.name, "s", "var", rule.bytecode);
 
     function compile(bc) {
       let ip = 0;
@@ -528,7 +528,7 @@ function generateJS(ast, options) {
 
           // istanbul ignore next Because we never generate invalid bytecode we cannot reach this branch
           default:
-            throw new Error("Invalid opcode: " + bc[ip] + ".");
+            throw new Error("Invalid opcode: " + bc[ip] + ".", { rule: rule.name, bytecode: bc });
         }
       }
 

lib/compiler/stack.js

@@ -8,15 +8,17 @@ class Stack {
    * @param {string} ruleName The name of rule that will be used in error messages
    * @param {string} varName The prefix for generated names of variables
    * @param {string} type The type of the variables. For JavaScript there are `var` or `let`
+   * @param {number[]} bytecode Bytecode for error messages
    */
-  constructor(ruleName, varName, type) {
+  constructor(ruleName, varName, type, bytecode) {
     /** Last used variable in the stack. */
     this.sp       = -1;
     /** Maximum stack size. */
     this.maxSp    = -1;
     this.varName  = varName;
     this.ruleName = ruleName;
     this.type     = type;
+    this.bytecode = bytecode;
   }
 
   /**
@@ -30,7 +32,7 @@ class Stack {
   name(i) {
     if (i < 0) {
       throw new RangeError(
-        `Rule '${this.ruleName}': The variable stack underflow: attempt to use a variable '${this.varName}<x>' at an index ${i}`
+        `Rule '${this.ruleName}': The variable stack underflow: attempt to use a variable '${this.varName}<x>' at an index ${i}.\nBytecode: ${this.bytecode}`
       );
     }
 
@@ -90,7 +92,7 @@ class Stack {
   index(i) {
     if (i < 0) {
       throw new RangeError(
-        `Rule '${this.ruleName}': The variable stack overflow: attempt to get a variable at a negative index ${i}`
+        `Rule '${this.ruleName}': The variable stack overflow: attempt to get a variable at a negative index ${i}.\nBytecode: ${this.bytecode}`
       );
     }
 
@@ -107,7 +109,7 @@ class Stack {
   result() {
     if (this.maxSp < 0) {
       throw new RangeError(
-        `Rule '${this.ruleName}': The variable stack is empty, can't get the result'`
+        `Rule '${this.ruleName}': The variable stack is empty, can't get the result.\nBytecode: ${this.bytecode}`
       );
     }
 
@@ -154,7 +156,8 @@ class Stack {
         throw new Error(
           "Rule '" + this.ruleName + "', position " + pos + ": "
           + "Branches of a condition can't move the stack pointer differently "
-          + "(before: " + baseSp + ", after then: " + thenSp + ", after else: " + this.sp + ")."
+          + "(before: " + baseSp + ", after then: " + thenSp + ", after else: " + this.sp + "). "
+          + "Bytecode: " + this.bytecode
         );
       }
     }
@@ -178,7 +181,8 @@ class Stack {
       throw new Error(
         "Rule '" + this.ruleName + "', position " + pos + ": "
         + "Body of a loop can't move the stack pointer "
-        + "(before: " + baseSp + ", after: " + this.sp + ")."
+        + "(before: " + baseSp + ", after: " + this.sp + "). "
+        + "Bytecode: " + this.bytecode
       );
     }
   }

lib/peg.d.ts

@@ -1079,7 +1079,7 @@ export type SourceOutputs =
   "source-with-inline-map";
 
 /** Base options for all source-generating formats. */
-interface SourceOptionsBase<Output extends SourceOutputs>
+interface SourceOptionsBase<Output>
   extends BuildOptionsBase {
   /**
    * If set to `"parser"`, the method will return generated parser object;
@@ -1246,5 +1246,26 @@ export function generate(
   options: SourceBuildOptions<SourceOutputs>
 ): string | SourceNode;
 
+/**
+ * Returns the generated AST for the grammar. Unlike result of the
+ * `peggy.compiler.compile(...)` an AST returned by this method is augmented
+ * with data from passes. In other words, the compiler gives you the raw AST,
+ * and this method provides the final AST after all optimizations and
+ * transformations.
+ *
+ * @param grammar String in the format described by the meta-grammar in the
+ *        `parser.pegjs` file
+ * @param options Options that allow you to customize returned AST
+ *
+ * @throws {SyntaxError}  If the grammar contains a syntax error, for example,
+ *         an unclosed brace
+ * @throws {GrammarError} If the grammar contains a semantic error, for example,
+ *         duplicated labels
+ */
+export function generate(
+  grammar: string,
+  options: SourceOptionsBase<"ast">
+): ast.Grammar;
+
 // Export all exported stuff under a global variable PEG in non-module environments
 export as namespace PEG;

package-lock.json

@@ -62,6 +62,7 @@
     "@typescript-eslint/parser": "^5.27.0",
     "browser-sync": "^2.27.10",
     "chai": "^4.3.6",
+    "chai-like": "^1.1.1",
     "copyfiles": "^2.4.1",
     "eslint": "^8.16.0",
     "express": "4.18.1",

package.json

@@ -10,6 +10,8 @@ exports.peggyVersion = function peggyVersion() {
   return peg.VERSION;
 };
 
+chai.use(require("chai-like"));
+
 beforeEach(() => {
   // In the browser, initialize SourceMapConsumer's wasm bits.
   // This is *async*, so make sure to return the promise to make
@@ -173,6 +175,15 @@ describe("Peggy API", () => {
           expect(eval(source).parse("a")).to.equal("a");
         });
       });
+
+      describe("when |output| is set to |\"ast\"|", () => {
+        it("returns generated parser AST", () => {
+          const ast = peg.generate(grammar, { output: "ast" });
+
+          expect(ast).to.be.an("object");
+          expect(ast).to.be.like(peg.parser.parse(grammar));
+        });
+      });
     });
 
     // The |format|, |exportVars|, and |dependencies| options are not tested