Add new output type "ast" (TODO: failed test)

Author: Mingun

CHANGELOG.md

@@ -97,6 +97,9 @@ Released: 2022-05-28
   XML and source-mapping, from @hildjj
 - [#204](https://github.com/peggyjs/peggy/pull/204): Increase coverage for the
   tests, from @Mingun
+- [#206](https://github.com/peggyjs/peggy/pull/206): New output type `ast` and a
+  flag `-a/--ast` for the CLI to get an internal grammar AST for investigation
+  (can be useful for plugin writers), from @Mingun
 - [#210](https://github.com/peggyjs/peggy/pull/210): Refactor CLI testing,
   from @hildjj
 

bin/peggy-cli.js

@@ -14,7 +14,7 @@ exports.CommanderError = CommanderError;
 exports.InvalidArgumentError = InvalidArgumentError;
 
 // Options that aren't for the API directly:
-const PROG_OPTIONS = ["input", "output", "sourceMap", "startRule", "test", "testFile", "verbose"];
+const PROG_OPTIONS = ["ast", "input", "output", "sourceMap", "startRule", "test", "testFile", "verbose"];
 const MODULE_FORMATS = ["amd", "bare", "commonjs", "es", "globals", "umd"];
 const MODULE_FORMATS_WITH_DEPS = ["amd", "commonjs", "es", "umd"];
 const MODULE_FORMATS_WITH_GLOBAL = ["globals", "umd"];
@@ -173,14 +173,22 @@ class PeggyCLI extends Command {
         "-m, --source-map [mapfile]",
         "Generate a source map. If name is not specified, the source map will be named \"<input_file>.map\" if input is a file and \"source.map\" if input is a standard input. If the special filename `inline` is given, the sourcemap will be embedded in the output file as a data URI.  If the filename is prefixed with `hidden:`, no mapping URL will be included so that the mapping can be specified with an HTTP SourceMap: header.  This option conflicts with the `-t/--test` and `-T/--test-file` options unless `-o/--output` is also specified"
       )
+      .addOption(
+        new Option(
+          "-a, --ast",
+          "Output a grammar AST instead of a parser code"
+        )
+          .default(false)
+          .conflicts(["test", "testFile", "sourceMap"])
+      )
       .option(
         "-S, --start-rule <rule>",
         "When testing, use the given rule as the start rule.  If this rule is not in the allowed start rules, it will be added."
       )
-      .addOption(new Option(
+      .option(
         "-t, --test <text>",
         "Test the parser with the given text, outputting the result of running the parser instead of the parser itself. If the input to be tested is not parsed, the CLI will exit with code 2"
-      ).conflicts("test-file"))
+      )
       .addOption(new Option(
         "-T, --test-file <filename>",
         "Test the parser with the contents of the given file, outputting the result of running the parser instead of the parser itself. If the input to be tested is not parsed, the CLI will exit with code 2"
@@ -313,6 +321,10 @@ class PeggyCLI extends Command {
           }
         }
 
+        if (this.progOptions.ast) {
+          this.argv.output = "ast";
+        }
+
         // Empty string is a valid test input.  Don't just test for falsy.
         if (typeof this.progOptions.test === "string") {
           this.testText = this.progOptions.test;
@@ -511,7 +523,7 @@ class PeggyCLI extends Command {
     });
   }
 
-  writeParser(outputStream, source) {
+  writeOutput(outputStream, source) {
     return new Promise((resolve, reject) => {
       if (!outputStream) {
         resolve();
@@ -626,18 +638,24 @@ class PeggyCLI extends Command {
       this.verbose("CLI", errorText = "parsing grammar");
       const source = peggy.generate(input, this.argv); // All of the real work.
 
-      this.verbose("CLI", errorText = "writing to output file");
+      this.verbose("CLI", errorText = "open output stream");
       const outputStream = await this.openOutputStream();
 
-      this.verbose("CLI", errorText = "writing sourceMap");
-      const mappedSource = await this.writeSourceMap(source);
+      // If option `--ast` is specified, `generate()` returns an AST object
+      if (this.progOptions.ast) {
+        this.verbose("CLI", errorText = "writing AST");
+        await this.writeOutput(outputStream, JSON.stringify(source, null, 2));
+      } else {
+        this.verbose("CLI", errorText = "writing sourceMap");
+        const mappedSource = await this.writeSourceMap(source);
 
-      this.verbose("CLI", errorText = "writing parser");
-      await this.writeParser(outputStream, mappedSource);
+        this.verbose("CLI", errorText = "writing parser");
+        await this.writeOutput(outputStream, mappedSource);
 
-      exitCode = 2;
-      this.verbose("CLI", errorText = "running test");
-      this.test(mappedSource);
+        exitCode = 2;
+        this.verbose("CLI", errorText = "running test");
+        this.test(mappedSource);
+      }
     } catch (error) {
       // Will either exit or throw.
       this.error(errorText, {

docs/documentation.html

@@ -171,6 +171,12 @@ <h3 id="generating-a-parser-command-line">Command Line</h3>
   <dd>Comma-separated list of rules the parser will be allowed to start parsing
   from (default: only the first rule in the grammar).</dd>
 
+  <dt><code>-a</code>, <code>--ast</code></dt>
+  <dd>Outputting an internal AST representation of the grammar after
+  all optimizations instead of the parser source code. Useful for plugin authors
+  to see how their plugin changes the AST. This option cannot be mixed with the
+  <code>-t/--test</code> and <code>-T/--test-file</code> options.</dd>
+
   <dt><code>--cache</code></dt>
   <dd>Makes the parser cache results, avoiding exponential parsing time in
   pathological cases but making the parser slower.</dd>
@@ -227,21 +233,21 @@ <h3 id="generating-a-parser-command-line">Command Line</h3>
   <dt><code>-t</code>, <code>--test &lt;text&gt;</code></dt>
   <dd>Test the parser with the given text, outputting the result of running
   the parser against this input.
-  If the input to be tested is not parsed, the CLI will exit with code 2</dd>
+  If the input to be tested is not parsed, the CLI will exit with code 2.</dd>
 
   <dt><code>-T</code>, <code>--test-file &lt;text&gt;</code></dt>
   <dd>Test the parser with the contents of the given file, outputting the
   result of running the parser against this input.
-  If the input to be tested is not parsed, the CLI will exit with code 2</dd>
+  If the input to be tested is not parsed, the CLI will exit with code 2.</dd>
 
   <dt><code>--trace</code></dt>
   <dd>Makes the parser trace its progress.</dd>
 
   <dt><code>-v</code>, <code>--version</code></dt>
-  <dd>Output the version number</dd>
+  <dd>Output the version number.</dd>
 
   <dt><code>-h</code>, <code>--help</code></dt>
-  <dd>Display help for the command</dd>
+  <dd>Display help for the command.</dd>
 
 </dl>
 
@@ -270,7 +276,7 @@ <h3 id="generating-a-parser-command-line">Command Line</h3>
 <p>
 You can test generated parser immediately if you specify the <code>-t/--test</code> or <code>-T/--test-file</code>
 option. This option conflicts with the option <code>-m/--source-map</code> unless <code>-o/--output</code> is
-also specified.
+also specified. This option conflicts with the <code>-a/--ast</code> option.
 </p>
 
 <p>The CLI will exit with the code:</p>
@@ -411,6 +417,9 @@ <h3 id="generating-a-parser-javascript-api">JavaScript API</h3>
         with an embedded source map as a <code>data:</code> URI.  This option
         leads to a larger output string, but is the easiest to integrate with
         developer tooling.</li>
+        <li><code>"ast"</code> - return the internal AST of the grammar as a JSON
+          string. Useful for plugin authors to explore internals of PeggyJs and
+          for automation.</li>
     </ul>
     <p>(default: <code>"parser"</code>)</p>
     <blockquote>
@@ -426,6 +435,11 @@ <h3 id="generating-a-parser-javascript-api">JavaScript API</h3>
     </blockquote>
   </dd>
 
+  <blockquote>
+  <p><strong>Note</strong>: because of bug <a href="https://github.com/mozilla/source-map/issues/444">source-map/444</a> you should also set <code>grammarSource</code> to
+  a not-empty string if you set this value to <code>"source-and-map"</code></p>
+  </blockquote></dd>
+
   <dt><code>plugins</code></dt>
   <dd>Plugins to use. See the <a href="#plugins-api">Plugins API</a> section.</dd>
 

lib/compiler/index.js

@@ -133,6 +133,9 @@ const compiler = {
 `;
       }
 
+      case "ast":
+        return ast;
+
       default:
         throw new Error("Invalid output format: " + options.output + ".");
     }

lib/peg.d.ts

@@ -1079,7 +1079,7 @@ export type SourceOutputs =
   "source-with-inline-map";
 
 /** Base options for all source-generating formats. */
-interface SourceOptionsBase<Output extends SourceOutputs>
+interface SourceOptionsBase<Output>
   extends BuildOptionsBase {
   /**
    * If set to `"parser"`, the method will return generated parser object;
@@ -1246,5 +1246,26 @@ export function generate(
   options: SourceBuildOptions<SourceOutputs>
 ): string | SourceNode;
 
+/**
+ * Returns the generated AST for the grammar. Unlike result of the
+ * `peggy.compiler.compile(...)` an AST returned by this method is augmented
+ * with data from passes. In other words, the compiler gives you the raw AST,
+ * and this method provides the final AST after all optimizations and
+ * transformations.
+ *
+ * @param grammar String in the format described by the meta-grammar in the
+ *        `parser.pegjs` file
+ * @param options Options that allow you to customize returned AST
+ *
+ * @throws {SyntaxError}  If the grammar contains a syntax error, for example,
+ *         an unclosed brace
+ * @throws {GrammarError} If the grammar contains a semantic error, for example,
+ *         duplicated labels
+ */
+export function generate(
+  grammar: string,
+  options: SourceOptionsBase<"ast">
+): ast.Grammar;
+
 // Export all exported stuff under a global variable PEG in non-module environments
 export as namespace PEG;

test/api/pegjs-api.spec.js

@@ -10,6 +10,8 @@ exports.peggyVersion = function peggyVersion() {
   return peg.VERSION;
 };
 
+chai.use(require("chai-like"));
+
 beforeEach(() => {
   // In the browser, initialize SourceMapConsumer's wasm bits.
   // This is *async*, so make sure to return the promise to make
@@ -173,6 +175,15 @@ describe("Peggy API", () => {
           expect(eval(source).parse("a")).to.equal("a");
         });
       });
+
+      describe("when |output| is set to |\"ast\"|", () => {
+        it("returns generated parser AST", () => {
+          const ast = peg.generate(grammar, { output: "ast" });
+
+          expect(ast).to.be.an("object");
+          expect(ast).to.be.like(peg.parser.parse(grammar));
+        });
+      });
     });
 
     // The |format|, |exportVars|, and |dependencies| options are not tested

test/cli/run.spec.ts

@@ -334,6 +334,8 @@ Options:
                                    This option conflicts with the \`-t/--test\`
                                    and \`-T/--test-file\` options unless
                                    \`-o/--output\` is also specified
+  -a, --ast                        Output a grammar AST instead of a parser
+                                   code (default: false)
   -S, --start-rule <rule>          When testing, use the given rule as the
                                    start rule.  If this rule is not in the
                                    allowed start rules, it will be added.
@@ -611,6 +613,43 @@ Options:
     });
   });
 
+  describe("handles ast output", () => {
+    it("conflicts with --test/--test-file", async() => {
+      await exec({
+        args: ["--ast", "--test", "1"],
+        stdin: 'foo = "1"',
+        exitCode: 1,
+        expected: "CommanderError: error: option '-a, --ast' cannot be used with option '-t, --test <text>'",
+      });
+      await exec({
+        args: ["--ast", "--test-file", "file"],
+        stdin: 'foo = "1"',
+        exitCode: 1,
+        expected: "CommanderError: error: option '-a, --ast' cannot be used with option '-t, --test <text>'",
+      });
+    });
+
+    it("produces AST", async() => {
+      const output = await exec({
+        args: ["--ast"],
+        stdin: 'foo = "1"',
+        exitCode: 0,
+      });
+
+      expect(output).toBeInstanceOf(String);
+      expect(() => JSON.parse(output)).toMatchObject({
+        type: "grammar",
+        rules: [{
+          type: "rule",
+          topLevelInitializer: null,
+          initializer: null,
+          name: "foo",
+          expression: {},
+        }],
+      });
+    });
+  });
+
   it("doesn't fail with optimize", async() => {
     await exec({
       args: ["--optimize", "anything"],

test/types/peg.test-d.ts

@@ -104,6 +104,9 @@ describe("peg.d.ts", () => {
 
     const p2 = peggy.generate(src, { output: "source", grammarSource: { foo: "src" } });
     expectType<string>(p2);
+
+    const p3 = peggy.generate(src, { output: "ast", grammarSource: { foo: "src" } });
+    expectType<peggy.ast.Grammar>(p3);
   });
 
   it("generates a source map", () => {