Add new output type "ast"

Author: Mingun

CHANGELOG.md

@@ -23,7 +23,10 @@ Released: TBD
 - New CLI [@hildjj](https://github.com/peggyjs/peggy/pull/167)
   - Backward compatible with the previous
   - New -t/--test and -T/--testfile flags to directly test the generated grammar
+  - New -a/--ast flag flags to output a grammar AST for exploration
 - Check allowedStartRules for validity [@hildjj](https://github.com/peggyjs/peggy/pull/175)
+- New output type `ast` to get an internal grammar AST for investigation (can be
+  useful for plugin writers) [@Mingun](https://github.com/peggyjs/peggy/pull/206)
 
 ### Bug fixes
 

README.md

@@ -103,6 +103,10 @@ You can tweak the generated parser with several options:
 - `--allowed-start-rules <rules>` — comma-separated list of rules the parser
   will be allowed to start parsing from (default: only the first rule in the
   grammar)
+- `-a`, `--ast` — outputting an internal AST representation of the grammar after
+  all optimizations instead of the parser source code. Useful for plugin authors
+  to see how their plugin changes the AST. This option cannot be mixed with the
+  `-t/--test` and `-T/--test-file` options
 - `--cache` — makes the parser cache results, avoiding exponential parsing
   time in pathological cases but making the parser slower
 - `-d`, `--dependency <[name:]module>` — makes the parser require a specified
@@ -117,14 +121,14 @@ You can tweak the generated parser with several options:
   or commonjs module format, as an object) to pass to `peg.generate`
 - `--format <format>` — format of the generated parser: `amd`, `es`, `commonjs`,
   `globals`, `umd` (default: `commonjs`)
-- `-o`, `--output <file>` - file to send output to.  Defaults to input file
+- `-o`, `--output <file>` — file to send output to.  Defaults to input file
   name with extension changed to `.js`, or stdout if no input file is given.
 - `--plugin <module>` — makes Peggy use a specified plugin (can be specified
   multiple times)
-- `-t`, `--test <text>` — Test the parser with the given text, outputting the
+- `-t`, `--test <text>` — test the parser with the given text, outputting the
   result of running the parser against this input.
   If the input to be tested is not parsed, the CLI will exit with code 2
-- `-T`, `--test-file <filename>` — Test the parser with the contents of the
+- `-T`, `--test-file <filename>` — test the parser with the contents of the
   given file, outputting the result of running the parser against this input.
   If the input to be tested is not parsed, the CLI will exit with code 2
 - `--source-map` — generate a source map file with an optionally specified name
@@ -156,7 +160,7 @@ module.exports = {
 
 You can test generated parser immediately if you specify the `-t/--test` or `-T/--test-file`
 option. This option conflicts with the option `-m/--source-map` unless `-o/--output` is
-also specified.
+also specified. This option conflicts with the `-a/--ast` option.
 
 The CLI will exit with the code:
 - `0` if all was success
@@ -260,7 +264,8 @@ object to `peg.generate`. The following options are supported:
   object; if set to `"source"`, it will return parser source code as a string.
   If set to `"source-and-map"`, it will return a [`SourceNode`] object; you can
   get source code by calling `toString()` method or source code and mapping by
-  calling `toStringWithSourceMap()` method, see the [`SourceNode`] documentation
+  calling `toStringWithSourceMap()` method, see the [`SourceNode`] documentation.
+  If set to `"ast"` then an internal grammar AST representation will be returned
   (default: `"parser"`)
 
   > **Note**: because of bug [source-map/444] you should also set `grammarSource` to

bin/peggy.js

@@ -3014,6 +3014,11 @@ const cliOptions = program
     "-m, --source-map [mapfile]",
     "Generate a source map. If name is not specified, the source map will be named \"<input_file>.map\" if input is a file and \"source.map\" if input is a standard input. This option conflicts with the `-t/--test` and `-T/--test-file` options unless `-o/--output` is also specified"
   )
+  .option(
+    "-a, --ast",
+    "Output a grammar AST instead of a parser code",
+    false
+  )
   .option(
     "-t, --test <text>",
     "Test the parser with the given text, outputting the result of running the parser instead of the parser itself. If the input to be tested is not parsed, the CLI will exit with code 2"
@@ -3057,6 +3062,7 @@ const PARSER_DEFAULTS = {
 
 // Default values for CLI arguments
 const PROG_DEFAULTS = {
+  ast: false,
   input: undefined,
   output: undefined,
   sourceMap: undefined,
@@ -3148,6 +3154,16 @@ if (options.exportVar !== undefined) {
   }
 }
 
+if (progOptions.ast && progOptions.sourceMap) {
+  abort("Can't use the -a/--ast option with the -m/--source-map option.");
+}
+if (progOptions.ast && progOptions.test) {
+  abort("Can't use the -a/--ast option with the -t/--test option.");
+}
+if (progOptions.ast && progOptions.testFile) {
+  abort("Can't use the -a/--ast option with the -T/--test-file option.");
+}
+
 verbose = progOptions.verbose;
 let inputFile = progOptions.input;
 if (verbose) {
@@ -3184,6 +3200,9 @@ if (progOptions.test && progOptions.testFile) {
   abort("The -t/--test and -T/--test-file options are mutually exclusive.");
 }
 
+if (progOptions.ast) {
+  options.output = "ast";
+} else
 // If CLI parameter was defined, enable source map generation
 if (progOptions.sourceMap !== undefined) {
   options.output = "source-and-map";
@@ -3254,6 +3273,10 @@ readStream(inputStream, input => {
     process.exit(1);
   }
 
+  if (progOptions.ast) {
+    source = JSON.stringify(source, null, 2);
+  }
+
   // If there is a valid outputFile, write the parser to it.  Otherwise,
   // if no test and no outputFile, write to stdout.
   let outputStream = null;

bin/peggy.mjs

@@ -132,6 +132,11 @@ const cliOptions = program
     "-m, --source-map [mapfile]",
     "Generate a source map. If name is not specified, the source map will be named \"<input_file>.map\" if input is a file and \"source.map\" if input is a standard input. This option conflicts with the `-t/--test` and `-T/--test-file` options unless `-o/--output` is also specified"
   )
+  .option(
+    "-a, --ast",
+    "Output a grammar AST instead of a parser code",
+    false
+  )
   .option(
     "-t, --test <text>",
     "Test the parser with the given text, outputting the result of running the parser instead of the parser itself. If the input to be tested is not parsed, the CLI will exit with code 2"
@@ -175,6 +180,7 @@ const PARSER_DEFAULTS = {
 
 // Default values for CLI arguments
 const PROG_DEFAULTS = {
+  ast: false,
   input: undefined,
   output: undefined,
   sourceMap: undefined,
@@ -266,6 +272,16 @@ if (options.exportVar !== undefined) {
   }
 }
 
+if (progOptions.ast && progOptions.sourceMap) {
+  abort("Can't use the -a/--ast option with the -m/--source-map option.");
+}
+if (progOptions.ast && progOptions.test) {
+  abort("Can't use the -a/--ast option with the -t/--test option.");
+}
+if (progOptions.ast && progOptions.testFile) {
+  abort("Can't use the -a/--ast option with the -T/--test-file option.");
+}
+
 verbose = progOptions.verbose;
 let inputFile = progOptions.input;
 if (verbose) {
@@ -302,6 +318,9 @@ if (progOptions.test && progOptions.testFile) {
   abort("The -t/--test and -T/--test-file options are mutually exclusive.");
 }
 
+if (progOptions.ast) {
+  options.output = "ast";
+} else
 // If CLI parameter was defined, enable source map generation
 if (progOptions.sourceMap !== undefined) {
   options.output = "source-and-map";
@@ -372,6 +391,10 @@ readStream(inputStream, input => {
     process.exit(1);
   }
 
+  if (progOptions.ast) {
+    source = JSON.stringify(source, null, 2);
+  }
+
   // If there is a valid outputFile, write the parser to it.  Otherwise,
   // if no test and no outputFile, write to stdout.
   let outputStream = null;

docs/documentation.html

@@ -150,6 +150,12 @@ <h3 id="generating-a-parser-command-line">Command Line</h3>
   <dd>Comma-separated list of rules the parser will be allowed to start parsing
   from (default: only the first rule in the grammar).</dd>
 
+  <dt><code>-a</code>, <code>--ast</code></dt>
+  <dd>Outputting an internal AST representation of the grammar after
+  all optimizations instead of the parser source code. Useful for plugin authors
+  to see how their plugin changes the AST. This option cannot be mixed with the
+  <code>-t/--test</code> and <code>-T/--test-file</code> options.</dd>
+
   <dt><code>--cache</code></dt>
   <dd>Makes the parser cache results, avoiding exponential parsing time in
   pathological cases but making the parser slower.</dd>
@@ -190,21 +196,21 @@ <h3 id="generating-a-parser-command-line">Command Line</h3>
   <dt><code>-t</code>, <code>--test &lt;text&gt;</code></dt>
   <dd>Test the parser with the given text, outputting the result of running
   the parser against this input.
-  If the input to be tested is not parsed, the CLI will exit with code 2</dd>
+  If the input to be tested is not parsed, the CLI will exit with code 2.</dd>
 
   <dt><code>-T</code>, <code>--test-file &lt;text&gt;</code></dt>
   <dd>Test the parser with the contents of the given file, outputting the
   result of running the parser against this input.
-  If the input to be tested is not parsed, the CLI will exit with code 2</dd>
+  If the input to be tested is not parsed, the CLI will exit with code 2.</dd>
 
   <dt><code>--trace</code></dt>
   <dd>Makes the parser trace its progress.</dd>
 
   <dt><code>-v</code>, <code>--version</code></dt>
-  <dd>Output the version number</dd>
+  <dd>Output the version number.</dd>
 
   <dt><code>-h</code>, <code>--help</code></dt>
-  <dd>Display help for the command</dd>
+  <dd>Display help for the command.</dd>
 
 </dl>
 
@@ -232,7 +238,7 @@ <h3 id="generating-a-parser-command-line">Command Line</h3>
 <p>
 You can test generated parser immediately if you specify the <code>-t/--test</code> or <code>-T/--test-file</code>
 option. This option conflicts with the option <code>-m/--source-map</code> unless <code>-o/--output</code> is
-also specified.
+also specified. This option conflicts with the <code>-a/--ast</code> option.
 </p>
 
 <p>The CLI will exit with the code:</p>
@@ -358,7 +364,8 @@ <h3 id="generating-a-parser-javascript-api">JavaScript API</h3>
   a string (default: <code>"parser"</code>).
   If set to <code>"source-and-map"</code>, it will return a <a href="https://github.com/mozilla/source-map#sourcenode"><code>SourceNode</code></a> object; you can
   get source code by calling <code>toString()</code> method or source code and mapping by
-  calling <code>toStringWithSourceMap()</code> method, see the <a href="https://github.com/mozilla/source-map#sourcenode"><code>SourceNode</code></a> documentation
+  calling <code>toStringWithSourceMap()</code> method, see the <a href="https://github.com/mozilla/source-map#sourcenode"><code>SourceNode</code></a> documentation.
+  If set to <code>"ast"</code> then an internal grammar AST representation will be returned
   (default: <code>"parser"</code>)</p>
   <blockquote>
   <p><strong>Note</strong>: because of bug <a href="https://github.com/mozilla/source-map/issues/444">source-map/444</a> you should also set <code>grammarSource</code> to

lib/compiler/index.js

@@ -113,6 +113,9 @@ const compiler = {
       case "source-and-map":
         return ast.code;
 
+      case "ast":
+        return ast;
+
       default:
         throw new Error("Invalid output format: " + options.output + ".");
     }

lib/peg.d.ts

@@ -1082,7 +1082,7 @@ export interface ParserBuildOptions extends BuildOptionsBase {
 export type SourceOutputs = "source" | "source-and-map";
 
 /** Base options for all source-generating formats. */
-interface SourceOptionsBase<Output extends SourceOutputs>
+interface SourceOptionsBase<Output>
   extends BuildOptionsBase {
   /**
    * If set to `"parser"`, the method will return generated parser object;
@@ -1225,5 +1225,26 @@ export function generate(
   options: SourceBuildOptions<SourceOutputs>
 ): string | SourceNode;
 
+/**
+ * Returns the generated AST for the grammar. Unlike result of the
+ * `peggy.compiler.compile(...)` an AST returned by this method is augmented
+ * with data from passes. In other words, the compiler gives you the raw AST,
+ * and this method provides the final AST after all optimizations and
+ * transformations.
+ *
+ * @param grammar String in the format described by the meta-grammar in the
+ *        `parser.pegjs` file
+ * @param options Options that allow you to customize returned AST
+ *
+ * @throws {SyntaxError}  If the grammar contains a syntax error, for example,
+ *         an unclosed brace
+ * @throws {GrammarError} If the grammar contains a semantic error, for example,
+ *         duplicated labels
+ */
+export function generate(
+  grammar: string,
+  options: SourceOptionsBase<"ast">
+): ast.Grammar;
+
 // Export all exported stuff under a global variable PEG in non-module environments
 export as namespace PEG;

test/api/pegjs-api.spec.js

@@ -6,6 +6,8 @@ const sinon = require("sinon");
 const pkg = require("../../package.json");
 const { SourceMapConsumer } = require("source-map");
 
+chai.use(require("chai-like"));
+
 beforeEach(() => {
   // In the browser, initialize SourceMapConsumer's wasm bits.
   // This is *async*, so make sure to return the promise to make
@@ -169,6 +171,15 @@ describe("Peggy API", () => {
           expect(eval(source).parse("a")).to.equal("a");
         });
       });
+
+      describe("when |output| is set to |\"ast\"|", () => {
+        it("returns generated parser AST", () => {
+          const ast = peg.generate(grammar, { output: "ast" });
+
+          expect(ast).to.be.an("object");
+          expect(ast).to.be.like(peg.parser.parse(grammar));
+        });
+      });
     });
 
     // The |format|, |exportVars|, and |dependencies| options are not tested

test/cli/run.spec.ts

@@ -170,6 +170,8 @@ Options:
                                    This option conflicts with the \`-t/--test\`
                                    and \`-T/--test-file\` options unless
                                    \`-o/--output\` is also specified
+  -a, --ast                        Output a grammar AST instead of a parser
+                                   code (default: false)
   -t, --test <text>                Test the parser with the given text,
                                    outputting the result of running the parser
                                    instead of the parser itself. If the input

test/types/peg.test-d.ts

@@ -57,6 +57,9 @@ describe("peg.d.ts", () => {
 
     const p2 = peggy.generate(src, { output: "source", grammarSource: { foo: "src" } });
     expectType<string>(p2);
+
+    const p3 = peggy.generate(src, { output: "ast", grammarSource: { foo: "src" } });
+    expectType<peggy.ast.Grammar>(p3);
   });
 
   it("generates a source map", () => {