More grammar changes (#36)

* more grammar fixes

* regenerate parser

* improve parsing script

* regenerate parser

* deal with last known failure

* test with more projects

* regenerate parser

* regenerate the parser

* add tests for queries

* improve error messages

* remove unnecessary step

.github/workflows/ci.yml

@@ -63,3 +63,6 @@ jobs:
 
       - name: Run the parsing tests
         run: npm run test-parsing
+
+      - name: Run the query tests
+        run: npm run test-queries

grammar.js

@@ -147,6 +147,7 @@ module.exports = grammar({
 
         TestDecl: ($) =>
             seq(
+                optional($.doc_comment),
                 keyword("test", $),
                 optional(choice($.STRINGLITERALSINGLE, $.IDENTIFIER)),
                 $.Block
@@ -205,14 +206,22 @@ module.exports = grammar({
             ),
 
         ContainerField: ($) =>
-            prec(PREC.curly,
+            prec(PREC.assign,
                 seq(
                     optional($.doc_comment),
                     optional(keyword("comptime", $)),
                     choice(
-                        seq(field("field_member", $.IDENTIFIER), optional(seq(COLON, $._TypeExpr))),
+                        seq(
+                          field("field_member", choice(
+                            $.IDENTIFIER,
+                            alias($.BuildinTypeExpr, $.IDENTIFIER)
+                          )),
+                          COLON,
+                          optional($.ArrayTypeStart),
+                          $._TypeExpr,
+                        ),
                         // NOTE: $._TypeExpr already include fn
-                        seq(optional(seq(field("field_member", $.IDENTIFIER), COLON)), $._TypeExpr)
+                        $._TypeExpr
                     ),
                     optional($.ByteAlign),
                     optional(seq(EQUAL, $._Expr))
@@ -549,7 +558,13 @@ module.exports = grammar({
                 seq(
                     optional($.doc_comment),
                     optional(keyword(choice("noalias", "comptime"), $)),
-                    optional(seq(field("parameter", $.IDENTIFIER), COLON)),
+                    optional(seq(
+                      field("parameter", choice(
+                        $.IDENTIFIER,
+                        alias($.BuildinTypeExpr, $.IDENTIFIER)
+                      )),
+                      COLON
+                    )),
                     $.ParamType
                 ),
                 DOT3
@@ -724,6 +739,7 @@ module.exports = grammar({
         */
         _SentinelTerminatedExpr: ($) => choice(
           seq(...blockLabel($), $._Expr),
+          seq(COLON, $._Expr),
           seq($._Expr, optional(seq(COLON, $._Expr)))
         ),
 
@@ -757,8 +773,13 @@ module.exports = grammar({
                 )
             ),
 
-        ArrayTypeStart: ($) =>
-            seq(LBRACKET, $._Expr, optional(seq(COLON, $._Expr)), RBRACKET),
+        ArrayTypeStart: ($) => seq(
+          LBRACKET,
+          choice($._Expr, $.IDENTIFIER),
+          optional(DOT2),
+          optional(seq(COLON, choice($._Expr, $.IDENTIFIER))),
+          RBRACKET
+        ),
 
         // ContainerDecl specific
         _ContainerDeclAuto: ($) =>
@@ -772,7 +793,10 @@ module.exports = grammar({
 
         ContainerDeclType: ($) =>
             choice(
-                keyword("struct", $),
+                seq(
+                  keyword("struct", $),
+                  optional(seq(LPAREN, $._Expr, RPAREN))
+                ),
                 keyword("opaque", $),
                 seq(keyword("enum", $), optional(seq(LPAREN, $._Expr, RPAREN))),
                 seq(

package.json

@@ -16,7 +16,9 @@
   "scripts": {
     "test": "tree-sitter test",
     "generate": "tree-sitter generate",
-    "test-parsing": "scripts/test-parsing"
+    "test-parsing": "scripts/test-parsing",
+    "test-queries": "scripts/test-queries",
+    "parse": "tree-sitter parse"
   },
   "tree-sitter": [
     {

scripts/known-parsing-failures.txt

@@ -1,32 +0,0 @@
-zig/test/behavior/struct.zig
-zig/test/behavior/slice.zig
-zig/test/behavior/array.zig
-zig/test/behavior/pointers.zig
-zig/test/behavior/slice_sentinel_comptime.zig
-zig/test/behavior/bugs/12972.zig
-zig/test/behavior/packed_struct_explicit_backing_int.zig
-zig/lib/std/os.zig
-zig/lib/std/fs.zig
-zig/lib/std/mem.zig
-zig/lib/std/zig/c_translation.zig
-zig/lib/std/os/test.zig
-zig/lib/std/os/linux/bpf/btf.zig
-zig/lib/std/os/linux/bpf.zig
-zig/lib/std/os/windows.zig
-zig/lib/std/os/uefi/protocols/ip6_protocol.zig
-zig/lib/std/os/uefi/protocols/hii.zig
-zig/lib/std/os/uefi/protocols/simple_text_input_ex_protocol.zig
-zig/lib/std/os/uefi/protocols/simple_network_protocol.zig
-zig/lib/std/os/uefi/protocols/absolute_pointer_protocol.zig
-zig/lib/std/os/uefi/protocols/edid_override_protocol.zig
-zig/lib/std/os/uefi/tables/boot_services.zig
-zig/lib/std/unicode.zig
-zig/lib/std/Thread.zig
-zig/lib/std/json/test.zig
-zig/lib/std/macho.zig
-zig/lib/std/c/openbsd.zig
-zig/src/link/Wasm/Object.zig
-zig/src/link/Wasm/Archive.zig
-zig/src/link/MachO/Archive.zig
-zig/src/codegen/llvm/bindings.zig
-zig/src/Autodoc.zig

scripts/test-parsing

@@ -45,25 +45,40 @@ clone_repo() {
 
 # zig 0.10.1 -> https://github.com/ziglang/zig/releases/tag/0.10.1
 clone_repo ziglang zig b57081f039bd3f8f82210e8896e336e3c3a6869b
+# the following projects should be compatible with the zig version above
+# the commits were chosen through `git log -1 --until="$ZIG_RELEASE_DATE"`
+clone_repo tigerbeetledb tigerbeetle eff4158fb7f39f7557f646d7eb1bde39d8459d22
+clone_repo zigtools zls fa5828496eb8b554ec69ed0beff7a9edcb53411a
+clone_repo oven-sh bun cd5f2ab11fb33e805271b9b48711f3f592f30b0f
 
+failures_file="$this_dir/known-parsing-failures.txt"
+inexistent_failures=()
+incorrect_failures=()
 known_failures=()
-while IFS= read -r line; do
-  if [[ "$line" =~ [^[:space:]] ]]; then
-    full_path="$sources_dir/$line"
-    if [ -e "$full_path" ]; then
-      if tree-sitter parse -q "$full_path" &>/dev/null; then
-        >&2 echo "File $full_path can be parsed without errors, but it's listed in $this_dir/known-parsing-failures.txt"
-        saw_error=true
+if [ -e "$failures_file" ]; then
+  while IFS= read -r line; do
+    if [[ "$line" =~ [^[:space:]] ]]; then
+      full_path="$sources_dir/$line"
+      if [ -e "$full_path" ]; then
+        if tree-sitter parse -q "$full_path" &>/dev/null; then
+          incorrect_failures+=("$line")
+        else
+          known_failures+=("$sources_dir/$line")
+        fi
       else
-        known_failures+=("$sources_dir/$line")
+        inexistent_failures+=("$line")
       fi
-    else
-      >&2 echo "File $full_path (listed in $this_dir/known-parsing-failures.txt) does not exist"
-      saw_error=true
     fi
-  fi
-done < "$this_dir/known-parsing-failures.txt"
-if [ "${saw_error:-}" ]; then
+  done < "$failures_file"
+fi
+if [ ${#incorrect_failures[*]} -gt 0 ]; then
+  >&2 echo "Some files can be parsed without errors, but they are listed in $failures_file. Please remove those lines from $failures_file and try again:"
+  >&2 printf -- "- %s\n" "${incorrect_failures[@]}"
+  exit 1
+fi
+if [ ${#inexistent_failures[*]} -gt 0 ]; then
+  >&2 echo "The following files could not be found in $sources_dir. Please remove those lines from $failures_file and try again:"
+  >&2 printf -- "- %s\n" "${inexistent_failures[@]}"
   exit 1
 fi
 
@@ -80,7 +95,7 @@ done < <(
     -not -path "$sources_dir/zig/test/cases/compile_errors/**"
 )
 if [ ${#parse_args[*]} -eq 0 ]; then
-  >&2 echo "No files found to test"
+  >&2 echo "No files found to parse"
   exit 1
 fi
 

scripts/test-queries

@@ -0,0 +1,24 @@
+#!/bin/env bash
+
+set -Eeu -o pipefail
+shopt -s inherit_errexit
+
+this_file="${BASH_SOURCE[0]}"
+if [ "${this_file::1}" != '/' ]; then
+  this_file="$PWD/$this_file"
+fi
+
+this_dir="${this_file%/*}"
+project_root="${this_dir%/*}"
+queries_dir="$project_root/queries"
+
+while IFS= read -r query; do
+  # this tests if the queries can be compiled, not if they are producing the
+  # expected output
+  if ! >/dev/null tree-sitter query --test "$query" <(echo ""); then
+    echo "Query $query failed!"
+    exit_code=1
+  fi
+done < <(find "$queries_dir" -name "*.scm")
+
+exit "${exit_code:-0}"