Improve lexing and parsing of invalid annotations (again)

.gitattributes

@@ -18,6 +18,8 @@ docs/demo/third_party/codemirror/codemirror.js binary
 test/parse/bad-crlf.txt binary
 test/parse/bad-string-eof.txt binary
 test/regress/regress-31.txt binary
+test/regress/bad-annotation* binary
+test/regress/unterminated-annotation* binary
 
 # Highlight tests like .wast files when displayed on GitHub.
 test/**/*.txt linguist-language=WebAssembly

include/wabt/wast-lexer.h

@@ -95,7 +95,7 @@ class WastLexer {
   Token GetInfToken();
   Token GetNanToken();
   Token GetNameEqNumToken(std::string_view name, TokenType);
-  Token GetIdToken();
+  Token GetIdChars();
   Token GetKeywordToken();
   Token GetReservedToken();
 

src/wast-lexer.cc

@@ -70,7 +70,7 @@ Token WastLexer::GetToken() {
           }
           return BareToken(TokenType::Eof);
         } else if (MatchString("(@")) {
-          GetIdToken();
+          GetIdChars();
           // offset=2 to skip the "(@" prefix
           return TextToken(TokenType::LparAnn, 2);
         } else {
@@ -151,7 +151,7 @@ Token WastLexer::GetToken() {
         return GetNumberToken(TokenType::Nat);
 
       case '$':
-        return GetIdToken();
+        return GetIdChars();  // Initial $ is idchar, so this produces id token
 
       case 'a':
         return GetNameEqNumToken("align=", TokenType::AlignEqNat);
@@ -187,6 +187,16 @@ Location WastLexer::GetLocation() {
 }
 
 std::string_view WastLexer::GetText(size_t offset) {
+  // Bounds checks are necessary because token_start may have been moved
+  // (e.g. if GetStringToken found a newline and reset token_start to
+  // point at it).
+
+  if (token_start_ + offset >= buffer_end_)
+    return {};
+
+  if (cursor_ <= token_start_ + offset)
+    return {};
+
   return std::string_view(token_start_ + offset,
                           (cursor_ - token_start_) - offset);
 }
@@ -577,8 +587,7 @@ Token WastLexer::GetNameEqNumToken(std::string_view name,
   return GetKeywordToken();
 }
 
-Token WastLexer::GetIdToken() {
-  ReadChar();
+Token WastLexer::GetIdChars() {
   if (ReadReservedChars() == ReservedChars::Id) {
     return TextToken(TokenType::Var);
   }

src/wast-parser.cc

@@ -623,6 +623,11 @@ TokenType WastParser::Peek(size_t n) {
             indent--;
             break;
 
+          case TokenType::Eof:
+            indent = 0;
+            Error(cur.loc, "unterminated annotation");
+            break;
+
           default:
             break;
         }
@@ -1956,10 +1961,7 @@ Result WastParser::ParseCodeMetadataAnnotation(ExprList* exprs) {
   CHECK_RESULT(ParseQuotedText(&data_text, false));
   std::vector<uint8_t> data(data_text.begin(), data_text.end());
   exprs->push_back(std::make_unique<CodeMetadataExpr>(name, std::move(data)));
-  TokenType rpar = Peek();
-  WABT_USE(rpar);
-  assert(rpar == TokenType::Rpar);
-  Consume();
+  EXPECT(Rpar);
   return Result::Ok;
 }
 

test/regress/bad-annotation.txt

@@ -3,10 +3,10 @@
 (@"annotation
 
 (;; STDERR ;;;
-out/test/regress/bad-annotation.txt:3:1: error: annotations not enabled: "annotation
+out/test/regress/bad-annotation.txt:3:14: error: newline in string
 (@"annotation
-^^^^^^^^^^^^^
-out/test/regress/bad-annotation.txt:3:1: error: unexpected token "Invalid", expected a module field or a module.
-(@"annotation
-^^^^^^^^^^^^^
+             ^
+out/test/regress/bad-annotation.txt:4:1: error: newline in string
+out/test/regress/bad-annotation.txt:5:1: error: annotations not enabled: 
+out/test/regress/bad-annotation.txt:5:1: error: unexpected token "Invalid", expected a module field or a module.
 ;;; STDERR ;;)

test/regress/bad-annotation2.txt

@@ -0,0 +1,11 @@
+;;; TOOL: wat2wasm
+;;; ERROR: 1
+(module (memory $mem 1)
+   (@_memory" (memory $mem))
+(;; STDERR ;;;
+out/test/regress/bad-annotation2.txt:4:29: error: newline in string
+   (@_memory" (memory $mem))
+                            ^
+out/test/regress/bad-annotation2.txt:5:1: error: annotations not enabled: 
+out/test/regress/bad-annotation2.txt:5:1: error: unexpected token Invalid, expected ).
+;;; STDERR ;;)

test/regress/unterminated-annotation.txt

@@ -0,0 +1,8 @@
+;;; TOOL: wat2wasm
+;;; ARGS: --enable-annotations --enable-code-metadata
+;;; ERROR: 1
+(module
+   (func (@metadata.code.data "this is a test"
+(;; STDERR ;;;
+out/test/regress/unterminated-annotation.txt:6:1: error: unexpected token EOF, expected ).
+;;; STDERR ;;)

test/regress/unterminated-annotation2.txt

@@ -0,0 +1,9 @@
+;;; TOOL: wat2wasm
+;;; ARGS: --enable-annotations
+;;; ERROR: 1
+(module
+   (func (@hello "this is a test"
+(;; STDERR ;;;
+out/test/regress/unterminated-annotation2.txt:6:1: error: unterminated annotation
+out/test/regress/unterminated-annotation2.txt:6:1: error: unexpected token EOF, expected ).
+;;; STDERR ;;)