[1.8>1.9] [MERGE #4628 @boingoing] Defer parse treats yield as keyword

Merge pull request #4628 from boingoing:DeferParseYieldToken

When we're defer parsing a function, we construct an enclosing function context to use as a parent function. That dummy parent function does not initialize the function flags, though, so they're random. When we're parsing a deferred lambda function, we look at those flags in the dummy parent function to see if it was a generator function to know if we should treat yield as an identifier or not. Because the flags are random, we sometimes throw a syntax error here thinking yield should be treated as a keyword. If we throw a syntax error upon defer-parse, we will hit an assert.

Fix by storing a bit of state in the ScopeInfo which indicates if the enclosing function is a generator or is an async function.

Fixes:
https://microsoft.visualstudio.com/web/wi.aspx?id=14502906

Author: boingoing

Diff for: lib/Parser/Parse.cpp

@@ -11611,6 +11611,10 @@ ParseNodePtr Parser::Parse(LPCUTF8 pszSrc, size_t offset, size_t length, charcou
             m_currentNodeFunc->sxFnc.SetStrictMode(!!this->m_fUseStrictMode);
 
             this->RestoreScopeInfo(scopeInfo);
+
+            m_currentNodeFunc->sxFnc.ClearFlags();
+            m_currentNodeFunc->sxFnc.SetIsGenerator(scopeInfo->IsGeneratorFunctionBody());
+            m_currentNodeFunc->sxFnc.SetIsAsync(scopeInfo->IsAsyncFunctionBody());
         }
     }
 

Diff for: lib/Runtime/ByteCode/ScopeInfo.cpp

@@ -71,7 +71,12 @@ namespace Js
         scopeInfo->mustInstantiate = scope->GetMustInstantiate();
         scopeInfo->isCached = (scope->GetFunc()->GetBodyScope() == scope) && scope->GetFunc()->GetHasCachedScope();
         scopeInfo->hasLocalInClosure = scope->GetHasOwnLocalInClosure();
-        
+
+        if (scope->GetScopeType() == ScopeType_FunctionBody)
+        {
+            scopeInfo->isGeneratorFunctionBody = scope->GetFunc()->byteCodeFunction->GetFunctionInfo()->IsGenerator();
+            scopeInfo->isAsyncFunctionBody = scope->GetFunc()->byteCodeFunction->GetFunctionInfo()->IsAsync();
+        }
 
         TRACE_BYTECODE(_u("\nSave ScopeInfo: %s #symbols: %d %s\n"),
             scope->GetFunc()->name, count,

Diff for: lib/Runtime/ByteCode/ScopeInfo.h

@@ -45,6 +45,8 @@ namespace Js {
         Field(BYTE) isCached : 1;              // indicates that local vars and functions are cached across invocations
         Field(BYTE) areNamesCached : 1;
         Field(BYTE) hasLocalInClosure : 1;
+        Field(BYTE) isGeneratorFunctionBody : 1;
+        Field(BYTE) isAsyncFunctionBody : 1;
 
         FieldNoBarrier(Scope *) scope;
         Field(::ScopeType) scopeType;
@@ -54,7 +56,7 @@ namespace Js {
 
     private:
         ScopeInfo(FunctionInfo * function, int symbolCount)
-            : functionInfo(function), /*funcExprScopeInfo(nullptr), paramScopeInfo(nullptr),*/ symbolCount(symbolCount), parent(nullptr), scope(nullptr), areNamesCached(false), hasLocalInClosure(false)/*, parentOnly(false)*/
+            : functionInfo(function), /*funcExprScopeInfo(nullptr), paramScopeInfo(nullptr),*/ symbolCount(symbolCount), parent(nullptr), scope(nullptr), areNamesCached(false), hasLocalInClosure(false), isGeneratorFunctionBody(false), isAsyncFunctionBody(false)/*, parentOnly(false)*/
         {
         }
 
@@ -259,6 +261,16 @@ namespace Js {
             return hasLocalInClosure;
         }
 
+        bool IsGeneratorFunctionBody() const
+        {
+            return this->isGeneratorFunctionBody;
+        }
+
+        bool IsAsyncFunctionBody() const
+        {
+            return this->isAsyncFunctionBody;
+        }
+
         static void SaveEnclosingScopeInfo(ByteCodeGenerator* byteCodeGenerator, /*FuncInfo* parentFunc,*/ FuncInfo* func);
 
         void EnsurePidTracking(ScriptContext* scriptContext);

Diff for: test/es6/DeferParseLambda.js

@@ -110,6 +110,54 @@ var tests = [
             `);
         }
     },
+    {
+        name: "Global functions using 'yield' as identifier",
+        body: function () {
+            WScript.LoadScript(`
+                var a = async (yield) => { yield };
+                assert.isTrue(a() instanceof Promise, "Async lambda with yield as a formal parameter name");
+
+                function b(yield) {
+                    return yield;
+                }
+                assert.areEqual('b', b('b'), "Function with yield as a formal parameter name");
+
+                var c = async (yield) => yield;
+                assert.isTrue(c() instanceof Promise, "Async lambda with yield as a formal parameter name and compact body");
+                
+                async function d(yield) {
+                    return yield;
+                }
+                assert.isTrue(d() instanceof Promise, "Async lambda with yield as a formal parameter name and compact body");
+            `);
+        }
+    },
+    {
+        name: "Nested functions using 'yield' as identifier",
+        body: function () {
+            var a = async (yield) => { yield };
+            assert.isTrue(a() instanceof Promise, "Async lambda with yield as a formal parameter name");
+
+            function b(yield) {
+                return yield;
+            }
+            assert.areEqual('b', b('b'), "Function with yield as a formal parameter name");
+
+            var c = async (yield) => yield;
+            assert.isTrue(c() instanceof Promise, "Async lambda with yield as a formal parameter name and compact body");
+
+            async function d(yield) {
+                return yield;
+            }
+            assert.isTrue(d() instanceof Promise, "Async lambda with yield as a formal parameter name and compact body");
+            
+            var e = async (a = yield) => { yield };
+            assert.isTrue(e() instanceof Promise, "Async lambda with yield in a default argument");
+            
+            var f = async (a = yield) => yield;
+            assert.isTrue(f() instanceof Promise, "Async lambda with compact body and yield in a default argument");
+        }
+    },
 ]
 
 testRunner.runTests(tests, { verbose: WScript.Arguments[0] != "summary" });

Diff for: test/es6/rlexe.xml

@@ -831,6 +831,13 @@
       <tags>exclude_arm</tags>
     </default>
   </test>
+  <test>
+    <default>
+      <files>generators-syntax.js</files>
+      <compile-flags>-ES6Generators -ES6Classes -ES6DefaultArgs -force:deferparse -args summary -endargs</compile-flags>
+      <tags>exclude_arm</tags>
+    </default>
+  </test>
   <test>
     <default>
       <files>generators-deferparse.js</files>