Skip to content

Add support for generators and yield #105

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
erikd merged 1 commit into from
Sep 24, 2019
Merged

Add support for generators and yield #105

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions src/Language/JavaScript/Parser/AST.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,7 @@ data JSStatement
| JSForOf !JSAnnot !JSAnnot !JSExpression !JSBinOp !JSExpression !JSAnnot !JSStatement -- ^for,lb,expr,in,expr,rb,stmt
| JSForVarOf !JSAnnot !JSAnnot !JSAnnot !JSExpression !JSBinOp !JSExpression !JSAnnot !JSStatement -- ^for,lb,var,vardecl,in,expr,rb,stmt
| JSFunction !JSAnnot !JSIdent !JSAnnot !(JSCommaList JSExpression) !JSAnnot !JSBlock !JSSemi -- ^fn,name, lb,parameter list,rb,block,autosemi
| JSGenerator !JSAnnot !JSAnnot !JSIdent !JSAnnot !(JSCommaList JSExpression) !JSAnnot !JSBlock !JSSemi -- ^fn,*,name, lb,parameter list,rb,block,autosemi
| JSIf !JSAnnot !JSAnnot !JSExpression !JSAnnot !JSStatement -- ^if,(,expr,),stmt
| JSIfElse !JSAnnot !JSAnnot !JSExpression !JSAnnot !JSStatement !JSAnnot !JSStatement -- ^if,(,expr,),stmt,else,rest
| JSLabelled !JSIdent !JSAnnot !JSStatement -- ^identifier,colon,stmt
Expand Down Expand Up @@ -182,6 +183,7 @@ data JSExpression
| JSExpressionTernary !JSExpression !JSAnnot !JSExpression !JSAnnot !JSExpression -- ^cond, ?, trueval, :, falseval
| JSArrowExpression !JSArrowParameterList !JSAnnot !JSStatement -- ^parameter list,arrow,block`
| JSFunctionExpression !JSAnnot !JSIdent !JSAnnot !(JSCommaList JSExpression) !JSAnnot !JSBlock -- ^fn,name,lb, parameter list,rb,block`
| JSGeneratorExpression !JSAnnot !JSAnnot !JSIdent !JSAnnot !(JSCommaList JSExpression) !JSAnnot !JSBlock -- ^fn,*,name,lb, parameter list,rb,block`
| JSMemberDot !JSExpression !JSAnnot !JSExpression -- ^firstpart, dot, name
| JSMemberExpression !JSExpression !JSAnnot !(JSCommaList JSExpression) !JSAnnot -- expr, lb, args, rb
| JSMemberNew !JSAnnot !JSExpression !JSAnnot !(JSCommaList JSExpression) !JSAnnot -- ^new, name, lb, args, rb
Expand All @@ -192,6 +194,8 @@ data JSExpression
| JSTemplateLiteral !(Maybe JSExpression) !JSAnnot !String ![JSTemplatePart] -- ^optional tag, lquot, head, parts
| JSUnaryExpression !JSUnaryOp !JSExpression
| JSVarInitExpression !JSExpression !JSVarInitializer -- ^identifier, initializer
| JSYieldExpression !JSAnnot !(Maybe JSExpression) -- ^yield, optional expr
| JSYieldFromExpression !JSAnnot !JSAnnot !JSExpression -- ^yield, *, expr
deriving (Data, Eq, Show, Typeable)

data JSArrowParameterList
Expand Down Expand Up @@ -364,6 +368,7 @@ instance ShowStripped JSStatement where
ss (JSForOf _ _lb x1s _i x2 _rb x3) = "JSForOf " ++ ss x1s ++ " (" ++ ss x2 ++ ") (" ++ ss x3 ++ ")"
ss (JSForVarOf _ _lb _v x1 _i x2 _rb x3) = "JSForVarOf (" ++ ss x1 ++ ") (" ++ ss x2 ++ ") (" ++ ss x3 ++ ")"
ss (JSFunction _ n _lb pl _rb x3 _) = "JSFunction " ++ ssid n ++ " " ++ ss pl ++ " (" ++ ss x3 ++ ")"
ss (JSGenerator _ _ n _lb pl _rb x3 _) = "JSGenerator " ++ ssid n ++ " " ++ ss pl ++ " (" ++ ss x3 ++ ")"
ss (JSIf _ _lb x1 _rb x2) = "JSIf (" ++ ss x1 ++ ") (" ++ ss x2 ++ ")"
ss (JSIfElse _ _lb x1 _rb x2 _e x3) = "JSIfElse (" ++ ss x1 ++ ") (" ++ ss x2 ++ ") (" ++ ss x3 ++ ")"
ss (JSLabelled x1 _c x2) = "JSLabelled (" ++ ss x1 ++ ") (" ++ ss x2 ++ ")"
Expand Down Expand Up @@ -395,6 +400,7 @@ instance ShowStripped JSExpression where
ss (JSExpressionTernary x1 _q x2 _c x3) = "JSExpressionTernary (" ++ ss x1 ++ "," ++ ss x2 ++ "," ++ ss x3 ++ ")"
ss (JSArrowExpression ps _ e) = "JSArrowExpression (" ++ ss ps ++ ") => " ++ ss e
ss (JSFunctionExpression _ n _lb pl _rb x3) = "JSFunctionExpression " ++ ssid n ++ " " ++ ss pl ++ " (" ++ ss x3 ++ ")"
ss (JSGeneratorExpression _ _ n _lb pl _rb x3) = "JSGeneratorExpression " ++ ssid n ++ " " ++ ss pl ++ " (" ++ ss x3 ++ ")"
ss (JSHexInteger _ s) = "JSHexInteger " ++ singleQuote s
ss (JSOctal _ s) = "JSOctal " ++ singleQuote s
ss (JSIdentifier _ s) = "JSIdentifier " ++ singleQuote s
Expand All @@ -410,6 +416,9 @@ instance ShowStripped JSExpression where
ss (JSStringLiteral _ s) = "JSStringLiteral " ++ s
ss (JSUnaryExpression op x) = "JSUnaryExpression (" ++ ss op ++ "," ++ ss x ++ ")"
ss (JSVarInitExpression x1 x2) = "JSVarInitExpression (" ++ ss x1 ++ ") " ++ ss x2
ss (JSYieldExpression _ Nothing) = "JSYieldExpression ()"
ss (JSYieldExpression _ (Just x)) = "JSYieldExpression (" ++ ss x ++ ")"
ss (JSYieldFromExpression _ _ x) = "JSYieldFromExpression (" ++ ss x ++ ")"
ss (JSSpreadExpression _ x1) = "JSSpreadExpression (" ++ ss x1 ++ ")"
ss (JSTemplateLiteral Nothing _ s ps) = "JSTemplateLiteral (()," ++ singleQuote s ++ "," ++ ss ps ++ ")"
ss (JSTemplateLiteral (Just t) _ s ps) = "JSTemplateLiteral ((" ++ ss t ++ ")," ++ singleQuote s ++ "," ++ ss ps ++ ")"
Expand Down
47 changes: 46 additions & 1 deletion src/Language/JavaScript/Parser/Grammar7.y
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,7 @@ import qualified Language.JavaScript.Parser.AST as AST
'void' { VoidToken {} }
'while' { WhileToken {} }
'with' { WithToken {} }
'yield' { YieldToken {} }


'ident' { IdentifierToken {} }
Expand Down Expand Up @@ -484,6 +485,7 @@ PrimaryExpression : 'this' { AST.JSLiteral (mkJSAnnot $1) "thi
| Literal { $1 {- 'PrimaryExpression2' -} }
| ArrayLiteral { $1 {- 'PrimaryExpression3' -} }
| ObjectLiteral { $1 {- 'PrimaryExpression4' -} }
| GeneratorExpression { $1 }
| TemplateLiteral { mkJSTemplateLiteral Nothing $1 {- 'PrimaryExpression6' -} }
| LParen Expression RParen { AST.JSExpressionParen $1 $2 $3 }

Expand All @@ -495,6 +497,12 @@ Identifier : 'ident' { AST.JSIdentifier (mkJSAnnot $1) (tokenLiteral $1) }
| 'get' { AST.JSIdentifier (mkJSAnnot $1) "get" }
| 'set' { AST.JSIdentifier (mkJSAnnot $1) "set" }
| 'from' { AST.JSIdentifier (mkJSAnnot $1) "from" }
| 'yield' { AST.JSIdentifier (mkJSAnnot $1) "yield" }

-- Must follow Identifier; when ambiguous, `yield` as a keyword should take
-- precedence over `yield` as an identifier name.
Yield :: { AST.JSAnnot }
Yield : 'yield' { mkJSAnnot $1 }


SpreadExpression :: { AST.JSExpression }
Expand Down Expand Up @@ -858,6 +866,7 @@ ConditionalExpressionNoIn : LogicalOrExpressionNoIn { $1 {- 'ConditionalExpressi
-- LeftHandSideExpression AssignmentOperator AssignmentExpression
AssignmentExpression :: { AST.JSExpression }
AssignmentExpression : ConditionalExpression { $1 {- 'AssignmentExpression1' -} }
| YieldExpression { $1 }
| LeftHandSideExpression AssignmentOperator AssignmentExpression
{ AST.JSAssignExpression $1 $2 $3 {- 'AssignmentExpression2' -} }
| SpreadExpression { $1 }
Expand All @@ -867,6 +876,7 @@ AssignmentExpression : ConditionalExpression { $1 {- 'AssignmentExpression1' -}
-- LeftHandSideExpression AssignmentOperator AssignmentExpressionNoIn
AssignmentExpressionNoIn :: { AST.JSExpression }
AssignmentExpressionNoIn : ConditionalExpressionNoIn { $1 {- 'AssignmentExpressionNoIn1' -} }
| YieldExpression { $1 }
| LeftHandSideExpression AssignmentOperator AssignmentExpressionNoIn
{ AST.JSAssignExpression $1 $2 $3 {- 'AssignmentExpressionNoIn1' -} }

Expand Down Expand Up @@ -1193,6 +1203,38 @@ LambdaExpression : Function LParen RParen FunctionBody
| Function LParen FormalParameterList RParen FunctionBody
{ AST.JSFunctionExpression $1 AST.JSIdentNone $2 $3 $4 $5 {- 'LambdaExpression2' -} }

-- GeneratorDeclaration :
-- function * BindingIdentifier ( FormalParameters ) { GeneratorBody }
-- function * ( FormalParameters ) { GeneratorBody }
GeneratorDeclaration :: { AST.JSStatement }
GeneratorDeclaration : NamedGeneratorExpression MaybeSemi { expressionToStatement $1 $2 }

-- GeneratorExpression :
-- function * BindingIdentifieropt ( FormalParameters ) { GeneratorBody }
-- GeneratorBody :
-- FunctionBody
GeneratorExpression :: { AST.JSExpression }
GeneratorExpression : NamedGeneratorExpression { $1 }
| Function '*' LParen RParen FunctionBody
{ AST.JSGeneratorExpression $1 (mkJSAnnot $2) AST.JSIdentNone $3 AST.JSLNil $4 $5 }
| Function '*' LParen FormalParameterList RParen FunctionBody
{ AST.JSGeneratorExpression $1 (mkJSAnnot $2) AST.JSIdentNone $3 $4 $5 $6 }

NamedGeneratorExpression :: { AST.JSExpression }
NamedGeneratorExpression : Function '*' Identifier LParen RParen FunctionBody
{ AST.JSGeneratorExpression $1 (mkJSAnnot $2) (identName $3) $4 AST.JSLNil $5 $6 }
| Function '*' Identifier LParen FormalParameterList RParen FunctionBody
{ AST.JSGeneratorExpression $1 (mkJSAnnot $2) (identName $3) $4 $5 $6 $7 }

-- YieldExpression :
-- yield
-- yield [no LineTerminator here] AssignmentExpression
-- yield [no LineTerminator here] * AssignmentExpression
YieldExpression :: { AST.JSExpression }
YieldExpression : Yield { AST.JSYieldExpression $1 Nothing }
| Yield AssignmentExpression { AST.JSYieldExpression $1 (Just $2) }
| Yield '*' AssignmentExpression { AST.JSYieldFromExpression $1 (mkJSAnnot $2) $3 }


IdentifierOpt :: { AST.JSIdent }
IdentifierOpt : Identifier { identName $1 {- 'IdentifierOpt1' -} }
Expand Down Expand Up @@ -1299,7 +1341,7 @@ ImportSpecifier : IdentifierName
-- [x] LexicalDeclaration
-- [ ] HoistableDeclaration :
-- [x] FunctionDeclaration
-- [ ] GeneratorDeclaration
-- [x] GeneratorDeclaration
-- [ ] AsyncFunctionDeclaration
-- [ ] AsyncGeneratorDeclaration
-- [ ] export default HoistableDeclaration[Default]
Expand All @@ -1314,6 +1356,8 @@ ExportDeclaration : ExportClause FromClause AutoSemi
{ AST.JSExport $1 $2 {- 'ExportDeclaration3' -} }
| FunctionDeclaration AutoSemi
{ AST.JSExport $1 $2 {- 'ExportDeclaration4' -} }
| GeneratorDeclaration AutoSemi
{ AST.JSExport $1 $2 {- 'ExportDeclaration5' -} }

-- ExportClause :
-- { }
Expand Down Expand Up @@ -1364,6 +1408,7 @@ blockToStatement (AST.JSBlock a b c) s = AST.JSStatementBlock a b c s

expressionToStatement :: AST.JSExpression -> AST.JSSemi -> AST.JSStatement
expressionToStatement (AST.JSFunctionExpression a b@(AST.JSIdentName{}) c d e f) s = AST.JSFunction a b c d e f s
expressionToStatement (AST.JSGeneratorExpression a b c@(AST.JSIdentName{}) d e f g) s = AST.JSGenerator a b c d e f g s
expressionToStatement (AST.JSAssignExpression lhs op rhs) s = AST.JSAssignStatement lhs op rhs s
expressionToStatement (AST.JSMemberExpression e l a r) s = AST.JSMethodCall e l a r s
expressionToStatement exp s = AST.JSExpressionStatement exp s
Expand Down
3 changes: 2 additions & 1 deletion src/Language/JavaScript/Parser/Lexer.x
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -561,6 +561,7 @@ keywordNames =
, ( "void", VoidToken )
, ( "while", WhileToken )
, ( "with", WithToken )
, ( "yield", YieldToken )
-- TODO: no idea if these are reserved or not, but they are needed
-- handled in parser, in the Identifier rule
, ( "as", AsToken ) -- not reserved
Expand Down Expand Up @@ -601,7 +602,7 @@ keywordNames =
, ( "public", FutureToken )
, ( "static", FutureToken )
-- ( "strict", FutureToken ) *** not any more
, ( "yield", FutureToken)
-- ( "yield", FutureToken) **** an actual token, used in productions
]
}

Expand Down
1 change: 1 addition & 0 deletions src/Language/JavaScript/Parser/Token.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ data Token
| VarToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| VoidToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| WhileToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| YieldToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| ImportToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| WithToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
| ExportToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation] }
Expand Down
4 changes: 4 additions & 0 deletions src/Language/JavaScript/Pretty/Printer.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ instance RenderJS JSExpression where
(|>) pacc (JSExpressionPostfix xs op) = pacc |> xs |> op
(|>) pacc (JSExpressionTernary cond h v1 c v2) = pacc |> cond |> h |> "?" |> v1 |> c |> ":" |> v2
(|>) pacc (JSFunctionExpression annot n lb x2s rb x3) = pacc |> annot |> "function" |> n |> lb |> "(" |> x2s |> rb |> ")" |> x3
(|>) pacc (JSGeneratorExpression annot s n lb x2s rb x3) = pacc |> annot |> "function" |> s |> "*" |> n |> lb |> "(" |> x2s |> rb |> ")" |> x3
(|>) pacc (JSMemberDot xs dot n) = pacc |> xs |> "." |> dot |> n
(|>) pacc (JSMemberExpression e lb a rb) = pacc |> e |> lb |> "(" |> a |> rb |> ")"
(|>) pacc (JSMemberNew a lb n rb s) = pacc |> a |> "new" |> lb |> "(" |> n |> rb |> ")" |> s
Expand All @@ -92,6 +93,8 @@ instance RenderJS JSExpression where
(|>) pacc (JSTemplateLiteral t a h ps) = pacc |> t |> a |> h |> ps
(|>) pacc (JSUnaryExpression op x) = pacc |> op |> x
(|>) pacc (JSVarInitExpression x1 x2) = pacc |> x1 |> x2
(|>) pacc (JSYieldExpression y x) = pacc |> y |> "yield" |> x
(|>) pacc (JSYieldFromExpression y s x) = pacc |> y |> "yield" |> s |> "*" |> x
(|>) pacc (JSSpreadExpression a e) = pacc |> a |> "..." |> e

instance RenderJS JSArrowParameterList where
Expand Down Expand Up @@ -238,6 +241,7 @@ instance RenderJS JSStatement where
(|>) pacc (JSForOf af alb x1s i x2 arb x3) = pacc |> af |> "for" |> alb |> "(" |> x1s |> i |> x2 |> arb |> ")" |> x3
(|>) pacc (JSForVarOf af alb v x1 i x2 arb x3) = pacc |> af |> "for" |> alb |> "(" |> "var" |> v |> x1 |> i |> x2 |> arb |> ")" |> x3
(|>) pacc (JSFunction af n alb x2s arb x3 s) = pacc |> af |> "function" |> n |> alb |> "(" |> x2s |> arb |> ")" |> x3 |> s
(|>) pacc (JSGenerator af as n alb x2s arb x3 s) = pacc |> af |> "function" |> as |> "*" |> n |> alb |> "(" |> x2s |> arb |> ")" |> x3 |> s
(|>) pacc (JSIf annot alb x1 arb x2s) = pacc |> annot |> "if" |> alb |> "(" |> x1 |> arb |> ")" |> x2s
(|>) pacc (JSIfElse annot alb x1 arb x2s ea x3s) = pacc |> annot |> "if" |> alb |> "(" |> x1 |> arb |> ")" |> x2s |> ea |> "else" |> x3s
(|>) pacc (JSLabelled l c v) = pacc |> l |> c |> ":" |> v
Expand Down
4 changes: 4 additions & 0 deletions src/Language/JavaScript/Process/Minify.hs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ fixStmt a s (JSForConstOf _ _ _ e1 op e2 _ st) = JSForConstOf a emptyAnnot space
fixStmt a s (JSForOf _ _ e1 op e2 _ st) = JSForOf a emptyAnnot (fixEmpty e1) (fixSpace op) (fixSpace e2) emptyAnnot (fixStmtE s st)
fixStmt a s (JSForVarOf _ _ _ e1 op e2 _ st) = JSForVarOf a emptyAnnot spaceAnnot (fixEmpty e1) (fixSpace op) (fixSpace e2) emptyAnnot (fixStmtE s st)
fixStmt a s (JSFunction _ n _ ps _ blk _) = JSFunction a (fixSpace n) emptyAnnot (fixEmpty ps) emptyAnnot (fixEmpty blk) s
fixStmt a s (JSGenerator _ _ n _ ps _ blk _) = JSGenerator a emptyAnnot (fixEmpty n) emptyAnnot (fixEmpty ps) emptyAnnot (fixEmpty blk) s
fixStmt a s (JSIf _ _ e _ st) = JSIf a emptyAnnot (fixEmpty e) emptyAnnot (fixIfElseBlock emptyAnnot s st)
fixStmt a s (JSIfElse _ _ e _ (JSEmptyStatement _) _ sf) = JSIfElse a emptyAnnot (fixEmpty e) emptyAnnot (JSEmptyStatement emptyAnnot) emptyAnnot (fixStmt spaceAnnot s sf)
fixStmt a s (JSIfElse _ _ e _ st _ sf) = JSIfElse a emptyAnnot (fixEmpty e) emptyAnnot (mkStatementBlock noSemi st) emptyAnnot (fixIfElseBlock spaceAnnot s sf)
Expand Down Expand Up @@ -162,6 +163,7 @@ instance MinifyJS JSExpression where
fix a (JSExpressionPostfix e op) = JSExpressionPostfix (fix a e) (fixEmpty op)
fix a (JSExpressionTernary cond _ v1 _ v2) = JSExpressionTernary (fix a cond) emptyAnnot (fixEmpty v1) emptyAnnot (fixEmpty v2)
fix a (JSFunctionExpression _ n _ x2s _ x3) = JSFunctionExpression a (fixSpace n) emptyAnnot (fixEmpty x2s) emptyAnnot (fixEmpty x3)
fix a (JSGeneratorExpression _ _ n _ x2s _ x3) = JSGeneratorExpression a emptyAnnot (fixEmpty n) emptyAnnot (fixEmpty x2s) emptyAnnot (fixEmpty x3)
fix a (JSMemberDot xs _ n) = JSMemberDot (fix a xs) emptyAnnot (fixEmpty n)
fix a (JSMemberExpression e _ args _) = JSMemberExpression (fix a e) emptyAnnot (fixEmpty args) emptyAnnot
fix a (JSMemberNew _ n _ s _) = JSMemberNew a (fix spaceAnnot n) emptyAnnot (fixEmpty s) emptyAnnot
Expand All @@ -171,6 +173,8 @@ instance MinifyJS JSExpression where
fix a (JSTemplateLiteral t _ s ps) = JSTemplateLiteral (fmap (fix a) t) emptyAnnot s (map fixEmpty ps)
fix a (JSUnaryExpression op x) = let (ta, fop) = fixUnaryOp a op in JSUnaryExpression fop (fix ta x)
fix a (JSVarInitExpression x1 x2) = JSVarInitExpression (fix a x1) (fixEmpty x2)
fix a (JSYieldExpression _ x) = JSYieldExpression a (fixSpace x)
fix a (JSYieldFromExpression _ _ x) = JSYieldFromExpression a emptyAnnot (fixEmpty x)
fix a (JSSpreadExpression _ e) = JSSpreadExpression a (fixEmpty e)

instance MinifyJS JSArrowParameterList where
Expand Down
Loading