Add support for template literals

src/Language/JavaScript/Parser/AST.hs

@@ -23,6 +23,7 @@ module Language.JavaScript.Parser.AST
     , JSCommaList (..)
     , JSCommaTrailingList (..)
     , JSArrowParameterList (..)
+    , JSTemplatePart (..)
 
     -- Modules
     , JSModuleItem (..)
@@ -185,6 +186,7 @@ data JSExpression
     | JSNewExpression !JSAnnot !JSExpression -- ^new, expr
     | JSObjectLiteral !JSAnnot !JSObjectPropertyList !JSAnnot -- ^lbrace contents rbrace
     | JSSpreadExpression !JSAnnot !JSExpression
+    | JSTemplateLiteral !(Maybe JSExpression) !JSAnnot !String ![JSTemplatePart] -- ^optional tag, lquot, head, parts
     | JSUnaryExpression !JSUnaryOp !JSExpression
     | JSVarInitExpression !JSExpression !JSVarInitializer -- ^identifier, initializer
     deriving (Data, Eq, Show, Typeable)
@@ -317,6 +319,10 @@ data JSCommaTrailingList a
     | JSCTLNone !(JSCommaList a) -- ^list
     deriving (Data, Eq, Show, Typeable)
 
+data JSTemplatePart
+    = JSTemplatePart !JSExpression !JSAnnot !String -- ^expr, rb, suffix
+    deriving (Data, Eq, Show, Typeable)
+
 -- -----------------------------------------------------------------------------
 -- | Show the AST elements stripped of their JSAnnot data.
 
@@ -397,6 +403,8 @@ instance ShowStripped JSExpression where
     ss (JSUnaryExpression op x) = "JSUnaryExpression (" ++ ss op ++ "," ++ ss x ++ ")"
     ss (JSVarInitExpression x1 x2) = "JSVarInitExpression (" ++ ss x1 ++ ") " ++ ss x2
     ss (JSSpreadExpression _ x1) = "JSSpreadExpression (" ++ ss x1 ++ ")"
+    ss (JSTemplateLiteral Nothing _ s ps) = "JSTemplateLiteral (()," ++ singleQuote s ++ "," ++ ss ps ++ ")"
+    ss (JSTemplateLiteral (Just t) _ s ps) = "JSTemplateLiteral ((" ++ ss t ++ ")," ++ singleQuote s ++ "," ++ ss ps ++ ")"
 
 instance ShowStripped JSArrowParameterList where
     ss (JSUnparenthesizedArrowParameter x) = ss x
@@ -537,6 +545,9 @@ instance ShowStripped JSArrayElement where
     ss (JSArrayElement e) = ss e
     ss (JSArrayComma _) = "JSComma"
 
+instance ShowStripped JSTemplatePart where
+    ss (JSTemplatePart e _ s) = "(" ++ ss e ++ "," ++ singleQuote s ++ ")"
+
 instance ShowStripped a => ShowStripped (JSCommaList a) where
     ss xs = "(" ++ commaJoin (map ss $ fromCommaList xs) ++ ")"
 

src/Language/JavaScript/Parser/Grammar7.y

@@ -131,6 +131,10 @@ import qualified Language.JavaScript.Parser.AST as AST
      'octal'      { OctalToken {} }
      'string'     { StringToken {} }
      'regex'      { RegExToken {} }
+     'tmplnosub'  { NoSubstitutionTemplateToken {} }
+     'tmplhead'   { TemplateHeadToken {} }
+     'tmplmiddle' { TemplateMiddleToken {} }
+     'tmpltail'   { TemplateTailToken {} }
 
      'future'     { FutureToken {} }
 
@@ -433,6 +437,7 @@ PrimaryExpression : 'this'                   { AST.JSLiteral (mkJSAnnot $1) "thi
                   | ArrayLiteral             { $1 {- 'PrimaryExpression3' -} }
                   | ObjectLiteral            { $1 {- 'PrimaryExpression4' -} }
                   | SpreadExpression         { $1 {- 'PrimaryExpression5' -} }
+                  | TemplateLiteral          { mkJSTemplateLiteral Nothing $1 {- 'PrimaryExpression6' -} }
                   | LParen Expression RParen { AST.JSExpressionParen $1 $2 $3 }
 
 -- Identifier ::                                                            See 7.6
@@ -494,6 +499,14 @@ IdentifierName : Identifier {$1}
 SpreadExpression :: { AST.JSExpression }
 SpreadExpression : Spread Expression  { AST.JSSpreadExpression $1 $2 {- 'SpreadExpression' -} }
 
+TemplateLiteral :: { JSUntaggedTemplate }
+TemplateLiteral : 'tmplnosub'              { JSUntaggedTemplate (mkJSAnnot $1) (tokenLiteral $1) [] }
+                | 'tmplhead' TemplateParts { JSUntaggedTemplate (mkJSAnnot $1) (tokenLiteral $1) $2 }
+
+TemplateParts :: { [AST.JSTemplatePart] }
+TemplateParts : Expression 'tmplmiddle' TemplateParts { AST.JSTemplatePart $1 (mkJSAnnot $2) (tokenLiteral $2) : $3 }
+              | Expression 'tmpltail'                 { AST.JSTemplatePart $1 (mkJSAnnot $2) (tokenLiteral $2) : [] }
+
 -- ArrayLiteral :                                                        See 11.1.4
 --        [ Elisionopt ]
 --        [ ElementList ]
@@ -580,6 +593,7 @@ MemberExpression : PrimaryExpression   { $1 {- 'MemberExpression1' -} }
                  | FunctionExpression  { $1 {- 'MemberExpression2' -} }
                  | MemberExpression LSquare Expression RSquare { AST.JSMemberSquare $1 $2 $3 $4 {- 'MemberExpression3' -} }
                  | MemberExpression Dot IdentifierName         { AST.JSMemberDot $1 $2 $3       {- 'MemberExpression4' -} }
+                 | MemberExpression TemplateLiteral            { mkJSTemplateLiteral (Just $1) $2 }
                  | New MemberExpression Arguments              { mkJSMemberNew $1 $2 $3         {- 'MemberExpression5' -} }
 
 -- NewExpression :                                              See 11.2
@@ -603,6 +617,8 @@ CallExpression : MemberExpression Arguments
                     { AST.JSCallExpressionSquare $1 $2 $3 $4 {- 'CallExpression3' -} }
                | CallExpression Dot IdentifierName
                     { AST.JSCallExpressionDot $1 $2 $3 {- 'CallExpression4' -} }
+               | CallExpression TemplateLiteral
+                    { mkJSTemplateLiteral (Just $1) $2 {- 'CallExpression5' -} }
 
 -- Arguments :                                                  See 11.2
 --        ()
@@ -1332,7 +1348,7 @@ StatementMain : StatementNoEmpty Eof	{ AST.JSAstStatement $1 $2   	{- 'Statement
 
 -- Need this type while build the AST, but is not actually part of the AST.
 data JSArguments = JSArguments AST.JSAnnot (AST.JSCommaList AST.JSExpression) AST.JSAnnot    -- ^lb, args, rb
-
+data JSUntaggedTemplate = JSUntaggedTemplate !AST.JSAnnot !String ![AST.JSTemplatePart] -- lquot, head, parts
 
 blockToStatement :: AST.JSBlock -> AST.JSSemi -> AST.JSStatement
 blockToStatement (AST.JSBlock a b c) s = AST.JSStatementBlock a b c s
@@ -1359,6 +1375,9 @@ parseError = alexError . show
 mkJSAnnot :: Token -> AST.JSAnnot
 mkJSAnnot a = AST.JSAnnot (tokenSpan a) (tokenComment a)
 
+mkJSTemplateLiteral :: Maybe AST.JSExpression -> JSUntaggedTemplate -> AST.JSExpression
+mkJSTemplateLiteral tag (JSUntaggedTemplate a h ps) = AST.JSTemplateLiteral tag a h ps
+
 -- ---------------------------------------------------------------------
 -- | mkUnary : The parser detects '+' and '-' as the binary version of these
 -- operator. This function converts from the binary version to the unary

src/Language/JavaScript/Parser/Lexer.x

@@ -187,6 +187,14 @@ $ZWJ  = [\x200d]
 @IdentifierPart = @IdentifierStart | $UnicodeCombiningMark | $UnicodeDigit | UnicodeConnectorPunctuation
         [\\] @UnicodeEscapeSequence | $ZWNJ | $ZWJ
 
+-- TemplateCharacter ::
+--         $ [lookahead ≠ { ]
+--         \ EscapeSequence
+--         LineContinuation
+--         LineTerminatorSequence
+--         SourceCharacter but not one of ` or \ or $ or LineTerminator
+@TemplateCharacters = (\$* ($any_unicode_char # [\$\\`\{] | \\ $any_unicode_char) | \\ $any_unicode_char | \{)* \$*
+
 -- ! ------------------------------------------------- Terminals
 tokens :-
 
@@ -246,6 +254,13 @@ tokens :-
 
 
 
+<reg,divide> "`" @TemplateCharacters "`" { adapt (mkString' NoSubstitutionTemplateToken) }
+<reg,divide> "`" @TemplateCharacters "${" { adapt (mkString' TemplateHeadToken) }
+<reg,divide> "}" @TemplateCharacters "${" { adapt (mkString' TemplateMiddleToken) }
+<reg,divide> "}" @TemplateCharacters "`" { adapt (mkString' TemplateTailToken) }
+
+
+
 -- TODO: Work in SignedInteger
 
 -- DecimalLiteral= {Non Zero Digits}+ '.' {Digit}* ('e' | 'E' ) {Non Zero Digits}+ {Digit}*

src/Language/JavaScript/Parser/LexerUtils.hs

@@ -14,6 +14,7 @@ module Language.JavaScript.Parser.LexerUtils
     ( StartCode
     , symbolToken
     , mkString
+    , mkString'
     , commentToken
     , wsToken
     , regExToken
@@ -37,6 +38,9 @@ symbolToken mkToken location _ _ = return (mkToken location [])
 mkString :: (Monad m) => (TokenPosn -> String -> Token) -> TokenPosn -> Int -> String -> m Token
 mkString toToken loc len str = return (toToken loc (take len str))
 
+mkString' :: (Monad m) => (TokenPosn -> String -> [CommentAnnotation] -> Token) -> TokenPosn -> Int -> String -> m Token
+mkString' toToken loc len str = return (toToken loc (take len str) [])
+
 decimalToken :: TokenPosn -> String -> Token
 decimalToken loc str = DecimalToken loc str []
 

src/Language/JavaScript/Parser/Token.hs

@@ -153,6 +153,12 @@ data Token
     | RightParenToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation]  }
     | CondcommentEndToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation]  }
 
+    -- Template literal lexical components
+    | NoSubstitutionTemplateToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation]  }
+    | TemplateHeadToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation]  }
+    | TemplateMiddleToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation]  }
+    | TemplateTailToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation]  }
+
     -- Special cases
     | AsToken { tokenSpan :: !TokenPosn, tokenLiteral :: !String, tokenComment :: ![CommentAnnotation]  }
     | TailToken { tokenSpan :: !TokenPosn, tokenComment :: ![CommentAnnotation]  } -- ^ Stuff between last JS and EOF

src/Language/JavaScript/Pretty/Printer.hs

@@ -89,6 +89,7 @@ instance RenderJS JSExpression where
     (|>) pacc (JSMemberSquare         xs als e ars)           = pacc |> xs |> als |> "[" |> e |> ars |> "]"
     (|>) pacc (JSNewExpression        n e)                    = pacc |> n |> "new" |> e
     (|>) pacc (JSObjectLiteral        alb xs arb)             = pacc |> alb |> "{" |> xs |> arb |> "}"
+    (|>) pacc (JSTemplateLiteral      t a h ps)               = pacc |> t |> a |> h |> ps
     (|>) pacc (JSUnaryExpression      op x)                   = pacc |> op |> x
     (|>) pacc (JSVarInitExpression    x1 x2)                  = pacc |> x1 |> x2
     (|>) pacc (JSSpreadExpression     a e)                    = pacc |> a |> "..." |> e
@@ -340,4 +341,10 @@ instance RenderJS JSVarInitializer where
     (|>) pacc (JSVarInit a x) = pacc |> a |> "=" |> x
     (|>) pacc JSVarInitNone   = pacc
 
+instance RenderJS [JSTemplatePart] where
+    (|>) = foldl' (|>)
+
+instance RenderJS JSTemplatePart where
+    (|>) pacc (JSTemplatePart e a s) = pacc |> e |> a |> s
+
 -- EOF

src/Language/JavaScript/Process/Minify.hs

@@ -165,6 +165,7 @@ instance MinifyJS JSExpression where
     fix a (JSMemberSquare         xs _ e _)           = JSMemberSquare (fix a xs) emptyAnnot (fixEmpty e) emptyAnnot
     fix a (JSNewExpression        _ e)                = JSNewExpression a (fixSpace e)
     fix _ (JSObjectLiteral        _ xs _)             = JSObjectLiteral emptyAnnot (fixEmpty xs) emptyAnnot
+    fix a (JSTemplateLiteral      t _ s ps)           = JSTemplateLiteral (fmap (fix a) t) emptyAnnot s (map fixEmpty ps)
     fix a (JSUnaryExpression      op x)               = let (ta, fop) = fixUnaryOp a op in JSUnaryExpression fop (fix ta x)
     fix a (JSVarInitExpression    x1 x2)              = JSVarInitExpression (fix a x1) (fixEmpty x2)
     fix a (JSSpreadExpression     _ e)                = JSSpreadExpression a (fixEmpty e)
@@ -395,6 +396,10 @@ instance MinifyJS JSVarInitializer where
     fix _ JSVarInitNone = JSVarInitNone
 
 
+instance MinifyJS JSTemplatePart where
+    fix _ (JSTemplatePart e _ s) = JSTemplatePart (fixEmpty e) emptyAnnot s
+
+
 spaceAnnot :: JSAnnot
 spaceAnnot = JSAnnot tokenPosnEmpty [WhiteSpace tokenPosnEmpty " "]
 

test/Test/Language/Javascript/ExpressionParser.hs

@@ -146,6 +146,18 @@ testExpressionParser = describe "Parse expressions:" $ do
     it "spread expression" $
         testExpr "... x"        `shouldBe` "Right (JSAstExpression (JSSpreadExpression (JSIdentifier 'x')))"
 
+    it "template literal" $ do
+        testExpr "``"            `shouldBe` "Right (JSAstExpression (JSTemplateLiteral ((),'``',[])))"
+        testExpr "`$`"           `shouldBe` "Right (JSAstExpression (JSTemplateLiteral ((),'`$`',[])))"
+        testExpr "`$\\n`"        `shouldBe` "Right (JSAstExpression (JSTemplateLiteral ((),'`$\\n`',[])))"
+        testExpr "`\\${x}`"      `shouldBe` "Right (JSAstExpression (JSTemplateLiteral ((),'`\\${x}`',[])))"
+        testExpr "`$ {x}`"       `shouldBe` "Right (JSAstExpression (JSTemplateLiteral ((),'`$ {x}`',[])))"
+        testExpr "`\n\n`"        `shouldBe` "Right (JSAstExpression (JSTemplateLiteral ((),'`\n\n`',[])))"
+        testExpr "`${x+y} ${z}`" `shouldBe` "Right (JSAstExpression (JSTemplateLiteral ((),'`${',[(JSExpressionBinary ('+',JSIdentifier 'x',JSIdentifier 'y'),'} ${'),(JSIdentifier 'z','}`')])))"
+        testExpr "`<${x} ${y}>`" `shouldBe` "Right (JSAstExpression (JSTemplateLiteral ((),'`<${',[(JSIdentifier 'x','} ${'),(JSIdentifier 'y','}>`')])))"
+        testExpr "tag `xyz`"     `shouldBe` "Right (JSAstExpression (JSTemplateLiteral ((JSIdentifier 'tag'),'`xyz`',[])))"
+        testExpr "tag()`xyz`"    `shouldBe` "Right (JSAstExpression (JSTemplateLiteral ((JSMemberExpression (JSIdentifier 'tag',JSArguments ())),'`xyz`',[])))"
+
 
 testExpr :: String -> String
 testExpr str = showStrippedMaybe (parseUsing parseExpression str "src")

test/Test/Language/Javascript/Minify.hs

@@ -136,6 +136,10 @@ testMinifyExpr = describe "Minify expressions:" $ do
     it "spread exporession" $
         minifyExpr " ... x " `shouldBe` "...x"
 
+    it "template literal" $ do
+        minifyExpr " ` a + b + ${ c + d } + ... ` " `shouldBe` "` a + b + ${c+d} + ... `"
+        minifyExpr " tagger () ` a + b ` " `shouldBe` "tagger()` a + b `"
+
 
 testMinifyStmt :: Spec
 testMinifyStmt = describe "Minify statements:" $ do

test/Test/Language/Javascript/RoundTrip.hs

@@ -69,6 +69,10 @@ testRoundTrip = describe "Roundtrip:" $ do
         testRT "(a, b) => a + b"
         testRT "() => { 42 }"
 
+        testRT "/*a*/`<${/*b*/x/*c*/}>`/*d*/"
+        testRT "`\\${}`"
+        testRT "`\n\n`"
+
 
     it "statement" $ do
         testRT "if (1) {}"