Implemented a 'from tokens' method for EvaluableExpression, and fixed…

… some lexer state inconsistencies that were hidden by parsing

EvaluableExpression.go

@@ -39,8 +39,43 @@ func NewEvaluableExpression(expression string) (*EvaluableExpression, error) {
 }
 
 /*
-	Parses a new EvaluableExpression from the given [expression] string.
-	Returns an error if the given expression has invalid syntax.
+	Similar to [NewEvaluableExpression], except that instead of a string, an already-tokenized expression is given.
+	This is useful in cases where you may be generating an expression automatically, or using some other parser (e.g., to parse from a query language)
+*/
+func NewEvaluableExpressionFromTokens(tokens []ExpressionToken) (*EvaluableExpression, error) {
+
+	var ret *EvaluableExpression
+	var err error
+
+	ret = new(EvaluableExpression)
+	ret.QueryDateFormat = isoDateFormat
+
+	err = checkBalance(tokens)
+	if(err != nil) {
+		return nil, err
+	}
+
+	err = checkExpressionSyntax(tokens)
+	if(err != nil) {
+		return nil, err
+	}
+
+	ret.tokens, err = optimizeTokens(tokens)
+	if(err != nil) {
+		return nil, err
+	}
+
+	ret.evaluationStages, err = planStages(ret.tokens)
+	if err != nil {
+		return nil, err
+	}
+
+	return ret, nil
+}
+
+/*
+	Similar to [NewEvaluableExpression], except enables the use of user-defined functions.
+	Functions passed into this will be available to the expression.
 */
 func NewEvaluableExpressionWithFunctions(expression string, functions map[string]ExpressionFunction) (*EvaluableExpression, error) {
 
@@ -56,6 +91,11 @@ func NewEvaluableExpressionWithFunctions(expression string, functions map[string
 		return nil, err
 	}
 
+	err = checkExpressionSyntax(ret.tokens)
+	if(err != nil) {
+		return nil, err
+	}
+
 	ret.evaluationStages, err = planStages(ret.tokens)
 	if err != nil {
 		return nil, err

lexerState.go

@@ -1,7 +1,14 @@
 package govaluate
 
+import (
+
+	"fmt"
+	"errors"
+)
+
 type lexerState struct {
 	isEOF          bool
+	isNullable	   bool
 	kind           TokenKind
 	validNextKinds []TokenKind
 }
@@ -14,12 +21,14 @@ var validLexerStates = []lexerState{
 
 		kind:  CLAUSE,
 		isEOF: false,
+		isNullable: true,
 		validNextKinds: []TokenKind{
 
 			PREFIX,
 			NUMERIC,
 			BOOLEAN,
 			VARIABLE,
+			PATTERN,
 			FUNCTION,
 			STRING,
 			TIME,
@@ -32,6 +41,7 @@ var validLexerStates = []lexerState{
 
 		kind:  CLAUSE_CLOSE,
 		isEOF: true,
+		isNullable: true,
 		validNextKinds: []TokenKind{
 
 			COMPARATOR,
@@ -41,6 +51,7 @@ var validLexerStates = []lexerState{
 			BOOLEAN,
 			VARIABLE,
 			STRING,
+			PATTERN,
 			TIME,
 			CLAUSE,
 			CLAUSE_CLOSE,
@@ -54,6 +65,7 @@ var validLexerStates = []lexerState{
 
 		kind:  NUMERIC,
 		isEOF: true,
+		isNullable: false,
 		validNextKinds: []TokenKind{
 
 			MODIFIER,
@@ -68,6 +80,7 @@ var validLexerStates = []lexerState{
 
 		kind:  BOOLEAN,
 		isEOF: true,
+		isNullable: false,
 		validNextKinds: []TokenKind{
 
 			MODIFIER,
@@ -82,6 +95,7 @@ var validLexerStates = []lexerState{
 
 		kind:  STRING,
 		isEOF: true,
+		isNullable: false,
 		validNextKinds: []TokenKind{
 
 			MODIFIER,
@@ -96,6 +110,21 @@ var validLexerStates = []lexerState{
 
 		kind:  TIME,
 		isEOF: true,
+		isNullable: false,
+		validNextKinds: []TokenKind{
+
+			MODIFIER,
+			COMPARATOR,
+			LOGICALOP,
+			CLAUSE_CLOSE,
+			SEPARATOR,
+		},
+	},
+	lexerState{
+
+		kind:  PATTERN,
+		isEOF: true,
+		isNullable: false,
 		validNextKinds: []TokenKind{
 
 			MODIFIER,
@@ -109,6 +138,7 @@ var validLexerStates = []lexerState{
 
 		kind:  VARIABLE,
 		isEOF: true,
+		isNullable: false,
 		validNextKinds: []TokenKind{
 
 			MODIFIER,
@@ -123,6 +153,7 @@ var validLexerStates = []lexerState{
 
 		kind:  MODIFIER,
 		isEOF: false,
+		isNullable: false,
 		validNextKinds: []TokenKind{
 
 			PREFIX,
@@ -139,6 +170,7 @@ var validLexerStates = []lexerState{
 
 		kind:  COMPARATOR,
 		isEOF: false,
+		isNullable: false,
 		validNextKinds: []TokenKind{
 
 			PREFIX,
@@ -150,12 +182,14 @@ var validLexerStates = []lexerState{
 			TIME,
 			CLAUSE,
 			CLAUSE_CLOSE,
+			PATTERN,
 		},
 	},
 	lexerState{
 
 		kind:  LOGICALOP,
 		isEOF: false,
+		isNullable: false,
 		validNextKinds: []TokenKind{
 
 			PREFIX,
@@ -173,6 +207,7 @@ var validLexerStates = []lexerState{
 
 		kind:  PREFIX,
 		isEOF: false,
+		isNullable: false,
 		validNextKinds: []TokenKind{
 
 			NUMERIC,
@@ -188,6 +223,7 @@ var validLexerStates = []lexerState{
 
 		kind:  TERNARY,
 		isEOF: false,
+		isNullable: false,
 		validNextKinds: []TokenKind{
 
 			PREFIX,
@@ -205,6 +241,7 @@ var validLexerStates = []lexerState{
 
 		kind:  FUNCTION,
 		isEOF: false,
+		isNullable: false,
 		validNextKinds: []TokenKind{
 			CLAUSE,
 		},
@@ -213,6 +250,7 @@ var validLexerStates = []lexerState{
 
 		kind:  SEPARATOR,
 		isEOF: false,
+		isNullable: true,
 		validNextKinds: []TokenKind{
 
 			PREFIX,
@@ -238,3 +276,54 @@ func (this lexerState) canTransitionTo(kind TokenKind) bool {
 
 	return false
 }
+
+func checkExpressionSyntax(tokens []ExpressionToken) error {
+
+	var state lexerState
+	var lastToken ExpressionToken
+	var err error
+
+	state = validLexerStates[0]
+
+	for _, token := range tokens {
+
+		if !state.canTransitionTo(token.Kind) {
+
+			firstStateName := fmt.Sprintf("%s [%v]", GetTokenKindString(state.kind), lastToken.Value)
+			nextStateName := fmt.Sprintf("%s [%v]", GetTokenKindString(token.Kind), token.Value)
+
+			return errors.New("Cannot transition token types from " + firstStateName + " to " + nextStateName)
+		}
+
+		state, err = getLexerStateForToken(token.Kind)
+		if(err != nil) {
+			return err
+		}
+
+		if(!state.isNullable && token.Value == nil) {
+
+			errorMsg := fmt.Sprintf("Token kind '%v' cannot have a nil value", GetTokenKindString(token.Kind))
+			return errors.New(errorMsg)
+		}
+
+		lastToken = token
+	}
+
+	if !state.isEOF {
+		return errors.New("Unexpected end of expression")
+	}
+	return nil
+}
+
+func getLexerStateForToken(kind TokenKind) (lexerState, error) {
+
+	for _, possibleState := range validLexerStates {
+
+		if possibleState.kind == kind {
+			return possibleState, nil
+		}
+	}
+
+	errorMsg := fmt.Sprintf("No lexer state found for token kind '%v'\n", GetTokenKindString(kind))
+	return validLexerStates[0], errors.New(errorMsg)
+}

parsing.go

@@ -13,14 +13,14 @@ import (
 func parseTokens(expression string, functions map[string]ExpressionFunction) ([]ExpressionToken, error) {
 
 	var ret []ExpressionToken
-	var token, lastToken ExpressionToken
-	var state lexerState
+	var token ExpressionToken
 	var stream *lexerStream
+	var state lexerState
 	var err error
 	var found bool
 
-	state = validLexerStates[0]
 	stream = newLexerStream(expression)
+	state = validLexerStates[0]
 
 	for stream.canRead() {
 
@@ -34,31 +34,13 @@ func parseTokens(expression string, functions map[string]ExpressionFunction) ([]
 			break
 		}
 
-		if !state.canTransitionTo(token.Kind) {
-
-			firstStateName := fmt.Sprintf("%s [%v]", GetTokenKindString(state.kind), lastToken.Value)
-			nextStateName := fmt.Sprintf("%s [%v]", GetTokenKindString(token.Kind), token.Value)
-
-			return ret, errors.New("Cannot transition token types from " + firstStateName + " to " + nextStateName)
+		state, err = getLexerStateForToken(token.Kind)
+		if(err != nil) {
+			return ret, err
 		}
 
-		// append this valid token, find new lexer state.
+		// append this valid token
 		ret = append(ret, token)
-
-		for _, possibleState := range validLexerStates {
-
-			if possibleState.kind == token.Kind {
-
-				state = possibleState
-				break
-			}
-		}
-
-		lastToken = token
-	}
-
-	if !state.isEOF {
-		return ret, errors.New("Unexpected end of expression")
 	}
 
 	err = checkBalance(ret)

parsing_test.go

@@ -1231,6 +1231,7 @@ func TestOriginalString(test *testing.T) {
 
 		test.Logf("failed to parse original string test: %v", err)
 		test.Fail()
+		return
 	}
 
 	if expression.String() != expressionString {