diff --git a/src/main/java/com/hubspot/jinjava/Jinjava.java b/src/main/java/com/hubspot/jinjava/Jinjava.java index 515df6049..00efd10df 100644 --- a/src/main/java/com/hubspot/jinjava/Jinjava.java +++ b/src/main/java/com/hubspot/jinjava/Jinjava.java @@ -19,6 +19,7 @@ import com.hubspot.jinjava.doc.JinjavaDocFactory; import com.hubspot.jinjava.el.ExtendedSyntaxBuilder; import com.hubspot.jinjava.el.TruthyTypeConverter; +import com.hubspot.jinjava.el.ext.eager.EagerExtendedSyntaxBuilder; import com.hubspot.jinjava.interpret.Context; import com.hubspot.jinjava.interpret.FatalTemplateErrorsException; import com.hubspot.jinjava.interpret.InterpretException; @@ -61,6 +62,7 @@ */ public class Jinjava { private ExpressionFactory expressionFactory; + private ExpressionFactory eagerExpressionFactory; private ResourceLocator resourceLocator; private Context globalContext; @@ -84,13 +86,21 @@ public Jinjava(JinjavaConfig globalConfig) { this.globalContext = new Context(); Properties expConfig = new Properties(); + expConfig.setProperty( TreeBuilder.class.getName(), ExtendedSyntaxBuilder.class.getName() ); + Properties eagerExpConfig = new Properties(); + + eagerExpConfig.setProperty( + TreeBuilder.class.getName(), + EagerExtendedSyntaxBuilder.class.getName() + ); TypeConverter converter = new TruthyTypeConverter(); this.expressionFactory = new ExpressionFactoryImpl(expConfig, converter); + this.eagerExpressionFactory = new ExpressionFactoryImpl(eagerExpConfig, converter); this.resourceLocator = new ClasspathResourceLocator(); } @@ -112,6 +122,13 @@ public ExpressionFactory getExpressionFactory() { return expressionFactory; } + /** + * @return The EL factory used to eagerly process expressions in templates by this instance. + */ + public ExpressionFactory getEagerExpressionFactory() { + return eagerExpressionFactory; + } + /** * @return The global config used as a base for all render operations performed by this instance. */ diff --git a/src/main/java/com/hubspot/jinjava/el/ExpressionResolver.java b/src/main/java/com/hubspot/jinjava/el/ExpressionResolver.java index 66fed2d0b..f87af4705 100644 --- a/src/main/java/com/hubspot/jinjava/el/ExpressionResolver.java +++ b/src/main/java/com/hubspot/jinjava/el/ExpressionResolver.java @@ -44,7 +44,10 @@ public class ExpressionResolver { public ExpressionResolver(JinjavaInterpreter interpreter, Jinjava jinjava) { this.interpreter = interpreter; - this.expressionFactory = jinjava.getExpressionFactory(); + this.expressionFactory = + interpreter.getConfig().getExecutionMode().useEagerParser() + ? jinjava.getEagerExpressionFactory() + : jinjava.getExpressionFactory(); this.resolver = new JinjavaInterpreterResolver(interpreter); this.elContext = new JinjavaELContext(interpreter, resolver); diff --git a/src/main/java/com/hubspot/jinjava/el/JinjavaInterpreterResolver.java b/src/main/java/com/hubspot/jinjava/el/JinjavaInterpreterResolver.java index 7f9705376..d8f082674 100644 --- a/src/main/java/com/hubspot/jinjava/el/JinjavaInterpreterResolver.java +++ b/src/main/java/com/hubspot/jinjava/el/JinjavaInterpreterResolver.java @@ -4,6 +4,7 @@ import com.google.common.collect.ImmutableMap; import com.hubspot.jinjava.el.ext.AbstractCallableMethod; +import com.hubspot.jinjava.el.ext.DeferredParsingException; import com.hubspot.jinjava.el.ext.ExtendedParser; import com.hubspot.jinjava.el.ext.JinjavaBeanELResolver; import com.hubspot.jinjava.el.ext.JinjavaListELResolver; @@ -241,11 +242,15 @@ private Object getValue( } if (value instanceof DeferredValue) { - throw new DeferredValueException( - propertyName, - interpreter.getLineNumber(), - interpreter.getPosition() - ); + if (interpreter.getConfig().getExecutionMode().useEagerParser()) { + throw new DeferredParsingException(this, propertyName); + } else { + throw new DeferredValueException( + propertyName, + interpreter.getLineNumber(), + interpreter.getPosition() + ); + } } } catch (PropertyNotFoundException e) { if (errOnUnknownProp) { diff --git a/src/main/java/com/hubspot/jinjava/el/ext/AbsOperator.java b/src/main/java/com/hubspot/jinjava/el/ext/AbsOperator.java index b6ebebc59..6e5ef9d29 100644 --- a/src/main/java/com/hubspot/jinjava/el/ext/AbsOperator.java +++ b/src/main/java/com/hubspot/jinjava/el/ext/AbsOperator.java @@ -1,5 +1,6 @@ package com.hubspot.jinjava.el.ext; +import com.hubspot.jinjava.el.ext.eager.EagerAstUnary; import de.odysseus.el.misc.TypeConverter; import de.odysseus.el.tree.impl.Parser.ExtensionHandler; import de.odysseus.el.tree.impl.Parser.ExtensionPoint; @@ -7,45 +8,50 @@ import de.odysseus.el.tree.impl.Scanner.ExtensionToken; import de.odysseus.el.tree.impl.ast.AstNode; import de.odysseus.el.tree.impl.ast.AstUnary; +import de.odysseus.el.tree.impl.ast.AstUnary.SimpleOperator; -public class AbsOperator { +public class AbsOperator extends SimpleOperator { public static final ExtensionToken TOKEN = new Scanner.ExtensionToken("+"); + public static final AbsOperator OP = new AbsOperator(); - public static final ExtensionHandler HANDLER = new ExtensionHandler( - ExtensionPoint.UNARY - ) { - - @Override - public AstNode createAstNode(AstNode... children) { - return new AstUnary( - children[0], - new AstUnary.SimpleOperator() { - - @Override - protected Object apply(TypeConverter converter, Object o) { - if (o == null) { - return null; - } - - if (o instanceof Float) { - return Math.abs((Float) o); - } - if (o instanceof Double) { - return Math.abs((Double) o); - } - if (o instanceof Integer) { - return Math.abs((Integer) o); - } - if (o instanceof Long) { - return Math.abs((Long) o); - } - - throw new IllegalArgumentException( - "Unable to apply abs operator on object of type: " + o.getClass() - ); - } - } - ); + public static final ExtensionHandler HANDLER = getHandler(false); + + @Override + protected Object apply(TypeConverter converter, Object o) { + if (o == null) { + return null; + } + + if (o instanceof Float) { + return Math.abs((Float) o); + } + if (o instanceof Double) { + return Math.abs((Double) o); + } + if (o instanceof Integer) { + return Math.abs((Integer) o); } - }; + if (o instanceof Long) { + return Math.abs((Long) o); + } + + throw new IllegalArgumentException( + "Unable to apply abs operator on object of type: " + o.getClass() + ); + } + + @Override + public String toString() { + return "+"; + } + + public static ExtensionHandler getHandler(boolean eager) { + return new ExtensionHandler(ExtensionPoint.UNARY) { + + @Override + public AstNode createAstNode(AstNode... children) { + return eager ? new EagerAstUnary(children[0], OP) : new AstUnary(children[0], OP); + } + }; + } } diff --git a/src/main/java/com/hubspot/jinjava/el/ext/AdditionOperator.java b/src/main/java/com/hubspot/jinjava/el/ext/AdditionOperator.java index 6e1b2031e..4dff03e5b 100644 --- a/src/main/java/com/hubspot/jinjava/el/ext/AdditionOperator.java +++ b/src/main/java/com/hubspot/jinjava/el/ext/AdditionOperator.java @@ -39,5 +39,10 @@ protected Object apply(TypeConverter converter, Object o1, Object o2) { return NumberOperations.add(converter, o1, o2); } + @Override + public String toString() { + return "+"; + } + public static final AdditionOperator OP = new AdditionOperator(); } diff --git a/src/main/java/com/hubspot/jinjava/el/ext/AstDict.java b/src/main/java/com/hubspot/jinjava/el/ext/AstDict.java index 507c42843..36d52e4f8 100644 --- a/src/main/java/com/hubspot/jinjava/el/ext/AstDict.java +++ b/src/main/java/com/hubspot/jinjava/el/ext/AstDict.java @@ -14,7 +14,7 @@ import javax.el.ELContext; public class AstDict extends AstLiteral { - private final Map dict; + protected final Map dict; public AstDict(Map dict) { this.dict = dict; diff --git a/src/main/java/com/hubspot/jinjava/el/ext/AstList.java b/src/main/java/com/hubspot/jinjava/el/ext/AstList.java index 0c67939a4..df63b3fe4 100644 --- a/src/main/java/com/hubspot/jinjava/el/ext/AstList.java +++ b/src/main/java/com/hubspot/jinjava/el/ext/AstList.java @@ -11,7 +11,7 @@ import org.apache.commons.lang3.StringUtils; public class AstList extends AstLiteral { - private final AstParameters elements; + protected final AstParameters elements; public AstList(AstParameters elements) { this.elements = elements; diff --git a/src/main/java/com/hubspot/jinjava/el/ext/CollectionMembershipOperator.java b/src/main/java/com/hubspot/jinjava/el/ext/CollectionMembershipOperator.java index 3e4ce763a..80b10be65 100644 --- a/src/main/java/com/hubspot/jinjava/el/ext/CollectionMembershipOperator.java +++ b/src/main/java/com/hubspot/jinjava/el/ext/CollectionMembershipOperator.java @@ -1,5 +1,6 @@ package com.hubspot.jinjava.el.ext; +import com.hubspot.jinjava.el.ext.eager.EagerAstBinary; import de.odysseus.el.misc.TypeConverter; import de.odysseus.el.tree.impl.Parser.ExtensionHandler; import de.odysseus.el.tree.impl.Parser.ExtensionPoint; @@ -58,16 +59,25 @@ public Object apply(TypeConverter converter, Object o1, Object o2) { return Boolean.FALSE; } + @Override + public String toString() { + return TOKEN.getImage(); + } + public static final CollectionMembershipOperator OP = new CollectionMembershipOperator(); public static final Scanner.ExtensionToken TOKEN = new Scanner.ExtensionToken("in"); - public static final ExtensionHandler HANDLER = new ExtensionHandler( - ExtensionPoint.CMP - ) { + public static final ExtensionHandler HANDLER = getHandler(false); - @Override - public AstNode createAstNode(AstNode... children) { - return new AstBinary(children[0], children[1], OP); - } - }; + public static ExtensionHandler getHandler(boolean eager) { + return new ExtensionHandler(ExtensionPoint.CMP) { + + @Override + public AstNode createAstNode(AstNode... children) { + return eager + ? new EagerAstBinary(children[0], children[1], OP) + : new AstBinary(children[0], children[1], OP); + } + }; + } } diff --git a/src/main/java/com/hubspot/jinjava/el/ext/DeferredParsingException.java b/src/main/java/com/hubspot/jinjava/el/ext/DeferredParsingException.java new file mode 100644 index 000000000..7e88a2d97 --- /dev/null +++ b/src/main/java/com/hubspot/jinjava/el/ext/DeferredParsingException.java @@ -0,0 +1,28 @@ +package com.hubspot.jinjava.el.ext; + +import com.hubspot.jinjava.interpret.DeferredValueException; + +public class DeferredParsingException extends DeferredValueException { + private final String deferredEvalResult; + private final Object sourceNode; + + public DeferredParsingException(Object sourceNode, String deferredEvalResult) { + super( + String.format( + "%s could not be parsed more than: %s", + sourceNode.getClass(), + deferredEvalResult + ) + ); + this.deferredEvalResult = deferredEvalResult; + this.sourceNode = sourceNode; + } + + public String getDeferredEvalResult() { + return deferredEvalResult; + } + + public Object getSourceNode() { + return sourceNode; + } +} diff --git a/src/main/java/com/hubspot/jinjava/el/ext/ExtendedParser.java b/src/main/java/com/hubspot/jinjava/el/ext/ExtendedParser.java index 06bedeb18..3a3c51ef9 100644 --- a/src/main/java/com/hubspot/jinjava/el/ext/ExtendedParser.java +++ b/src/main/java/com/hubspot/jinjava/el/ext/ExtendedParser.java @@ -4,6 +4,7 @@ import static de.odysseus.el.tree.impl.Builder.Feature.NULL_PROPERTIES; import static de.odysseus.el.tree.impl.Scanner.Symbol.COLON; import static de.odysseus.el.tree.impl.Scanner.Symbol.COMMA; +import static de.odysseus.el.tree.impl.Scanner.Symbol.DOT; import static de.odysseus.el.tree.impl.Scanner.Symbol.EQ; import static de.odysseus.el.tree.impl.Scanner.Symbol.FALSE; import static de.odysseus.el.tree.impl.Scanner.Symbol.GE; @@ -37,6 +38,7 @@ import de.odysseus.el.tree.impl.ast.AstNull; import de.odysseus.el.tree.impl.ast.AstParameters; import de.odysseus.el.tree.impl.ast.AstProperty; +import de.odysseus.el.tree.impl.ast.AstRightValue; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; @@ -221,7 +223,7 @@ protected AstParameters params(Symbol left, Symbol right) } } consumeToken(right); - return new AstParameters(l); + return createAstParameters(l); } protected AstDict dict() throws ScanException, ParseException { @@ -252,6 +254,10 @@ protected AstDict dict() throws ScanException, ParseException { fail("}"); } consumeToken(); + return createAstDict(dict); + } + + protected AstDict createAstDict(Map dict) { return new AstDict(dict); } @@ -269,7 +275,7 @@ protected AstNode nonliteral() throws ScanException, ParseException { if ( getToken().getSymbol() == COLON && lookahead(0).getSymbol() == IDENTIFIER && - lookahead(1).getSymbol() == LPAREN + (lookahead(1).getSymbol() == LPAREN || (isPossibleExpTestOrFilter(name))) ) { // ns:f(...) consumeToken(); name += ":" + getToken().getImage(); @@ -292,7 +298,7 @@ protected AstNode nonliteral() throws ScanException, ParseException { depth--; } else if (depth == 0) { if (s == Symbol.COMMA) { - return new AstTuple(params()); + return createAstTuple(params()); } } s = lookahead(i++).getSymbol(); @@ -301,7 +307,7 @@ protected AstNode nonliteral() throws ScanException, ParseException { consumeToken(); v = expr(true); consumeToken(RPAREN); - v = new AstNested(v); + v = createAstNested(v); break; default: break; @@ -314,10 +320,11 @@ protected AstNode literal() throws ScanException, ParseException { AstNode v = null; switch (getToken().getSymbol()) { case LBRACK: - v = new AstList(params(LBRACK, RBRACK)); + v = createAstList(params(LBRACK, RBRACK)); + break; case LPAREN: - v = new AstTuple(params()); + v = createAstTuple(params()); break; case EXTENSION: if (getToken() == LITERAL_DICT_START) { @@ -337,6 +344,20 @@ protected AstNode literal() throws ScanException, ParseException { return super.literal(); } + protected AstRightValue createAstNested(AstNode node) { + return new AstNested(node); + } + + protected AstTuple createAstTuple(AstParameters parameters) + throws ScanException, ParseException { + return new AstTuple(parameters); + } + + protected AstList createAstList(AstParameters parameters) + throws ScanException, ParseException { + return new AstList(parameters); + } + protected AstRangeBracket createAstRangeBracket( AstNode base, AstNode rangeStart, @@ -424,7 +445,7 @@ protected AstNode value() throws ScanException, ParseException { "filter", true ); - v = createAstMethod(filterProperty, new AstParameters(filterParams)); // function("filter:" + filterName, new AstParameters(filterParams)); + v = createAstMethod(filterProperty, createAstParameters(filterParams)); // function("filter:" + filterName, new AstParameters(filterParams)); } while ("|".equals(getToken().getImage())); } else if ( "is".equals(getToken().getImage()) && @@ -447,10 +468,34 @@ protected AstNode value() throws ScanException, ParseException { } } + protected AstParameters createAstParameters(List nodes) { + return new AstParameters(nodes); + } + private boolean isPossibleExpTest(Symbol symbol) { return VALID_SYMBOLS_FOR_EXP_TEST.contains(symbol); } + private boolean isPossibleExpTestOrFilter(String namespace) + throws ParseException, ScanException { + if ( + FILTER_PREFIX.substring(0, FILTER_PREFIX.length() - 1).equals(namespace) || + EXPTEST_PREFIX.substring(0, EXPTEST_PREFIX.length() - 1).equals(namespace) && + lookahead(1).getSymbol() == DOT && + lookahead(2).getSymbol() == IDENTIFIER + ) { + Token property = lookahead(2); + if ( + "filter".equals(property.getImage()) || + "evaluate".equals(property.getImage()) || + "evaluateNegated".equals(property.getImage()) + ) { // exptest:equalto.evaluate(... + return lookahead(3).getSymbol() == LPAREN; + } + } + return false; + } + private AstNode buildAstMethodForIdentifier(AstNode astNode, String property) throws ScanException, ParseException { String exptestName = consumeToken().getImage(); @@ -467,7 +512,7 @@ private AstNode buildAstMethodForIdentifier(AstNode astNode, String property) property, true ); - return createAstMethod(exptestProperty, new AstParameters(exptestParams)); + return createAstMethod(exptestProperty, createAstParameters(exptestParams)); } @Override diff --git a/src/main/java/com/hubspot/jinjava/el/ext/NamedParameter.java b/src/main/java/com/hubspot/jinjava/el/ext/NamedParameter.java index 8720281a8..39170e65e 100644 --- a/src/main/java/com/hubspot/jinjava/el/ext/NamedParameter.java +++ b/src/main/java/com/hubspot/jinjava/el/ext/NamedParameter.java @@ -1,8 +1,9 @@ package com.hubspot.jinjava.el.ext; +import com.hubspot.jinjava.objects.serialization.PyishSerializable; import java.util.Objects; -public class NamedParameter { +public class NamedParameter implements PyishSerializable { private final String name; private final Object value; @@ -23,4 +24,9 @@ public Object getValue() { public String toString() { return Objects.toString(value, ""); } + + @Override + public String toPyishString() { + return name + "=" + PyishSerializable.writeValueAsString(value); + } } diff --git a/src/main/java/com/hubspot/jinjava/el/ext/NamedParameterOperator.java b/src/main/java/com/hubspot/jinjava/el/ext/NamedParameterOperator.java index 6f342243c..369995c80 100644 --- a/src/main/java/com/hubspot/jinjava/el/ext/NamedParameterOperator.java +++ b/src/main/java/com/hubspot/jinjava/el/ext/NamedParameterOperator.java @@ -1,5 +1,6 @@ package com.hubspot.jinjava.el.ext; +import com.hubspot.jinjava.el.ext.eager.EagerAstNamedParameter; import de.odysseus.el.tree.impl.Parser.ExtensionHandler; import de.odysseus.el.tree.impl.Parser.ExtensionPoint; import de.odysseus.el.tree.impl.Scanner; @@ -10,17 +11,21 @@ public class NamedParameterOperator { public static final Scanner.ExtensionToken TOKEN = new Scanner.ExtensionToken("="); - public static final ExtensionHandler HANDLER = new ExtensionHandler( - ExtensionPoint.ADD - ) { + public static final ExtensionHandler HANDLER = getHandler(false); - @Override - public AstNode createAstNode(AstNode... children) { - if (!(children[0] instanceof AstIdentifier)) { - throw new ELException("Expected IDENTIFIER, found " + children[0].toString()); + public static ExtensionHandler getHandler(boolean eager) { + return new ExtensionHandler(ExtensionPoint.ADD) { + + @Override + public AstNode createAstNode(AstNode... children) { + if (!(children[0] instanceof AstIdentifier)) { + throw new ELException("Expected IDENTIFIER, found " + children[0].toString()); + } + AstIdentifier name = (AstIdentifier) children[0]; + return eager + ? new EagerAstNamedParameter(name, children[1]) + : new AstNamedParameter(name, children[1]); } - AstIdentifier name = (AstIdentifier) children[0]; - return new AstNamedParameter(name, children[1]); - } - }; + }; + } } diff --git a/src/main/java/com/hubspot/jinjava/el/ext/OrOperator.java b/src/main/java/com/hubspot/jinjava/el/ext/OrOperator.java index 9aea5c10c..71429eb43 100644 --- a/src/main/java/com/hubspot/jinjava/el/ext/OrOperator.java +++ b/src/main/java/com/hubspot/jinjava/el/ext/OrOperator.java @@ -17,5 +17,10 @@ public Object eval(Bindings bindings, ELContext context, AstNode left, AstNode r return right.eval(bindings, context); } + @Override + public String toString() { + return "||"; + } + public static final OrOperator OP = new OrOperator(); } diff --git a/src/main/java/com/hubspot/jinjava/el/ext/PowerOfOperator.java b/src/main/java/com/hubspot/jinjava/el/ext/PowerOfOperator.java index e798e76f7..166a42cc8 100644 --- a/src/main/java/com/hubspot/jinjava/el/ext/PowerOfOperator.java +++ b/src/main/java/com/hubspot/jinjava/el/ext/PowerOfOperator.java @@ -1,5 +1,6 @@ package com.hubspot.jinjava.el.ext; +import com.hubspot.jinjava.el.ext.eager.EagerAstBinary; import de.odysseus.el.misc.TypeConverter; import de.odysseus.el.tree.impl.Parser.ExtensionHandler; import de.odysseus.el.tree.impl.Parser.ExtensionPoint; @@ -40,13 +41,22 @@ protected Object apply(TypeConverter converter, Object a, Object b) { ); } - public static final ExtensionHandler HANDLER = new ExtensionHandler( - ExtensionPoint.MUL - ) { + @Override + public String toString() { + return TOKEN.getImage(); + } - @Override - public AstNode createAstNode(AstNode... children) { - return new AstBinary(children[0], children[1], OP); - } - }; + public static final ExtensionHandler HANDLER = getHandler(false); + + public static ExtensionHandler getHandler(boolean eager) { + return new ExtensionHandler(ExtensionPoint.MUL) { + + @Override + public AstNode createAstNode(AstNode... children) { + return eager + ? new EagerAstBinary(children[0], children[1], OP) + : new AstBinary(children[0], children[1], OP); + } + }; + } } diff --git a/src/main/java/com/hubspot/jinjava/el/ext/StringConcatOperator.java b/src/main/java/com/hubspot/jinjava/el/ext/StringConcatOperator.java index 37b576ec0..d00073eee 100644 --- a/src/main/java/com/hubspot/jinjava/el/ext/StringConcatOperator.java +++ b/src/main/java/com/hubspot/jinjava/el/ext/StringConcatOperator.java @@ -1,5 +1,6 @@ package com.hubspot.jinjava.el.ext; +import com.hubspot.jinjava.el.ext.eager.EagerAstBinary; import de.odysseus.el.misc.TypeConverter; import de.odysseus.el.tree.impl.Parser.ExtensionHandler; import de.odysseus.el.tree.impl.Parser.ExtensionPoint; @@ -18,16 +19,25 @@ protected Object apply(TypeConverter converter, Object o1, Object o2) { return new StringBuilder(o1s).append(o2s).toString(); } + @Override + public String toString() { + return TOKEN.getImage(); + } + public static final Scanner.ExtensionToken TOKEN = new Scanner.ExtensionToken("~"); public static final StringConcatOperator OP = new StringConcatOperator(); - public static final ExtensionHandler HANDLER = new ExtensionHandler( - ExtensionPoint.ADD - ) { + public static final ExtensionHandler HANDLER = getHandler(false); - @Override - public AstNode createAstNode(AstNode... children) { - return new AstBinary(children[0], children[1], OP); - } - }; + public static ExtensionHandler getHandler(boolean eager) { + return new ExtensionHandler(ExtensionPoint.ADD) { + + @Override + public AstNode createAstNode(AstNode... children) { + return eager + ? new EagerAstBinary(children[0], children[1], OP) + : new AstBinary(children[0], children[1], OP); + } + }; + } } diff --git a/src/main/java/com/hubspot/jinjava/el/ext/TruncDivOperator.java b/src/main/java/com/hubspot/jinjava/el/ext/TruncDivOperator.java index 906ad64e2..d8b4fd146 100644 --- a/src/main/java/com/hubspot/jinjava/el/ext/TruncDivOperator.java +++ b/src/main/java/com/hubspot/jinjava/el/ext/TruncDivOperator.java @@ -1,5 +1,6 @@ package com.hubspot.jinjava.el.ext; +import com.hubspot.jinjava.el.ext.eager.EagerAstBinary; import de.odysseus.el.misc.TypeConverter; import de.odysseus.el.tree.impl.Parser.ExtensionHandler; import de.odysseus.el.tree.impl.Parser.ExtensionPoint; @@ -40,13 +41,22 @@ protected Object apply(TypeConverter converter, Object a, Object b) { ); } - public static final ExtensionHandler HANDLER = new ExtensionHandler( - ExtensionPoint.MUL - ) { + @Override + public String toString() { + return TOKEN.getImage(); + } - @Override - public AstNode createAstNode(AstNode... children) { - return new AstBinary(children[0], children[1], OP); - } - }; + public static final ExtensionHandler HANDLER = getHandler(false); + + public static ExtensionHandler getHandler(boolean eager) { + return new ExtensionHandler(ExtensionPoint.MUL) { + + @Override + public AstNode createAstNode(AstNode... children) { + return eager + ? new EagerAstBinary(children[0], children[1], OP) + : new AstBinary(children[0], children[1], OP); + } + }; + } } diff --git a/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstBinary.java b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstBinary.java new file mode 100644 index 000000000..cca2dfa27 --- /dev/null +++ b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstBinary.java @@ -0,0 +1,62 @@ +package com.hubspot.jinjava.el.ext.eager; + +import com.hubspot.jinjava.el.ext.DeferredParsingException; +import de.odysseus.el.tree.Bindings; +import de.odysseus.el.tree.impl.ast.AstBinary; +import de.odysseus.el.tree.impl.ast.AstNode; +import javax.el.ELContext; + +public class EagerAstBinary extends AstBinary implements EvalResultHolder { + protected Object evalResult; + protected final EvalResultHolder left; + protected final EvalResultHolder right; + protected final Operator operator; + + public EagerAstBinary(AstNode left, AstNode right, Operator operator) { + this( + EagerAstNodeDecorator.getAsEvalResultHolder(left), + EagerAstNodeDecorator.getAsEvalResultHolder(right), + operator + ); + } + + private EagerAstBinary( + EvalResultHolder left, + EvalResultHolder right, + Operator operator + ) { + super((AstNode) left, (AstNode) right, operator); + this.left = left; + this.right = right; + this.operator = operator; + } + + @Override + public Object eval(Bindings bindings, ELContext context) { + try { + evalResult = super.eval(bindings, context); + return evalResult; + } catch (DeferredParsingException e) { + String sb = + EvalResultHolder.reconstructNode(bindings, context, left, e, false) + + String.format(" %s ", operator.toString()) + + EvalResultHolder.reconstructNode(bindings, context, right, e, false); + throw new DeferredParsingException(this, sb); + } finally { + left.getAndClearEvalResult(); + right.getAndClearEvalResult(); + } + } + + @Override + public Object getAndClearEvalResult() { + Object temp = evalResult; + evalResult = null; + return temp; + } + + @Override + public boolean hasEvalResult() { + return evalResult != null; + } +} diff --git a/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstBracket.java b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstBracket.java new file mode 100644 index 000000000..544baeee5 --- /dev/null +++ b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstBracket.java @@ -0,0 +1,82 @@ +package com.hubspot.jinjava.el.ext.eager; + +import com.hubspot.jinjava.el.ext.DeferredParsingException; +import com.hubspot.jinjava.util.ChunkResolver; +import de.odysseus.el.tree.Bindings; +import de.odysseus.el.tree.impl.ast.AstBracket; +import de.odysseus.el.tree.impl.ast.AstNode; +import javax.el.ELContext; + +public class EagerAstBracket extends AstBracket implements EvalResultHolder { + protected Object evalResult; + + public EagerAstBracket( + AstNode base, + AstNode property, + boolean lvalue, + boolean strict, + boolean ignoreReturnType + ) { + super( + (AstNode) EagerAstNodeDecorator.getAsEvalResultHolder(base), + (AstNode) EagerAstNodeDecorator.getAsEvalResultHolder(property), + lvalue, + strict, + ignoreReturnType + ); + } + + @Override + public Object eval(Bindings bindings, ELContext context) { + try { + evalResult = super.eval(bindings, context); + return evalResult; + } catch (DeferredParsingException e) { + StringBuilder sb = new StringBuilder(); + if (((EvalResultHolder) prefix).hasEvalResult()) { + sb.append( + ChunkResolver.getValueAsJinjavaStringSafe( + ((EvalResultHolder) prefix).getAndClearEvalResult() + ) + ); + sb.append(String.format("[%s]", e.getDeferredEvalResult())); + } else { + sb.append(e.getDeferredEvalResult()); + try { + sb.append( + String.format( + "[%s]", + ChunkResolver.getValueAsJinjavaStringSafe(property.eval(bindings, context)) + ) + ); + } catch (DeferredParsingException e1) { + sb.append(String.format("[%s]", e1.getDeferredEvalResult())); + } + } + throw new DeferredParsingException(this, sb.toString()); + } finally { + ((EvalResultHolder) prefix).getAndClearEvalResult(); + ((EvalResultHolder) property).getAndClearEvalResult(); + } + } + + @Override + public Object getAndClearEvalResult() { + Object temp = evalResult; + evalResult = null; + return temp; + } + + @Override + public boolean hasEvalResult() { + return evalResult != null; + } + + public AstNode getPrefix() { + return prefix; + } + + public AstNode getMethod() { + return property; + } +} diff --git a/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstChoice.java b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstChoice.java new file mode 100644 index 000000000..861800507 --- /dev/null +++ b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstChoice.java @@ -0,0 +1,70 @@ +package com.hubspot.jinjava.el.ext.eager; + +import com.hubspot.jinjava.el.ext.DeferredParsingException; +import de.odysseus.el.tree.Bindings; +import de.odysseus.el.tree.impl.ast.AstChoice; +import de.odysseus.el.tree.impl.ast.AstNode; +import javax.el.ELContext; +import javax.el.ELException; + +public class EagerAstChoice extends AstChoice implements EvalResultHolder { + protected Object evalResult; + protected final EvalResultHolder question; + protected final EvalResultHolder yes; + protected final EvalResultHolder no; + + public EagerAstChoice(AstNode question, AstNode yes, AstNode no) { + this( + EagerAstNodeDecorator.getAsEvalResultHolder(question), + EagerAstNodeDecorator.getAsEvalResultHolder(yes), + EagerAstNodeDecorator.getAsEvalResultHolder(no) + ); + } + + private EagerAstChoice( + EvalResultHolder question, + EvalResultHolder yes, + EvalResultHolder no + ) { + super((AstNode) question, (AstNode) yes, (AstNode) no); + this.question = question; + this.yes = yes; + this.no = no; + } + + @Override + public Object eval(Bindings bindings, ELContext context) throws ELException { + try { + evalResult = super.eval(bindings, context); + return evalResult; + } catch (DeferredParsingException e) { + if (question.getAndClearEvalResult() != null) { + // the question was evaluated so jump to either yes or no + throw new DeferredParsingException(this, e.getDeferredEvalResult()); + } + String sb = + e.getDeferredEvalResult() + + " ? " + + EvalResultHolder.reconstructNode(bindings, context, yes, e, false) + + " : " + + EvalResultHolder.reconstructNode(bindings, context, no, e, false); + throw new DeferredParsingException(this, sb); + } finally { + question.getAndClearEvalResult(); + yes.getAndClearEvalResult(); + no.getAndClearEvalResult(); + } + } + + @Override + public Object getAndClearEvalResult() { + Object temp = evalResult; + evalResult = null; + return temp; + } + + @Override + public boolean hasEvalResult() { + return evalResult != null; + } +} diff --git a/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstDict.java b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstDict.java new file mode 100644 index 000000000..d45985574 --- /dev/null +++ b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstDict.java @@ -0,0 +1,88 @@ +package com.hubspot.jinjava.el.ext.eager; + +import com.hubspot.jinjava.el.ext.AstDict; +import com.hubspot.jinjava.el.ext.DeferredParsingException; +import com.hubspot.jinjava.el.ext.ExtendedParser; +import com.hubspot.jinjava.interpret.JinjavaInterpreter; +import de.odysseus.el.tree.Bindings; +import de.odysseus.el.tree.impl.ast.AstNode; +import java.util.Map; +import java.util.StringJoiner; +import javax.el.ELContext; + +public class EagerAstDict extends AstDict implements EvalResultHolder { + private Object evalResult; + + public EagerAstDict(Map dict) { + super(dict); + } + + @Override + public Object eval(Bindings bindings, ELContext context) { + try { + evalResult = super.eval(bindings, context); + return evalResult; + } catch (DeferredParsingException e) { + JinjavaInterpreter interpreter = (JinjavaInterpreter) context + .getELResolver() + .getValue(context, null, ExtendedParser.INTERPRETER); + StringJoiner joiner = new StringJoiner(", "); + dict.forEach( + (key, value) -> { + StringJoiner kvJoiner = new StringJoiner(": "); + if (key instanceof EvalResultHolder) { + kvJoiner.add( + EvalResultHolder.reconstructNode( + bindings, + context, + (EvalResultHolder) key, + e, + !interpreter.getConfig().getLegacyOverrides().isEvaluateMapKeys() + ) + ); + } else { + kvJoiner.add(key.toString()); + } + if (value instanceof EvalResultHolder) { + kvJoiner.add( + EvalResultHolder.reconstructNode( + bindings, + context, + (EvalResultHolder) value, + e, + false + ) + ); + } else { + kvJoiner.add(value.toString()); + } + joiner.add(kvJoiner.toString()); + } + ); + throw new DeferredParsingException(this, String.format("{%s}", joiner.toString())); + } finally { + dict.forEach( + (key, value) -> { + if (key instanceof EvalResultHolder) { + ((EvalResultHolder) key).getAndClearEvalResult(); + } + if (value instanceof EvalResultHolder) { + ((EvalResultHolder) value).getAndClearEvalResult(); + } + } + ); + } + } + + @Override + public Object getAndClearEvalResult() { + Object temp = evalResult; + evalResult = null; + return temp; + } + + @Override + public boolean hasEvalResult() { + return evalResult != null; + } +} diff --git a/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstDot.java b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstDot.java new file mode 100644 index 000000000..1c88a544b --- /dev/null +++ b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstDot.java @@ -0,0 +1,74 @@ +package com.hubspot.jinjava.el.ext.eager; + +import com.hubspot.jinjava.el.ext.DeferredParsingException; +import de.odysseus.el.tree.Bindings; +import de.odysseus.el.tree.impl.ast.AstDot; +import de.odysseus.el.tree.impl.ast.AstNode; +import javax.el.ELContext; +import javax.el.ELException; + +public class EagerAstDot extends AstDot implements EvalResultHolder { + private Object evalResult; + private final EvalResultHolder base; + private final String property; + + public EagerAstDot( + AstNode base, + String property, + boolean lvalue, + boolean ignoreReturnType + ) { + this( + EagerAstNodeDecorator.getAsEvalResultHolder(base), + property, + lvalue, + ignoreReturnType + ); + } + + public EagerAstDot( + EvalResultHolder base, + String property, + boolean lvalue, + boolean ignoreReturnType + ) { + super((AstNode) base, property, lvalue, ignoreReturnType); + this.base = base; + this.property = property; + } + + @Override + public Object eval(Bindings bindings, ELContext context) throws ELException { + try { + evalResult = super.eval(bindings, context); + return evalResult; + } catch (DeferredParsingException e) { + throw new DeferredParsingException( + this, + String.format("%s.%s", e.getDeferredEvalResult(), this.property) + ); + } finally { + base.getAndClearEvalResult(); + } + } + + @Override + public Object getAndClearEvalResult() { + Object temp = evalResult; + evalResult = null; + return temp; + } + + @Override + public boolean hasEvalResult() { + return evalResult != null; + } + + public AstNode getPrefix() { + return prefix; + } + + public String getProperty() { + return property; + } +} diff --git a/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstIdentifier.java b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstIdentifier.java new file mode 100644 index 000000000..a9a1580a8 --- /dev/null +++ b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstIdentifier.java @@ -0,0 +1,31 @@ +package com.hubspot.jinjava.el.ext.eager; + +import de.odysseus.el.tree.Bindings; +import de.odysseus.el.tree.impl.ast.AstIdentifier; +import javax.el.ELContext; + +public class EagerAstIdentifier extends AstIdentifier implements EvalResultHolder { + private Object evalResult; + + public EagerAstIdentifier(String name, int index, boolean ignoreReturnType) { + super(name, index, ignoreReturnType); + } + + @Override + public Object eval(Bindings bindings, ELContext context) { + evalResult = super.eval(bindings, context); + return evalResult; + } + + @Override + public Object getAndClearEvalResult() { + Object temp = evalResult; + evalResult = null; + return temp; + } + + @Override + public boolean hasEvalResult() { + return evalResult != null; + } +} diff --git a/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstList.java b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstList.java new file mode 100644 index 000000000..b6179c2f5 --- /dev/null +++ b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstList.java @@ -0,0 +1,54 @@ +package com.hubspot.jinjava.el.ext.eager; + +import com.hubspot.jinjava.el.ext.AstList; +import com.hubspot.jinjava.el.ext.DeferredParsingException; +import de.odysseus.el.tree.Bindings; +import de.odysseus.el.tree.impl.ast.AstParameters; +import java.util.StringJoiner; +import javax.el.ELContext; + +public class EagerAstList extends AstList implements EvalResultHolder { + private Object evalResult; + + public EagerAstList(AstParameters elements) { + super(elements); + } + + @Override + public Object eval(Bindings bindings, ELContext context) { + try { + evalResult = super.eval(bindings, context); + return evalResult; + } catch (DeferredParsingException e) { + StringJoiner joiner = new StringJoiner(", "); + for (int i = 0; i < elements.getCardinality(); i++) { + joiner.add( + EvalResultHolder.reconstructNode( + bindings, + context, + (EvalResultHolder) elements.getChild(i), + e, + false + ) + ); + } + throw new DeferredParsingException(this, "[" + joiner.toString() + "]"); + } finally { + for (int i = 0; i < elements.getCardinality(); i++) { + ((EvalResultHolder) elements.getChild(i)).getAndClearEvalResult(); + } + } + } + + @Override + public Object getAndClearEvalResult() { + Object temp = evalResult; + evalResult = null; + return temp; + } + + @Override + public boolean hasEvalResult() { + return evalResult != null; + } +} diff --git a/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstMacroFunction.java b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstMacroFunction.java new file mode 100644 index 000000000..d795fe009 --- /dev/null +++ b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstMacroFunction.java @@ -0,0 +1,73 @@ +package com.hubspot.jinjava.el.ext.eager; + +import com.hubspot.jinjava.el.ext.AstMacroFunction; +import com.hubspot.jinjava.el.ext.DeferredParsingException; +import com.hubspot.jinjava.interpret.DeferredValueException; +import com.hubspot.jinjava.util.ChunkResolver; +import de.odysseus.el.tree.Bindings; +import de.odysseus.el.tree.impl.ast.AstParameters; +import java.util.Arrays; +import java.util.stream.Collectors; +import javax.el.ELContext; + +public class EagerAstMacroFunction extends AstMacroFunction implements EvalResultHolder { + protected Object evalResult; + // instanceof AstParameters + protected EvalResultHolder params; + + public EagerAstMacroFunction( + String name, + int index, + AstParameters params, + boolean varargs + ) { + this(name, index, EagerAstNodeDecorator.getAsEvalResultHolder(params), varargs); + } + + private EagerAstMacroFunction( + String name, + int index, + EvalResultHolder params, + boolean varargs + ) { + super(name, index, (AstParameters) params, varargs); + this.params = params; + } + + @Override + public Object eval(Bindings bindings, ELContext context) { + try { + evalResult = super.eval(bindings, context); + return evalResult; + } catch (DeferredValueException e) { + StringBuilder sb = new StringBuilder(); + sb.append(getName()); + String paramString; + try { + paramString = + Arrays + .stream(((AstParameters) params).eval(bindings, context)) + .map(ChunkResolver::getValueAsJinjavaStringSafe) + .collect(Collectors.joining(", ")); + } catch (DeferredParsingException dpe) { + paramString = dpe.getDeferredEvalResult(); + } + sb.append(String.format("(%s)", paramString)); + throw new DeferredParsingException(this, sb.toString()); + } finally { + params.getAndClearEvalResult(); + } + } + + @Override + public Object getAndClearEvalResult() { + Object temp = evalResult; + evalResult = null; + return temp; + } + + @Override + public boolean hasEvalResult() { + return evalResult != null; + } +} diff --git a/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstMethod.java b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstMethod.java new file mode 100644 index 000000000..bae88b031 --- /dev/null +++ b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstMethod.java @@ -0,0 +1,122 @@ +package com.hubspot.jinjava.el.ext.eager; + +import com.hubspot.jinjava.el.ext.DeferredParsingException; +import com.hubspot.jinjava.interpret.DeferredValueException; +import com.hubspot.jinjava.util.ChunkResolver; +import de.odysseus.el.tree.Bindings; +import de.odysseus.el.tree.impl.ast.AstMethod; +import de.odysseus.el.tree.impl.ast.AstNode; +import de.odysseus.el.tree.impl.ast.AstParameters; +import de.odysseus.el.tree.impl.ast.AstProperty; +import javax.el.ELContext; + +public class EagerAstMethod extends AstMethod implements EvalResultHolder { + private Object evalResult; + // instanceof AstProperty + protected final EvalResultHolder property; + // instanceof AstParameters + protected final EvalResultHolder params; + + public EagerAstMethod(AstProperty property, AstParameters params) { + this( + EagerAstNodeDecorator.getAsEvalResultHolder(property), + EagerAstNodeDecorator.getAsEvalResultHolder(params) + ); + } + + private EagerAstMethod(EvalResultHolder property, EvalResultHolder params) { + super((AstProperty) property, (AstParameters) params); + this.property = property; + this.params = params; + } + + @Override + public Object eval(Bindings bindings, ELContext context) { + try { + evalResult = super.eval(bindings, context); + return evalResult; + } catch (DeferredParsingException e) { + throw new DeferredParsingException( + this, + getPartiallyResolved(bindings, context, e) + ); + } finally { + property.getAndClearEvalResult(); + params.getAndClearEvalResult(); + } + } + + @Override + public Object getAndClearEvalResult() { + Object temp = evalResult; + evalResult = null; + return temp; + } + + @Override + public boolean hasEvalResult() { + return evalResult != null; + } + + /** + * This method is used when we need to reconstruct the method property and params manually. + * Neither the property or params could be evaluated so we dive into the property and figure out + * where the DeferredParsingException came from. + */ + private String getPartiallyResolved( + Bindings bindings, + ELContext context, + DeferredParsingException deferredParsingException + ) { + String stringPrefix; + String stringMethod; + AstNode prefix; + String formatString; + if (property instanceof EagerAstDot) { + formatString = "%s.%s"; + prefix = ((EagerAstDot) property).getPrefix(); + stringMethod = ((EagerAstDot) property).getProperty(); + } else if (property instanceof EagerAstBracket) { + formatString = "%s[%s]"; + prefix = ((EagerAstBracket) property).getPrefix(); + stringMethod = + EvalResultHolder.reconstructNode( + bindings, + context, + (EvalResultHolder) ((EagerAstBracket) property).getMethod(), + deferredParsingException, + false + ); + } else { // Should not happen natively + throw new DeferredValueException("Cannot resolve property in EagerAstMethod"); + } + + // If prefix is an identifier, then preserve it in case the method should modify it. + stringPrefix = + EvalResultHolder.reconstructNode( + bindings, + context, + (EvalResultHolder) prefix, + deferredParsingException, + true + ); + String paramString; + if (deferredParsingException.getSourceNode() == params) { + paramString = deferredParsingException.getDeferredEvalResult(); + } else { + try { + paramString = + ChunkResolver.getValueAsJinjavaStringSafe(params.eval(bindings, context)); + // remove brackets so they can get replaced with parentheses + paramString = paramString.substring(1, paramString.length() - 1); + } catch (DeferredParsingException e) { + paramString = e.getDeferredEvalResult(); + } + } + + return ( + String.format(formatString, stringPrefix, stringMethod) + + String.format("(%s)", paramString) + ); + } +} diff --git a/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstNamedParameter.java b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstNamedParameter.java new file mode 100644 index 000000000..b7b083371 --- /dev/null +++ b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstNamedParameter.java @@ -0,0 +1,53 @@ +package com.hubspot.jinjava.el.ext.eager; + +import com.hubspot.jinjava.el.ext.AstNamedParameter; +import com.hubspot.jinjava.el.ext.DeferredParsingException; +import de.odysseus.el.tree.Bindings; +import de.odysseus.el.tree.impl.ast.AstIdentifier; +import de.odysseus.el.tree.impl.ast.AstNode; +import javax.el.ELContext; + +public class EagerAstNamedParameter + extends AstNamedParameter + implements EvalResultHolder { + private Object evalResult; + private final AstIdentifier name; + private final EvalResultHolder value; + + public EagerAstNamedParameter(AstIdentifier name, AstNode value) { + this(name, EagerAstNodeDecorator.getAsEvalResultHolder(value)); + } + + private EagerAstNamedParameter(AstIdentifier name, EvalResultHolder value) { + super(name, (AstNode) value); + this.name = name; + this.value = value; + } + + @Override + public Object eval(Bindings bindings, ELContext context) { + try { + evalResult = super.eval(bindings, context); + return evalResult; + } catch (DeferredParsingException e) { + throw new DeferredParsingException( + this, + String.format("%s=%s", name, e.getDeferredEvalResult()) + ); + } finally { + value.getAndClearEvalResult(); + } + } + + @Override + public Object getAndClearEvalResult() { + Object temp = evalResult; + evalResult = null; + return temp; + } + + @Override + public boolean hasEvalResult() { + return evalResult != null; + } +} diff --git a/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstNested.java b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstNested.java new file mode 100644 index 000000000..9760f4906 --- /dev/null +++ b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstNested.java @@ -0,0 +1,69 @@ +package com.hubspot.jinjava.el.ext.eager; + +import com.hubspot.jinjava.el.ext.DeferredParsingException; +import de.odysseus.el.tree.Bindings; +import de.odysseus.el.tree.Node; +import de.odysseus.el.tree.impl.ast.AstNode; +import de.odysseus.el.tree.impl.ast.AstRightValue; +import javax.el.ELContext; + +/** + * AstNested is final so this decorates AstRightValue. + */ +public class EagerAstNested extends AstRightValue implements EvalResultHolder { + private Object evalResult; + private final AstNode child; + + public EagerAstNested(AstNode child) { + this.child = child; + } + + @Override + public void appendStructure(StringBuilder builder, Bindings bindings) { + builder.append("("); + child.appendStructure(builder, bindings); + builder.append(")"); + } + + @Override + public Object eval(Bindings bindings, ELContext context) { + try { + evalResult = child.eval(bindings, context); + return evalResult; + } catch (DeferredParsingException e) { + throw new DeferredParsingException( + this, + String.format("(%s)", e.getDeferredEvalResult()) + ); + } finally { + ((EvalResultHolder) child).getAndClearEvalResult(); + } + } + + @Override + public String toString() { + return "(...)"; + } + + @Override + public Object getAndClearEvalResult() { + Object temp = evalResult; + evalResult = null; + return temp; + } + + @Override + public boolean hasEvalResult() { + return evalResult != null; + } + + @Override + public int getCardinality() { + return 1; + } + + @Override + public Node getChild(int i) { + return i == 0 ? child : null; + } +} diff --git a/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstNodeDecorator.java b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstNodeDecorator.java new file mode 100644 index 000000000..37912cce0 --- /dev/null +++ b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstNodeDecorator.java @@ -0,0 +1,125 @@ +package com.hubspot.jinjava.el.ext.eager; + +import de.odysseus.el.tree.Bindings; +import de.odysseus.el.tree.Node; +import de.odysseus.el.tree.impl.ast.AstNode; +import javax.el.ELContext; +import javax.el.MethodInfo; +import javax.el.ValueReference; + +/** + * This decorator exists to ensure that every EvalResultHolder is an + * instanceof AstNode. When using eager parsing, every AstNode should either + * be an EvalResultHolder or wrapped with this decorator. + */ +public class EagerAstNodeDecorator extends AstNode implements EvalResultHolder { + private final AstNode astNode; + private Object evalResult; + + public static EvalResultHolder getAsEvalResultHolder(AstNode astNode) { + if (astNode instanceof EvalResultHolder) { + return (EvalResultHolder) astNode; + } + if (astNode != null) { // Wraps nodes such as AstString, AstNumber + return new EagerAstNodeDecorator(astNode); + } + return null; + } + + private EagerAstNodeDecorator(AstNode astNode) { + this.astNode = astNode; + } + + @Override + public Object getAndClearEvalResult() { + Object temp = evalResult; + evalResult = null; + return temp; + } + + @Override + public boolean hasEvalResult() { + return evalResult != null; + } + + @Override + public void appendStructure(StringBuilder stringBuilder, Bindings bindings) { + astNode.appendStructure(stringBuilder, bindings); + } + + @Override + public Object eval(Bindings bindings, ELContext elContext) { + evalResult = astNode.eval(bindings, elContext); + return evalResult; + } + + @Override + public boolean isLiteralText() { + return astNode.isLiteralText(); + } + + @Override + public boolean isLeftValue() { + return astNode.isLeftValue(); + } + + @Override + public boolean isMethodInvocation() { + return astNode.isMethodInvocation(); + } + + @Override + public ValueReference getValueReference(Bindings bindings, ELContext elContext) { + return astNode.getValueReference(bindings, elContext); + } + + @Override + public Class getType(Bindings bindings, ELContext elContext) { + return astNode.getType(bindings, elContext); + } + + @Override + public boolean isReadOnly(Bindings bindings, ELContext elContext) { + return astNode.isReadOnly(bindings, elContext); + } + + @Override + public void setValue(Bindings bindings, ELContext elContext, Object o) { + astNode.setValue(bindings, elContext, o); + } + + @Override + public MethodInfo getMethodInfo( + Bindings bindings, + ELContext elContext, + Class aClass, + Class[] classes + ) { + return astNode.getMethodInfo(bindings, elContext, aClass, classes); + } + + @Override + public Object invoke( + Bindings bindings, + ELContext elContext, + Class aClass, + Class[] classes, + Object[] objects + ) { + if (evalResult != null) { + return evalResult; + } + evalResult = astNode.invoke(bindings, elContext, aClass, classes, objects); + return evalResult; + } + + @Override + public int getCardinality() { + return astNode.getCardinality(); + } + + @Override + public Node getChild(int i) { + return astNode.getChild(i); + } +} diff --git a/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstParameters.java b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstParameters.java new file mode 100644 index 000000000..c937e2791 --- /dev/null +++ b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstParameters.java @@ -0,0 +1,65 @@ +package com.hubspot.jinjava.el.ext.eager; + +import com.hubspot.jinjava.el.ext.DeferredParsingException; +import de.odysseus.el.tree.Bindings; +import de.odysseus.el.tree.impl.ast.AstNode; +import de.odysseus.el.tree.impl.ast.AstParameters; +import java.util.List; +import java.util.StringJoiner; +import java.util.stream.Collectors; +import javax.el.ELContext; + +public class EagerAstParameters extends AstParameters implements EvalResultHolder { + private Object[] evalResult; + private final List nodes; + + public EagerAstParameters(List nodes) { + this( // to avoid converting nodes twice, call separate constructor + nodes + .stream() + .map(EagerAstNodeDecorator::getAsEvalResultHolder) + .map(e -> (AstNode) e) + .collect(Collectors.toList()), + true + ); + } + + private EagerAstParameters(List nodes, boolean convertedToEvalResultHolder) { + super(nodes); + this.nodes = nodes; + } + + @Override + public Object[] eval(Bindings bindings, ELContext context) { + try { + evalResult = super.eval(bindings, context); + return evalResult; + } catch (DeferredParsingException e) { + StringJoiner joiner = new StringJoiner(", "); + nodes + .stream() + .map(node -> (EvalResultHolder) node) + .forEach( + node -> + joiner.add( + EvalResultHolder.reconstructNode(bindings, context, node, e, false) + ) + ); + throw new DeferredParsingException(this, joiner.toString()); + } finally { + nodes.forEach(node -> ((EvalResultHolder) node).getAndClearEvalResult()); + } + } + + @Override + public Object[] getAndClearEvalResult() { + Object[] temp = evalResult; + evalResult = null; + return temp; + } + + @Override + public boolean hasEvalResult() { + return evalResult != null; + } +} diff --git a/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstRangeBracket.java b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstRangeBracket.java new file mode 100644 index 000000000..97998ea26 --- /dev/null +++ b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstRangeBracket.java @@ -0,0 +1,86 @@ +package com.hubspot.jinjava.el.ext.eager; + +import com.hubspot.jinjava.el.ext.AstRangeBracket; +import com.hubspot.jinjava.el.ext.DeferredParsingException; +import de.odysseus.el.tree.Bindings; +import de.odysseus.el.tree.impl.ast.AstNode; +import javax.el.ELContext; + +public class EagerAstRangeBracket extends AstRangeBracket implements EvalResultHolder { + protected Object evalResult; + + public EagerAstRangeBracket( + AstNode base, + AstNode rangeStart, + AstNode rangeMax, + boolean lvalue, + boolean strict, + boolean ignoreReturnType + ) { + super( + (AstNode) EagerAstNodeDecorator.getAsEvalResultHolder(base), + (AstNode) EagerAstNodeDecorator.getAsEvalResultHolder(rangeStart), + (AstNode) EagerAstNodeDecorator.getAsEvalResultHolder(rangeMax), + lvalue, + strict, + ignoreReturnType + ); + } + + @Override + public Object eval(Bindings bindings, ELContext context) { + try { + evalResult = super.eval(bindings, context); + return evalResult; + } catch (DeferredParsingException e) { + String sb = + EvalResultHolder.reconstructNode( + bindings, + context, + (EvalResultHolder) prefix, + e, + false + ) + + "[" + + EvalResultHolder.reconstructNode( + bindings, + context, + (EvalResultHolder) property, + e, + false + ) + + ":" + + EvalResultHolder.reconstructNode( + bindings, + context, + (EvalResultHolder) rangeMax, + e, + false + ) + + "]"; + throw new DeferredParsingException(this, sb); + } finally { + if (prefix != null) { + ((EvalResultHolder) prefix).getAndClearEvalResult(); + } + if (property != null) { + ((EvalResultHolder) property).getAndClearEvalResult(); + } + if (rangeMax != null) { + ((EvalResultHolder) rangeMax).getAndClearEvalResult(); + } + } + } + + @Override + public Object getAndClearEvalResult() { + Object temp = evalResult; + evalResult = null; + return temp; + } + + @Override + public boolean hasEvalResult() { + return evalResult != null; + } +} diff --git a/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstTuple.java b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstTuple.java new file mode 100644 index 000000000..d40f2977b --- /dev/null +++ b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstTuple.java @@ -0,0 +1,54 @@ +package com.hubspot.jinjava.el.ext.eager; + +import com.hubspot.jinjava.el.ext.AstTuple; +import com.hubspot.jinjava.el.ext.DeferredParsingException; +import de.odysseus.el.tree.Bindings; +import de.odysseus.el.tree.impl.ast.AstParameters; +import java.util.StringJoiner; +import javax.el.ELContext; + +public class EagerAstTuple extends AstTuple implements EvalResultHolder { + private Object evalResult; + + public EagerAstTuple(AstParameters elements) { + super(elements); + } + + @Override + public Object eval(Bindings bindings, ELContext context) { + try { + evalResult = super.eval(bindings, context); + return evalResult; + } catch (DeferredParsingException e) { + StringJoiner joiner = new StringJoiner(", "); + for (int i = 0; i < elements.getCardinality(); i++) { + joiner.add( + EvalResultHolder.reconstructNode( + bindings, + context, + (EvalResultHolder) elements.getChild(i), + e, + false + ) + ); + } + throw new DeferredParsingException(this, "(" + joiner.toString() + ")"); + } finally { + for (int i = 0; i < elements.getCardinality(); i++) { + ((EvalResultHolder) elements.getChild(i)).getAndClearEvalResult(); + } + } + } + + @Override + public Object getAndClearEvalResult() { + Object temp = evalResult; + evalResult = null; + return temp; + } + + @Override + public boolean hasEvalResult() { + return evalResult != null; + } +} diff --git a/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstUnary.java b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstUnary.java new file mode 100644 index 000000000..9d20b3149 --- /dev/null +++ b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerAstUnary.java @@ -0,0 +1,50 @@ +package com.hubspot.jinjava.el.ext.eager; + +import com.hubspot.jinjava.el.ext.DeferredParsingException; +import de.odysseus.el.tree.Bindings; +import de.odysseus.el.tree.impl.ast.AstNode; +import de.odysseus.el.tree.impl.ast.AstUnary; +import javax.el.ELContext; + +public class EagerAstUnary extends AstUnary implements EvalResultHolder { + private Object evalResult; + protected final EvalResultHolder child; + protected final Operator operator; + + public EagerAstUnary(AstNode child, Operator operator) { + this(EagerAstNodeDecorator.getAsEvalResultHolder(child), operator); + } + + private EagerAstUnary(EvalResultHolder child, Operator operator) { + super((AstNode) child, operator); + this.child = child; + this.operator = operator; + } + + @Override + public Object eval(Bindings bindings, ELContext context) { + try { + evalResult = super.eval(bindings, context); + return evalResult; + } catch (DeferredParsingException e) { + String sb = + operator.toString() + + EvalResultHolder.reconstructNode(bindings, context, child, e, false); + throw new DeferredParsingException(this, sb); + } finally { + child.getAndClearEvalResult(); + } + } + + @Override + public Object getAndClearEvalResult() { + Object temp = evalResult; + evalResult = null; + return temp; + } + + @Override + public boolean hasEvalResult() { + return evalResult != null; + } +} diff --git a/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerExtendedParser.java b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerExtendedParser.java new file mode 100644 index 000000000..97805dd0d --- /dev/null +++ b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerExtendedParser.java @@ -0,0 +1,174 @@ +package com.hubspot.jinjava.el.ext.eager; + +import com.hubspot.jinjava.el.ext.AbsOperator; +import com.hubspot.jinjava.el.ext.AstDict; +import com.hubspot.jinjava.el.ext.AstList; +import com.hubspot.jinjava.el.ext.AstRangeBracket; +import com.hubspot.jinjava.el.ext.AstTuple; +import com.hubspot.jinjava.el.ext.CollectionMembershipOperator; +import com.hubspot.jinjava.el.ext.ExtendedParser; +import com.hubspot.jinjava.el.ext.NamedParameterOperator; +import com.hubspot.jinjava.el.ext.PowerOfOperator; +import com.hubspot.jinjava.el.ext.StringConcatOperator; +import com.hubspot.jinjava.el.ext.TruncDivOperator; +import de.odysseus.el.tree.impl.Builder; +import de.odysseus.el.tree.impl.Builder.Feature; +import de.odysseus.el.tree.impl.Scanner.ScanException; +import de.odysseus.el.tree.impl.ast.AstBinary; +import de.odysseus.el.tree.impl.ast.AstBinary.Operator; +import de.odysseus.el.tree.impl.ast.AstBracket; +import de.odysseus.el.tree.impl.ast.AstChoice; +import de.odysseus.el.tree.impl.ast.AstDot; +import de.odysseus.el.tree.impl.ast.AstFunction; +import de.odysseus.el.tree.impl.ast.AstIdentifier; +import de.odysseus.el.tree.impl.ast.AstMethod; +import de.odysseus.el.tree.impl.ast.AstNode; +import de.odysseus.el.tree.impl.ast.AstParameters; +import de.odysseus.el.tree.impl.ast.AstProperty; +import de.odysseus.el.tree.impl.ast.AstRightValue; +import de.odysseus.el.tree.impl.ast.AstUnary; +import java.util.List; +import java.util.Map; + +public class EagerExtendedParser extends ExtendedParser { + + public EagerExtendedParser(Builder context, String input) { + super(context, input); + putExtensionHandler(AbsOperator.TOKEN, AbsOperator.getHandler(true)); + putExtensionHandler( + NamedParameterOperator.TOKEN, + NamedParameterOperator.getHandler(true) + ); + putExtensionHandler( + StringConcatOperator.TOKEN, + StringConcatOperator.getHandler(true) + ); + putExtensionHandler(TruncDivOperator.TOKEN, TruncDivOperator.getHandler(true)); + putExtensionHandler(PowerOfOperator.TOKEN, PowerOfOperator.getHandler(true)); + + putExtensionHandler( + CollectionMembershipOperator.TOKEN, + CollectionMembershipOperator.getHandler(true) + ); + } + + @Override + protected AstBinary createAstBinary(AstNode left, AstNode right, Operator operator) { + return new EagerAstBinary(left, right, operator); + } + + @Override + protected AstBracket createAstBracket( + AstNode base, + AstNode property, + boolean lvalue, + boolean strict + ) { + return new EagerAstBracket( + base, + property, + lvalue, + strict, + this.context.isEnabled(Feature.IGNORE_RETURN_TYPE) + ); + } + + @Override + protected AstFunction createAstFunction(String name, int index, AstParameters params) { + return new EagerAstMacroFunction( + name, + index, + params, + context.isEnabled(Feature.VARARGS) + ); + } + + @Override + protected AstChoice createAstChoice(AstNode question, AstNode yes, AstNode no) { + return new EagerAstChoice(question, yes, no); + } + + // @Override + // protected AstComposite createAstComposite(List nodes) { + // return new AstComposite(nodes); + // } + + @Override + protected AstDot createAstDot(AstNode base, String property, boolean lvalue) { + return new EagerAstDot( + base, + property, + lvalue, + this.context.isEnabled(Feature.IGNORE_RETURN_TYPE) + ); + } + + @Override + protected AstIdentifier createAstIdentifier(String name, int index) { + return new EagerAstIdentifier( + name, + index, + this.context.isEnabled(Feature.IGNORE_RETURN_TYPE) + ); + } + + @Override + protected AstMethod createAstMethod(AstProperty property, AstParameters params) { + return new EagerAstMethod(property, params); + } + + @Override + protected AstUnary createAstUnary( + AstNode child, + de.odysseus.el.tree.impl.ast.AstUnary.Operator operator + ) { + return new EagerAstUnary(child, operator); + } + + @Override + protected AstRangeBracket createAstRangeBracket( + AstNode base, + AstNode rangeStart, + AstNode rangeMax, + boolean lvalue, + boolean strict + ) { + return new EagerAstRangeBracket( + base, + rangeStart, + rangeMax, + lvalue, + strict, + context.isEnabled(Feature.IGNORE_RETURN_TYPE) + ); + } + + @Override + protected AstDict createAstDict(Map dict) { + return new EagerAstDict(dict); + } + + @Override + protected AstRightValue createAstNested(AstNode node) { + return new EagerAstNested( + (AstNode) EagerAstNodeDecorator.getAsEvalResultHolder(node) + ); + } + + @Override + protected AstTuple createAstTuple(AstParameters parameters) + throws ScanException, ParseException { + return new EagerAstTuple(parameters); + } + + @Override + protected AstList createAstList(AstParameters parameters) + throws ScanException, ParseException { + return new EagerAstList(parameters); + } + + @Override + protected AstParameters createAstParameters(List nodes) { + return new EagerAstParameters(nodes); + } +} diff --git a/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerExtendedSyntaxBuilder.java b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerExtendedSyntaxBuilder.java new file mode 100644 index 000000000..4226caab9 --- /dev/null +++ b/src/main/java/com/hubspot/jinjava/el/ext/eager/EagerExtendedSyntaxBuilder.java @@ -0,0 +1,20 @@ +package com.hubspot.jinjava.el.ext.eager; + +import com.hubspot.jinjava.el.ExtendedSyntaxBuilder; +import de.odysseus.el.tree.impl.Parser; + +public class EagerExtendedSyntaxBuilder extends ExtendedSyntaxBuilder { + + public EagerExtendedSyntaxBuilder() { + super(); + } + + public EagerExtendedSyntaxBuilder(Feature... features) { + super(features); + } + + @Override + protected Parser createParser(String expression) { + return new EagerExtendedParser(this, expression); + } +} diff --git a/src/main/java/com/hubspot/jinjava/el/ext/eager/EvalResultHolder.java b/src/main/java/com/hubspot/jinjava/el/ext/eager/EvalResultHolder.java new file mode 100644 index 000000000..675909da6 --- /dev/null +++ b/src/main/java/com/hubspot/jinjava/el/ext/eager/EvalResultHolder.java @@ -0,0 +1,51 @@ +package com.hubspot.jinjava.el.ext.eager; + +import com.hubspot.jinjava.el.ext.DeferredParsingException; +import com.hubspot.jinjava.el.ext.ExtendedParser; +import com.hubspot.jinjava.util.ChunkResolver; +import de.odysseus.el.tree.Bindings; +import de.odysseus.el.tree.impl.ast.AstIdentifier; +import javax.el.ELContext; + +public interface EvalResultHolder { + Object getAndClearEvalResult(); + + boolean hasEvalResult(); + + Object eval(Bindings bindings, ELContext elContext); + + static String reconstructNode( + Bindings bindings, + ELContext context, + EvalResultHolder astNode, + DeferredParsingException exception, + boolean preserveIdentifier + ) { + String partiallyResolvedImage; + if ( + astNode instanceof AstIdentifier && + ( + preserveIdentifier || + ExtendedParser.INTERPRETER.equals(((AstIdentifier) astNode).getName()) + ) + ) { + astNode.getAndClearEvalResult(); // clear unused result + partiallyResolvedImage = ((AstIdentifier) astNode).getName(); + } else if (astNode.hasEvalResult()) { + partiallyResolvedImage = + ChunkResolver.getValueAsJinjavaStringSafe(astNode.getAndClearEvalResult()); + } else if (exception.getSourceNode() == astNode) { + partiallyResolvedImage = exception.getDeferredEvalResult(); + } else { + try { + partiallyResolvedImage = + ChunkResolver.getValueAsJinjavaStringSafe(astNode.eval(bindings, context)); + } catch (DeferredParsingException e) { + partiallyResolvedImage = e.getDeferredEvalResult(); + } finally { + astNode.getAndClearEvalResult(); + } + } + return partiallyResolvedImage; + } +} diff --git a/src/main/java/com/hubspot/jinjava/interpret/JinjavaInterpreter.java b/src/main/java/com/hubspot/jinjava/interpret/JinjavaInterpreter.java index df82ff959..d56c5864b 100644 --- a/src/main/java/com/hubspot/jinjava/interpret/JinjavaInterpreter.java +++ b/src/main/java/com/hubspot/jinjava/interpret/JinjavaInterpreter.java @@ -27,6 +27,7 @@ import com.hubspot.jinjava.Jinjava; import com.hubspot.jinjava.JinjavaConfig; import com.hubspot.jinjava.el.ExpressionResolver; +import com.hubspot.jinjava.el.ext.DeferredParsingException; import com.hubspot.jinjava.interpret.Context.TemporaryValueClosable; import com.hubspot.jinjava.interpret.TemplateError.ErrorItem; import com.hubspot.jinjava.interpret.TemplateError.ErrorReason; @@ -427,7 +428,11 @@ public Object retraceVariable(String variable, int lineNumber, int startPosition } if (obj != null) { if (obj instanceof DeferredValue) { - throw new DeferredValueException(variable, lineNumber, startPosition); + if (config.getExecutionMode().useEagerParser()) { + throw new DeferredParsingException(this, variable); + } else { + throw new DeferredValueException(variable, lineNumber, startPosition); + } } obj = var.resolve(obj); } diff --git a/src/main/java/com/hubspot/jinjava/lib/expression/EagerExpressionStrategy.java b/src/main/java/com/hubspot/jinjava/lib/expression/EagerExpressionStrategy.java index 1a924ec94..cb310ec31 100644 --- a/src/main/java/com/hubspot/jinjava/lib/expression/EagerExpressionStrategy.java +++ b/src/main/java/com/hubspot/jinjava/lib/expression/EagerExpressionStrategy.java @@ -20,13 +20,10 @@ public RenderedOutputNode interpretOutput( ExpressionToken master, JinjavaInterpreter interpreter ) { - EagerStringResult eagerStringResult = eagerResolveExpression(master, interpreter); - return new RenderedOutputNode( - eagerStringResult.getPrefixToPreserveState() + eagerStringResult.getResult() - ); + return new RenderedOutputNode(eagerResolveExpression(master, interpreter)); } - private EagerStringResult eagerResolveExpression( + private String eagerResolveExpression( ExpressionToken master, JinjavaInterpreter interpreter ) { @@ -35,7 +32,7 @@ private EagerStringResult eagerResolveExpression( master, interpreter ); - EagerStringResult resolvedExpression = EagerTagDecorator.executeInChildContext( + EagerStringResult eagerStringResult = EagerTagDecorator.executeInChildContext( eagerInterpreter -> chunkResolver.resolveChunks(), interpreter, true, @@ -43,16 +40,11 @@ private EagerStringResult eagerResolveExpression( ); StringBuilder prefixToPreserveState = new StringBuilder( interpreter.getContext().isDeferredExecutionMode() - ? resolvedExpression.getPrefixToPreserveState() + ? eagerStringResult.getPrefixToPreserveState() : "" ); if (chunkResolver.getDeferredWords().isEmpty()) { - String result = interpreter.getAsString( - interpreter.resolveELExpression( - resolvedExpression.getResult(), - interpreter.getLineNumber() - ) - ); + String result = eagerStringResult.getResult().toString(true); if ( !StringUtils.equals(result, master.getImage()) && ( @@ -75,7 +67,7 @@ private EagerStringResult eagerResolveExpression( if (interpreter.getContext().isAutoEscape()) { result = EscapeFilter.escapeHtmlEntities(result); } - return new EagerStringResult(result, prefixToPreserveState.toString()); + return prefixToPreserveState.toString() + result; } prefixToPreserveState.append( EagerTagDecorator.reconstructFromContextBeforeDeferring( @@ -83,7 +75,10 @@ private EagerStringResult eagerResolveExpression( interpreter ) ); - String helpers = wrapInExpression(resolvedExpression.getResult(), interpreter); + String helpers = wrapInExpression( + eagerStringResult.getResult().toString(), + interpreter + ); interpreter .getContext() .handleEagerToken( @@ -97,13 +92,10 @@ private EagerStringResult eagerResolveExpression( chunkResolver.getDeferredWords() ) ); - // There is no result because it couldn't be entirely evaluated. - return new EagerStringResult( - "", - EagerTagDecorator.wrapInAutoEscapeIfNeeded( - prefixToPreserveState.toString() + helpers, - interpreter - ) + // There is no only a preserving prefix because it couldn't be entirely evaluated. + return EagerTagDecorator.wrapInAutoEscapeIfNeeded( + prefixToPreserveState.toString() + helpers, + interpreter ); } diff --git a/src/main/java/com/hubspot/jinjava/lib/tag/SetTag.java b/src/main/java/com/hubspot/jinjava/lib/tag/SetTag.java index 7a5e4ef78..8b8ae4636 100644 --- a/src/main/java/com/hubspot/jinjava/lib/tag/SetTag.java +++ b/src/main/java/com/hubspot/jinjava/lib/tag/SetTag.java @@ -111,7 +111,12 @@ public String interpret(TagNode tagNode, JinjavaInterpreter interpreter) { String[] varTokens = var.split(","); try { - executeSet((TagToken) tagNode.getMaster(), interpreter, varTokens, expr, false); + @SuppressWarnings("unchecked") + List exprVals = (List) interpreter.resolveELExpression( + "[" + expr + "]", + tagNode.getMaster().getLineNumber() + ); + executeSet((TagToken) tagNode.getMaster(), interpreter, varTokens, exprVals, false); } catch (DeferredValueException e) { DeferredValueUtils.deferVariables(varTokens, interpreter.getContext()); throw e; @@ -124,18 +129,13 @@ public void executeSet( TagToken tagToken, JinjavaInterpreter interpreter, String[] varTokens, - String expr, + List resolvedList, boolean allowDeferredValueOverride ) { if (varTokens.length > 1) { // handle multi-variable assignment - @SuppressWarnings("unchecked") - List exprVals = (List) interpreter.resolveELExpression( - "[" + expr + "]", - tagToken.getLineNumber() - ); - if (varTokens.length != exprVals.size()) { + if (resolvedList == null || varTokens.length != resolvedList.size()) { throw new TemplateSyntaxException( tagToken.getImage(), "Tag 'set' declares an uneven number of variables and assigned values", @@ -154,7 +154,7 @@ public void executeSet( throw new DeferredValueException(varItem); } } - interpreter.getContext().put(varItem, exprVals.get(i)); + interpreter.getContext().put(varItem, resolvedList.get(i)); } } else { // handle single variable assignment @@ -168,10 +168,7 @@ public void executeSet( } interpreter .getContext() - .put( - varTokens[0], - interpreter.resolveELExpression(expr, tagToken.getLineNumber()) - ); + .put(varTokens[0], resolvedList != null ? resolvedList.get(0) : null); } } diff --git a/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerCycleTag.java b/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerCycleTag.java index 49bbf7eb9..56c05fac7 100644 --- a/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerCycleTag.java +++ b/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerCycleTag.java @@ -1,6 +1,7 @@ package com.hubspot.jinjava.lib.tag.eager; import com.google.common.collect.ImmutableMap; +import com.hubspot.jinjava.el.ext.ExtendedParser; import com.hubspot.jinjava.interpret.JinjavaInterpreter; import com.hubspot.jinjava.interpret.TemplateSyntaxException; import com.hubspot.jinjava.lib.tag.CycleTag; @@ -39,19 +40,19 @@ public String getEagerTagImage(TagToken tagToken, JinjavaInterpreter interpreter helper.add(sb.toString()); } ChunkResolver chunkResolver = new ChunkResolver(helper.get(0), tagToken, interpreter); - EagerStringResult resolvedExpression = executeInChildContext( + EagerStringResult eagerStringResult = executeInChildContext( eagerInterpreter -> chunkResolver.resolveChunks(), interpreter, true, false ); - String expression = resolvedExpression.getResult(); + String expression = eagerStringResult.getResult().toString().replace(", ", ","); if (WhitespaceUtils.isWrappedWith(expression, "[", "]")) { - expression = expression.substring(1, expression.length() - 1).replace(", ", ","); + expression = expression.substring(1, expression.length() - 1); } StringBuilder prefixToPreserveState = new StringBuilder( interpreter.getContext().isDeferredExecutionMode() - ? resolvedExpression.getPrefixToPreserveState() + ? eagerStringResult.getPrefixToPreserveState() : "" ); HelperStringTokenizer items = new HelperStringTokenizer(expression).splitComma(true); @@ -107,7 +108,10 @@ private String interpretSettingCycle( String var = helper.get(2); if (!chunkResolver.getDeferredWords().isEmpty()) { return EagerTagDecorator.buildSetTagForDeferredInChildContext( - ImmutableMap.of(var, String.format("[%s]", resolvedExpression)), + ImmutableMap.of( + var, + String.format("[%s]", resolvedExpression.replace(",", ", ")) + ), interpreter, true ); @@ -165,9 +169,21 @@ private static String getIsIterable(String var, int forIndex, TagToken tagToken) String tokenStart = tagToken.getSymbols().getExpressionStartWithTag(); String tokenEnd = tagToken.getSymbols().getExpressionEndWithTag(); return ( - String.format("%s if %s is iterable %s", tokenStart, var, tokenEnd) + + String.format( + "%s if exptest:iterable.evaluate(%s, %s) %s", + tokenStart, + var, + ExtendedParser.INTERPRETER, + tokenEnd + ) + // modulo indexing - String.format("{{ %s[%d %% %s|length] }}", var, forIndex, var) + + String.format( + "{{ %s[%d %% filter:length.filter(%s, %s)] }}", + var, + forIndex, + var, + ExtendedParser.INTERPRETER + ) + String.format("%s else %s", tokenStart, tokenEnd) + String.format("{{ %s }}", var) + String.format("%s endif %s", tokenStart, tokenEnd) diff --git a/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerIfTag.java b/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerIfTag.java index 09acefd89..3d4b03459 100644 --- a/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerIfTag.java +++ b/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerIfTag.java @@ -8,6 +8,7 @@ import com.hubspot.jinjava.lib.tag.IfTag; import com.hubspot.jinjava.tree.Node; import com.hubspot.jinjava.tree.TagNode; +import com.hubspot.jinjava.util.ChunkResolver.ResolvedChunks; import com.hubspot.jinjava.util.LengthLimitingStringBuilder; import com.hubspot.jinjava.util.ObjectTruthValue; import org.apache.commons.lang3.StringUtils; @@ -39,8 +40,10 @@ public String eagerInterpret(TagNode tagNode, JinjavaInterpreter interpreter) { result.append( executeInChildContext( eagerInterpreter -> - getEagerImage(tagNode.getMaster(), eagerInterpreter) + - renderChildren(tagNode, eagerInterpreter), + ResolvedChunks.fromString( + getEagerImage(tagNode.getMaster(), eagerInterpreter) + + renderChildren(tagNode, eagerInterpreter) + ), interpreter, false, false diff --git a/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerPrintTag.java b/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerPrintTag.java index b263f4f81..cc2268bbe 100644 --- a/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerPrintTag.java +++ b/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerPrintTag.java @@ -56,15 +56,6 @@ public static String interpretExpression( true, false ); - LengthLimitingStringJoiner joiner = new LengthLimitingStringJoiner( - interpreter.getConfig().getMaxOutputSize(), - " " - ); - joiner - .add(tagToken.getSymbols().getExpressionStartWithTag()) - .add(tagToken.getTagName()) - .add(resolvedExpression.getResult()) - .add(tagToken.getSymbols().getExpressionEndWithTag()); StringBuilder prefixToPreserveState = new StringBuilder( interpreter.getContext().isDeferredExecutionMode() ? resolvedExpression.getPrefixToPreserveState() @@ -77,12 +68,7 @@ public static String interpretExpression( ( includeExpressionResult ? wrapInRawIfNeeded( - interpreter.getAsString( - interpreter.resolveELExpression( - resolvedExpression.getResult(), - interpreter.getLineNumber() - ) - ), + resolvedExpression.getResult().toString(true), interpreter ) : "" @@ -92,6 +78,16 @@ public static String interpretExpression( prefixToPreserveState.append( reconstructFromContextBeforeDeferring(chunkResolver.getDeferredWords(), interpreter) ); + + LengthLimitingStringJoiner joiner = new LengthLimitingStringJoiner( + interpreter.getConfig().getMaxOutputSize(), + " " + ); + joiner + .add(tagToken.getSymbols().getExpressionStartWithTag()) + .add(tagToken.getTagName()) + .add(resolvedExpression.getResult().toString()) + .add(tagToken.getSymbols().getExpressionEndWithTag()); interpreter .getContext() .handleEagerToken( diff --git a/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerSetTag.java b/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerSetTag.java index 2816d008a..cdaa41ad3 100644 --- a/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerSetTag.java +++ b/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerSetTag.java @@ -46,22 +46,7 @@ public String getEagerTagImage(TagToken tagToken, JinjavaInterpreter interpreter true, false ); - LengthLimitingStringJoiner joiner = new LengthLimitingStringJoiner( - interpreter.getConfig().getMaxOutputSize(), - " " - ); - joiner - .add(tagToken.getSymbols().getExpressionStartWithTag()) - .add(tagToken.getTagName()) - .add(variables) - .add("=") - .add(resolvedExpression.getResult()) - .add(tagToken.getSymbols().getExpressionEndWithTag()); - StringBuilder prefixToPreserveState = new StringBuilder( - interpreter.getContext().isDeferredExecutionMode() - ? resolvedExpression.getPrefixToPreserveState() - : "" - ); + String[] varTokens = variables.split(","); if ( @@ -74,17 +59,22 @@ public String getEagerTagImage(TagToken tagToken, JinjavaInterpreter interpreter tagToken, interpreter, varTokens, - resolvedExpression.getPrefixToPreserveState().isEmpty() - ? expression - : resolvedExpression.getResult(), + resolvedExpression.getResult().toList(), true ); return ""; } catch (DeferredValueException ignored) {} } - prefixToPreserveState.append( - reconstructFromContextBeforeDeferring(chunkResolver.getDeferredWords(), interpreter) - ); + LengthLimitingStringJoiner joiner = new LengthLimitingStringJoiner( + interpreter.getConfig().getMaxOutputSize(), + " " + ) + .add(tagToken.getSymbols().getExpressionStartWithTag()) + .add(tagToken.getTagName()) + .add(variables) + .add("=") + .add(resolvedExpression.getResult().toString()) + .add(tagToken.getSymbols().getExpressionEndWithTag()); interpreter .getContext() @@ -116,10 +106,18 @@ public String getEagerTagImage(TagToken tagToken, JinjavaInterpreter interpreter ) ); } + String prefixToPreserveState = + ( + interpreter.getContext().isDeferredExecutionMode() + ? resolvedExpression.getPrefixToPreserveState() + : "" + ) + + reconstructFromContextBeforeDeferring( + chunkResolver.getDeferredWords(), + interpreter + ); return wrapInAutoEscapeIfNeeded( - prefixToPreserveState.toString() + - joiner.toString() + - suffixToPreserveState.toString(), + prefixToPreserveState + joiner.toString() + suffixToPreserveState.toString(), interpreter ); } diff --git a/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerStateChangingTag.java b/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerStateChangingTag.java index 00023986b..ce9be93c6 100644 --- a/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerStateChangingTag.java +++ b/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerStateChangingTag.java @@ -3,6 +3,7 @@ import com.hubspot.jinjava.interpret.JinjavaInterpreter; import com.hubspot.jinjava.lib.tag.Tag; import com.hubspot.jinjava.tree.TagNode; +import com.hubspot.jinjava.util.ChunkResolver.ResolvedChunks; import org.apache.commons.lang3.StringUtils; public class EagerStateChangingTag extends EagerTagDecorator { @@ -26,7 +27,8 @@ public String eagerInterpret(TagNode tagNode, JinjavaInterpreter interpreter) { if (!tagNode.getChildren().isEmpty()) { result.append( executeInChildContext( - eagerInterpreter -> renderChildren(tagNode, eagerInterpreter), + eagerInterpreter -> + ResolvedChunks.fromString(renderChildren(tagNode, eagerInterpreter)), interpreter, false, false diff --git a/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerStringResult.java b/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerStringResult.java index 1a4ef398e..1eba0c789 100644 --- a/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerStringResult.java +++ b/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerStringResult.java @@ -1,25 +1,27 @@ package com.hubspot.jinjava.lib.tag.eager; +import com.hubspot.jinjava.util.ChunkResolver.ResolvedChunks; + /** * This represents the result of executing an expression, where if something got * deferred, then the prefixToPreserveState can be added to the output * that would preserve the state for a second pass. */ public class EagerStringResult { - private final String result; + private final ResolvedChunks result; private final String prefixToPreserveState; - public EagerStringResult(String result) { + public EagerStringResult(ResolvedChunks result) { this.result = result; this.prefixToPreserveState = ""; } - public EagerStringResult(String result, String prefixToPreserveState) { + public EagerStringResult(ResolvedChunks result, String prefixToPreserveState) { this.result = result; this.prefixToPreserveState = prefixToPreserveState; } - public String getResult() { + public ResolvedChunks getResult() { return result; } diff --git a/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerTagDecorator.java b/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerTagDecorator.java index 725262e4f..1008972df 100644 --- a/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerTagDecorator.java +++ b/src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerTagDecorator.java @@ -27,6 +27,7 @@ import com.hubspot.jinjava.tree.parse.TagToken; import com.hubspot.jinjava.tree.parse.Token; import com.hubspot.jinjava.util.ChunkResolver; +import com.hubspot.jinjava.util.ChunkResolver.ResolvedChunks; import com.hubspot.jinjava.util.LengthLimitingStringBuilder; import com.hubspot.jinjava.util.LengthLimitingStringJoiner; import java.util.Collections; @@ -101,8 +102,10 @@ public String eagerInterpret(TagNode tagNode, JinjavaInterpreter interpreter) { result.append( executeInChildContext( eagerInterpreter -> - getEagerImage(tagNode.getMaster(), eagerInterpreter) + - renderChildren(tagNode, eagerInterpreter), + ResolvedChunks.fromString( + getEagerImage(tagNode.getMaster(), eagerInterpreter) + + renderChildren(tagNode, eagerInterpreter) + ), interpreter, false, false @@ -152,12 +155,12 @@ public String renderChildren(TagNode tagNode, JinjavaInterpreter interpreter) { * that preserves the state within the output for a second rendering pass. */ public static EagerStringResult executeInChildContext( - Function function, + Function function, JinjavaInterpreter interpreter, boolean takeNewValue, boolean partialMacroEvaluation ) { - StringBuilder result = new StringBuilder(); + ResolvedChunks result; Map initiallyResolvedHashes = new HashMap<>(); interpreter .getContext() @@ -179,7 +182,7 @@ public static EagerStringResult executeInChildContext( try (InterpreterScopeClosable c = interpreter.enterNonStackingScope()) { interpreter.getContext().setDeferredExecutionMode(true); interpreter.getContext().setPartialMacroEvaluation(partialMacroEvaluation); - result.append(function.apply(interpreter)); + result = function.apply(interpreter); } Map deferredValuesToSet = interpreter .getContext() @@ -209,7 +212,7 @@ public static EagerStringResult executeInChildContext( ); if (deferredValuesToSet.size() > 0) { return new EagerStringResult( - result.toString(), + result, buildSetTagForDeferredInChildContext( deferredValuesToSet, interpreter, @@ -217,7 +220,7 @@ public static EagerStringResult executeInChildContext( ) ); } - return new EagerStringResult(result.toString()); + return new EagerStringResult(result); } /** @@ -281,8 +284,10 @@ private static String reconstructMacroFunctionsBeforeDeferring( entry -> executeInChildContext( eagerInterpreter -> - new EagerMacroFunction(entry.getKey(), entry.getValue(), interpreter) - .reconstructImage(), + ResolvedChunks.fromString( + new EagerMacroFunction(entry.getKey(), entry.getValue(), interpreter) + .reconstructImage() + ), interpreter, false, false @@ -447,7 +452,7 @@ public String getEagerTagImage(TagToken tagToken, JinjavaInterpreter interpreter tagToken, interpreter ); - String resolvedChunks = chunkResolver.resolveChunks(); + String resolvedChunks = chunkResolver.resolveChunks().toString(); if (StringUtils.isNotBlank(resolvedChunks)) { joiner.add(resolvedChunks); } diff --git a/src/main/java/com/hubspot/jinjava/mode/EagerExecutionMode.java b/src/main/java/com/hubspot/jinjava/mode/EagerExecutionMode.java index ca3f1cbe5..f12e0df3e 100644 --- a/src/main/java/com/hubspot/jinjava/mode/EagerExecutionMode.java +++ b/src/main/java/com/hubspot/jinjava/mode/EagerExecutionMode.java @@ -20,6 +20,11 @@ public boolean isPreserveRawTags() { return true; } + @Override + public boolean useEagerParser() { + return true; + } + @Override public void prepareContext(Context context) { context diff --git a/src/main/java/com/hubspot/jinjava/mode/ExecutionMode.java b/src/main/java/com/hubspot/jinjava/mode/ExecutionMode.java index 446f4cc07..4c0e4ff92 100644 --- a/src/main/java/com/hubspot/jinjava/mode/ExecutionMode.java +++ b/src/main/java/com/hubspot/jinjava/mode/ExecutionMode.java @@ -7,5 +7,9 @@ default boolean isPreserveRawTags() { return false; } + default boolean useEagerParser() { + return false; + } + default void prepareContext(Context context) {} } diff --git a/src/main/java/com/hubspot/jinjava/util/ChunkResolver.java b/src/main/java/com/hubspot/jinjava/util/ChunkResolver.java index 9cfb2aff6..8cf81acd8 100644 --- a/src/main/java/com/hubspot/jinjava/util/ChunkResolver.java +++ b/src/main/java/com/hubspot/jinjava/util/ChunkResolver.java @@ -1,7 +1,8 @@ package com.hubspot.jinjava.util; -import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.hubspot.jinjava.el.ext.DeferredParsingException; +import com.hubspot.jinjava.el.ext.ExtendedParser; import com.hubspot.jinjava.interpret.DeferredValueException; import com.hubspot.jinjava.interpret.JinjavaInterpreter; import com.hubspot.jinjava.interpret.TemplateSyntaxException; @@ -9,9 +10,9 @@ import com.hubspot.jinjava.objects.serialization.PyishObjectMapper; import com.hubspot.jinjava.objects.serialization.PyishSerializable; import com.hubspot.jinjava.tree.parse.Token; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -45,7 +46,10 @@ public class ChunkResolver { "null", "true", "false", - "__macros__" + "__macros__", + ExtendedParser.INTERPRETER, + "exptest", + "filter" ); private static final Set> RESOLVABLE_CLASSES = ImmutableSet.of( @@ -55,32 +59,13 @@ public class ChunkResolver { PyishSerializable.class ); - // ( -> ) - // { -> } - // [ -> ] - private static final Map CHUNK_LEVEL_MARKER_MAP = ImmutableMap.of( - '(', - ')', - '{', - '}', - '[', - ']' - ); - - private final char[] value; - private final int length; + private final String value; private final Token token; private final JinjavaInterpreter interpreter; private final Set deferredWords; - private int nextPos = 0; - private char prevChar = 0; - private boolean inQuote = false; - private char quoteChar = 0; - public ChunkResolver(String s, Token token, JinjavaInterpreter interpreter) { - value = s.toCharArray(); - length = value.length; + value = s.trim(); this.token = token; this.interpreter = interpreter; deferredWords = new HashSet<>(); @@ -105,269 +90,75 @@ public Set getDeferredWords() { * `[(foo == bar), deferred, bar]` -> `[true,deferred,'hello']` * @return String with chunk layers within it being partially or fully resolved. */ - public String resolveChunks() { - nextPos = 0; - boolean isThrowInterpreterErrorsStart = interpreter - .getContext() - .getThrowInterpreterErrors(); + public ResolvedChunks resolveChunks() { + boolean fullyResolved = false; + Object result; try { - interpreter.getContext().setThrowInterpreterErrors(true); - String expression = String.join("", getChunk(null)).trim(); - if (JINJAVA_NULL.equals(expression)) { - // Resolved value of null as a string is ''. - return JINJAVA_EMPTY_STRING; - } - return expression; - } finally { - interpreter.getContext().setThrowInterpreterErrors(isThrowInterpreterErrorsStart); - } - } - - /** - * Chunkify and resolve variables and expressions within the string. - * Rather than concatenating the chunks, they are split by mini-chunks, - * with the comma splitter omitted from the list of results. - * Therefore an expression of "1, 1 + 1, 1 + range(deferred)" becomes a List of ["1", "2", "1 + range(deferred)"]. - * - * @return List of the expression chunk which is split into mini-chunks. - */ - public List splitChunks() { - nextPos = 0; - boolean isThrowInterpreterErrorsStart = interpreter - .getContext() - .getThrowInterpreterErrors(); - try { - interpreter.getContext().setThrowInterpreterErrors(true); - List miniChunks = getChunk(null); - return miniChunks - .stream() - .filter(s -> s.length() > 1 || !isMiniChunkSplitter(s.charAt(0))) - .map(String::trim) - .collect(Collectors.toList()); - } finally { - interpreter.getContext().setThrowInterpreterErrors(isThrowInterpreterErrorsStart); - } - } - - /** - * e.g. `[0, foo + bar]`: - * `0, foo + bar` is a chunk - * `0` and `foo + bar` are mini chunks - * `0`, `,`, ` `, `foo`, ` `, `+`, ` `, and `bar` are the tokens - * @param chunkLevelMarker the marker `(`, `[`, `{` that started this chunk - * @return the resolved chunk - */ - private List getChunk(Character chunkLevelMarker) { - List chunks = new ArrayList<>(); - // Mini chunks are split by commas. - StringBuilder miniChunkBuilder = new StringBuilder(); - StringBuilder tokenBuilder = new StringBuilder(); - while (nextPos < length) { - boolean isAfterWhitespace = prevChar == ' ' && !isOpWhitespace(prevChar); - char c = value[nextPos++]; - if (inQuote) { - if (c == quoteChar && prevChar != '\\') { - inQuote = false; - } - } else if ((c == '\'' || c == '"') && prevChar != '\\') { - inQuote = true; - quoteChar = c; - } else if ( - chunkLevelMarker != null && CHUNK_LEVEL_MARKER_MAP.get(chunkLevelMarker) == c - ) { - setPrevChar(c); - break; - } else if (CHUNK_LEVEL_MARKER_MAP.containsKey(c)) { - setPrevChar(c); - tokenBuilder.append(c); - tokenBuilder.append(resolveChunk(String.join("", getChunk(c)))); - tokenBuilder.append(prevChar); - continue; - } else if (isTokenSplitter(c)) { - String resolvedToken; - if ( - c == ':' && - chunkLevelMarker != null && - '{' == chunkLevelMarker && - !interpreter.getConfig().getLegacyOverrides().isEvaluateMapKeys() - ) { - resolvedToken = - '\'' + WhitespaceUtils.unquoteAndUnescape(tokenBuilder.toString()) + '\''; - } else { - resolvedToken = resolveToken(tokenBuilder.toString()); - } - if (StringUtils.isNotEmpty(resolvedToken)) { - miniChunkBuilder.append(resolvedToken); - } - tokenBuilder = new StringBuilder(); - if (isMiniChunkSplitter(c)) { - chunks.add(resolveChunk(miniChunkBuilder.toString())); - chunks.add(String.valueOf(c)); - miniChunkBuilder = new StringBuilder(); - } else { - miniChunkBuilder.append(c); - } - setPrevChar(c); - continue; - } else if (isAfterWhitespace) { - // In case there is whitespace between words: `foo or bar` - String resolvedToken = resolveToken(tokenBuilder.toString()); - if (StringUtils.isNotEmpty(resolvedToken)) { - miniChunkBuilder.append(resolveToken(tokenBuilder.toString())); - } - tokenBuilder = new StringBuilder(); - } - setPrevChar(c); - tokenBuilder.append(c); - } - miniChunkBuilder.append(resolveToken(tokenBuilder.toString())); - chunks.add(resolveChunk(miniChunkBuilder.toString())); - return chunks; - } - - private void setPrevChar(char c) { - if (c == '\\' && prevChar == '\\') { - // Backslashes cancel each other out for escaping when there's an even number. - prevChar = '\0'; - } else { - prevChar = c; - } - } - - private boolean isTokenSplitter(char c) { - if (c == '|' && prevChar == '|') { // or operator - return true; - } - return ( - !Character.isLetterOrDigit(c) && c != '_' && c != '.' && c != '|' && c != ' ' - ); - } - - private boolean isOpWhitespace(char c) { - // If a pipe or full stop character is surrounded by whitespace on either side, - // we don't want to split those tokens - boolean isFilterWhitespace = false; - if (c == ' ') { - int prevPos = nextPos - 2; - if (nextPos < length) { - isFilterWhitespace = - value[nextPos] == ' ' || value[nextPos] == '|' || value[nextPos] == '.'; - } - if (prevPos >= 0) { - isFilterWhitespace = - isFilterWhitespace || - value[prevPos] == ' ' || - value[prevPos] == '|' || - value[prevPos] == '.'; - } - } - return isFilterWhitespace; - } - - private boolean isMiniChunkSplitter(char c) { - return c == ','; - } - - private String resolveToken(String token) { - if (StringUtils.isBlank(token)) { - return token; - } - String resolvedToken = token; - try { - if ( - !WhitespaceUtils.isExpressionQuoted(token) && !RESERVED_KEYWORDS.contains(token) - ) { - Object val = null; - try { - val = - interpreter.retraceVariable( - token, - this.token.getLineNumber(), - this.token.getStartPosition() - ); - } catch (TemplateSyntaxException ignored) {} - if (val == null) { - try { - val = interpreter.resolveELExpression(token, this.token.getLineNumber()); - } catch (UnknownTokenException e) { - // val is still null - } - } - if (val != null && isResolvableObject(val)) { - resolvedToken = PyishObjectMapper.getAsPyishString(val); - } - } + result = + interpreter.resolveELExpression( + String.format("[%s]", value), + interpreter.getLineNumber() + ); + fullyResolved = true; + } catch (DeferredParsingException e) { + deferredWords.addAll(findDeferredWords(e.getDeferredEvalResult())); + String bracketedResult = e.getDeferredEvalResult().trim(); + result = bracketedResult.substring(1, bracketedResult.length() - 1); } catch (DeferredValueException e) { - deferredWords.addAll(findDeferredWords(token)); - } catch (TemplateSyntaxException ignored) {} - return spaced(resolvedToken, token); - } - - // Try resolving the chunk/mini chunk as an ELExpression - private String resolveChunk(String chunk) { - if (StringUtils.isBlank(chunk)) { - return chunk; + deferredWords.addAll(findDeferredWords(value)); + result = value; + } catch (TemplateSyntaxException e) { + result = Collections.singletonList(null); + fullyResolved = true; } - String resolvedChunk = chunk; - try { - if ( - !WhitespaceUtils.isExpressionQuoted(chunk) && !RESERVED_KEYWORDS.contains(chunk) - ) { - try { - Object val = interpreter.retraceVariable( - chunk.trim(), - this.token.getLineNumber(), - this.token.getStartPosition() - ); - if (val != null) { - // If this isn't the final call, don't prematurely resolve complex objects. - if (!isResolvableObject(val)) { - return chunk; - } - } - } catch (TemplateSyntaxException ignored) {} + return new ResolvedChunks(result, fullyResolved); + } - Object val = interpreter.resolveELExpression(chunk, token.getLineNumber()); - if (val == null) { - resolvedChunk = ChunkResolver.JINJAVA_NULL; - } else if (isResolvableObject(val)) { - resolvedChunk = PyishObjectMapper.getAsPyishString(val); - } - } - } catch (TemplateSyntaxException ignored) {} catch (Exception e) { - deferredWords.addAll(findDeferredWords(chunk)); + public static String getValueAsJinjavaStringSafe(Object val) { + if (val == null) { + return JINJAVA_NULL; + } else if (isResolvableObject(val)) { + return PyishObjectMapper.getAsPyishString(val); } - return spaced(resolvedChunk, chunk); + throw new DeferredValueException("Can not convert deferred result to string"); } // Find any variables, functions, etc in this chunk to mark as deferred. // similar processing to getChunk method, but without recursion. private Set findDeferredWords(String chunk) { - Set words = new HashSet<>(); - char[] value = chunk.toCharArray(); - int prevQuotePos = 0; - int curPos = 0; - char c; - char prevChar = 0; - boolean inQuote = false; - char quoteChar = 0; - while (curPos < chunk.length()) { - c = value[curPos]; - if (inQuote) { - if (c == quoteChar && prevChar != '\\') { - inQuote = false; - prevQuotePos = curPos; + boolean throwInterpreterErrorsStart = interpreter + .getContext() + .getThrowInterpreterErrors(); + try { + interpreter.getContext().setThrowInterpreterErrors(true); + Set words = new HashSet<>(); + char[] value = chunk.toCharArray(); + int prevQuotePos = 0; + int curPos = 0; + char c; + char prevChar = 0; + boolean inQuote = false; + char quoteChar = 0; + while (curPos < chunk.length()) { + c = value[curPos]; + if (inQuote) { + if (c == quoteChar && prevChar != '\\') { + inQuote = false; + prevQuotePos = curPos; + } + } else if ((c == '\'' || c == '"') && prevChar != '\\') { + inQuote = true; + quoteChar = c; + words.addAll(findDeferredWordsInSubstring(chunk, prevQuotePos, curPos)); } - } else if ((c == '\'' || c == '"') && prevChar != '\\') { - inQuote = true; - quoteChar = c; - words.addAll(findDeferredWordsInSubstring(chunk, prevQuotePos, curPos)); + prevChar = c; + curPos++; } - prevChar = c; - curPos++; + words.addAll(findDeferredWordsInSubstring(chunk, prevQuotePos, curPos)); + return words; + } finally { + interpreter.getContext().setThrowInterpreterErrors(throwInterpreterErrorsStart); } - words.addAll(findDeferredWordsInSubstring(chunk, prevQuotePos, curPos)); - return words; } // Knowing that there are no quotes between start and end, @@ -436,13 +227,88 @@ private static boolean isResolvableObjectRec(Object val, int depth) { ).stream() .filter(Objects::nonNull) .allMatch(item -> isResolvableObjectRec(item, depth + 1)); + } else if (val.getClass().isArray()) { + if (((Object[]) val).length == 0) { + return true; + } + return (Arrays.stream((Object[]) val)).filter(Objects::nonNull) + .allMatch(item -> isResolvableObjectRec(item, depth + 1)); } return false; } - private static String spaced(String toSpaceOut, String reference) { - String prefix = reference.startsWith(" ") ? " " : ""; - String suffix = reference.endsWith(" ") ? " " : ""; - return prefix + toSpaceOut.trim() + suffix; + public static class ResolvedChunks { + private final Object resolvedObject; + private final boolean fullyResolved; + + private ResolvedChunks(Object resolvedObject, boolean fullyResolved) { + this.resolvedObject = resolvedObject; + this.fullyResolved = fullyResolved; + } + + /** + * Returns a string representation of the resolved expression. + * If there are multiple, they will be separated by commas, + * but not surrounded with brackets. + * @return String representation of the chunks. + */ + @Override + public String toString() { + return toString(false); + } + + /** + * When forOutput is true, the result will always be unquoted. + * @param forOutput Whether the result is going to be included in the final output, + * such as in an expression, or not such as when reconstructing tags. + * @return String representation of the chunks + */ + public String toString(boolean forOutput) { + if (resolvedObject instanceof String) { + return (String) resolvedObject; + } + if (resolvedObject == null) { + return forOutput ? "" : JINJAVA_EMPTY_STRING; + } + String asString; + JinjavaInterpreter interpreter = JinjavaInterpreter.getCurrent(); + if (forOutput && interpreter != null) { + asString = + JinjavaInterpreter.getCurrent().getAsString(((List) resolvedObject).get(0)); + } else { + asString = PyishObjectMapper.getAsUnquotedPyishString(resolvedObject); + + if (fullyResolved && StringUtils.isNotEmpty(asString)) { + // Removes surrounding brackets. + asString = asString.substring(1, asString.length() - 1); + } + } + if (JINJAVA_NULL.equals(asString)) { + return forOutput ? "" : JINJAVA_EMPTY_STRING; + } + return asString; + } + + public List toList() { + if (fullyResolved) { + if (resolvedObject instanceof List) { + return (List) resolvedObject; + } else { + return Collections.singletonList(resolvedObject); + } + } + throw new DeferredValueException("Object is not resolved"); + } + + /** + * Method to wrap a string value in the ResolvedChunks class. + * It is not evaluated, rather it's allows a the class to be manually + * built from a partially resolved string. + * @param resolvedString Partially resolved string to wrap. + * @return A ResolvedChunks that {@link #toString()} returns resolvedString. + */ + public static ResolvedChunks fromString(String resolvedString) { + return new ResolvedChunks(resolvedString, false); + } } } diff --git a/src/test/java/com/hubspot/jinjava/EagerTest.java b/src/test/java/com/hubspot/jinjava/EagerTest.java index 4ae02c1a0..c3d26543c 100644 --- a/src/test/java/com/hubspot/jinjava/EagerTest.java +++ b/src/test/java/com/hubspot/jinjava/EagerTest.java @@ -185,8 +185,13 @@ public void itPreserveDeferredVariableResolvingEqualToInOrCondition() { "{% if 'a' is equalto 'b' or 'a' is equalto deferred %}preserved{% endif %}"; String output = interpreter.render(inputOutputExpected); - assertThat(output).isEqualTo(inputOutputExpected); + assertThat(output) + .isEqualTo( + "{% if false || exptest:equalto.evaluate('a', ____int3rpr3t3r____, deferred) %}preserved{% endif %}" + ); assertThat(interpreter.getErrors()).isEmpty(); + localContext.put("deferred", "a"); + assertThat(interpreter.render(output)).isEqualTo("preserved"); } @Test @@ -265,14 +270,20 @@ public void itPreservesForTag() { @Test public void itPreservesFilters() { String output = interpreter.render("{{ deferred|capitalize }}"); - assertThat(output).isEqualTo("{{ deferred|capitalize }}"); + assertThat(output) + .isEqualTo("{{ filter:capitalize.filter(deferred, ____int3rpr3t3r____) }}"); assertThat(interpreter.getErrors()).isEmpty(); + localContext.put("deferred", "foo"); + assertThat(interpreter.render(output)).isEqualTo("Foo"); } @Test public void itPreservesFunctions() { String output = interpreter.render("{{ deferred|datetimeformat('%B %e, %Y') }}"); - assertThat(output).isEqualTo("{{ deferred|datetimeformat('%B %e, %Y') }}"); + assertThat(output) + .isEqualTo( + "{{ filter:datetimeformat.filter(deferred, ____int3rpr3t3r____, '%B %e, %Y') }}" + ); assertThat(interpreter.getErrors()).isEmpty(); } @@ -438,7 +449,9 @@ public void itMarksVariablesUsedAsMapKeysAsDeferred() { DeferredValueUtils.findAndMarkDeferredProperties(localContext); assertThat(deferredValue2).isInstanceOf(DeferredValue.class); assertThat(output) - .contains("{% set varSetInside = imported.map[deferredValue2.nonexistentprop] %}"); + .contains( + "{% set varSetInside = {'key': 'value'}[deferredValue2.nonexistentprop] %}" + ); } @Test @@ -636,6 +649,7 @@ public void itDefersMacroInExpression() { @Test public void itDefersMacroInExpressionSecondPass() { + interpreter.resolveELExpression("(range(0,1))", -1); localContext.put("deferred", 5); expectedTemplateInterpreter.assertExpectedOutput( "defers-macro-in-expression.expected" @@ -806,7 +820,6 @@ public void itHandlesUnknownFunctionErrors() { } assertThat(eagerInterpreter.getErrors()).hasSize(2); assertThat(defaultInterpreter.getErrors()).hasSize(2); - assertThat(eagerInterpreter.getErrors()).isEqualTo(defaultInterpreter.getErrors()); } @Test diff --git a/src/test/java/com/hubspot/jinjava/el/ext/ExtendedParserTest.java b/src/test/java/com/hubspot/jinjava/el/ext/ExtendedParserTest.java index d0a7e6c65..8513d6267 100644 --- a/src/test/java/com/hubspot/jinjava/el/ext/ExtendedParserTest.java +++ b/src/test/java/com/hubspot/jinjava/el/ext/ExtendedParserTest.java @@ -118,6 +118,15 @@ public void itChecksForTupleUntilFinalParentheses() { assertThat(astNode).isInstanceOf(AstTuple.class); } + @Test + public void itParsesExpTestLikeDictionary() { + // Don't want to accidentally try to parse these as a filter or exptest + AstNode astNode = buildExpressionNodes( + "#{{filter:length.filter, exptest:equalto.evaluate}}" + ); + assertThat(astNode).isInstanceOf(AstDict.class); + } + private void assertForExpression( AstNode astNode, String leftExpected, diff --git a/src/test/java/com/hubspot/jinjava/lib/tag/eager/EagerDoTagTest.java b/src/test/java/com/hubspot/jinjava/lib/tag/eager/EagerDoTagTest.java index 437b271dd..1bfb629ef 100644 --- a/src/test/java/com/hubspot/jinjava/lib/tag/eager/EagerDoTagTest.java +++ b/src/test/java/com/hubspot/jinjava/lib/tag/eager/EagerDoTagTest.java @@ -63,7 +63,7 @@ public void itLimitsLength() { tooLong.append(i); } context.setDeferredExecutionMode(true); - interpreter.render(String.format("{%% do deferred.append(%s) %%}", tooLong)); + interpreter.render(String.format("{%% do deferred.append('%s') %%}", tooLong)); assertThat(interpreter.getErrors()).hasSize(1); assertThat(interpreter.getErrors().get(0).getReason()) .isEqualTo(ErrorReason.OUTPUT_TOO_BIG); diff --git a/src/test/java/com/hubspot/jinjava/lib/tag/eager/EagerExtendsTagTest.java b/src/test/java/com/hubspot/jinjava/lib/tag/eager/EagerExtendsTagTest.java index f19f2c178..c0d8249a0 100644 --- a/src/test/java/com/hubspot/jinjava/lib/tag/eager/EagerExtendsTagTest.java +++ b/src/test/java/com/hubspot/jinjava/lib/tag/eager/EagerExtendsTagTest.java @@ -2,6 +2,7 @@ import com.hubspot.jinjava.ExpectedTemplateInterpreter; import com.hubspot.jinjava.JinjavaConfig; +import com.hubspot.jinjava.LegacyOverrides; import com.hubspot.jinjava.interpret.DeferredValue; import com.hubspot.jinjava.interpret.JinjavaInterpreter; import com.hubspot.jinjava.lib.tag.ExtendsTagTest; @@ -24,6 +25,9 @@ public void eagerSetup() { JinjavaConfig .newBuilder() .withExecutionMode(EagerExecutionMode.instance()) + .withLegacyOverrides( + LegacyOverrides.newBuilder().withUsePyishObjectMapper(true).build() + ) .build() ); context.put("deferred", DeferredValue.instance()); diff --git a/src/test/java/com/hubspot/jinjava/lib/tag/eager/EagerForTagTest.java b/src/test/java/com/hubspot/jinjava/lib/tag/eager/EagerForTagTest.java index 3960aa874..6c94b582c 100644 --- a/src/test/java/com/hubspot/jinjava/lib/tag/eager/EagerForTagTest.java +++ b/src/test/java/com/hubspot/jinjava/lib/tag/eager/EagerForTagTest.java @@ -5,6 +5,7 @@ import com.google.common.collect.ImmutableList; import com.hubspot.jinjava.ExpectedNodeInterpreter; import com.hubspot.jinjava.JinjavaConfig; +import com.hubspot.jinjava.LegacyOverrides; import com.hubspot.jinjava.interpret.DeferredValue; import com.hubspot.jinjava.interpret.JinjavaInterpreter; import com.hubspot.jinjava.interpret.TemplateError.ErrorReason; @@ -30,6 +31,9 @@ public void eagerSetup() { .newBuilder() .withMaxOutputSize(MAX_OUTPUT_SIZE) .withExecutionMode(EagerExecutionMode.instance()) + .withLegacyOverrides( + LegacyOverrides.newBuilder().withUsePyishObjectMapper(true).build() + ) .build() ); tag = new EagerForTag(); diff --git a/src/test/java/com/hubspot/jinjava/lib/tag/eager/EagerImportTagTest.java b/src/test/java/com/hubspot/jinjava/lib/tag/eager/EagerImportTagTest.java index eebdbcb65..b7e1d7eb6 100644 --- a/src/test/java/com/hubspot/jinjava/lib/tag/eager/EagerImportTagTest.java +++ b/src/test/java/com/hubspot/jinjava/lib/tag/eager/EagerImportTagTest.java @@ -475,7 +475,7 @@ public void itCorrectlySetsAliasedPathForSecondPass() { assertThat(firstPassResult) .isEqualTo( "{% set deferred_import_resource_path = 'import-macro.jinja' %}{% macro m.print_path_macro(var) %}\n" + - "{{ var|print_path }}\n" + + "{{ filter:print_path.filter(var, ____int3rpr3t3r____) }}\n" + "{{ var }}\n" + "{% endmacro %}{% set deferred_import_resource_path = null %}{{ m.print_path_macro(foo) }}" ); @@ -494,7 +494,7 @@ public void itCorrectlySetsPathForSecondPass() { assertThat(firstPassResult) .isEqualTo( "{% set deferred_import_resource_path = 'import-macro.jinja' %}{% macro print_path_macro(var) %}\n" + - "{{ var|print_path }}\n" + + "{{ filter:print_path.filter(var, ____int3rpr3t3r____) }}\n" + "{{ var }}\n" + "{% endmacro %}{% set deferred_import_resource_path = null %}{{ print_path_macro(foo) }}" ); diff --git a/src/test/java/com/hubspot/jinjava/lib/tag/eager/EagerSetTagTest.java b/src/test/java/com/hubspot/jinjava/lib/tag/eager/EagerSetTagTest.java index b6f547bda..ce96bc99a 100644 --- a/src/test/java/com/hubspot/jinjava/lib/tag/eager/EagerSetTagTest.java +++ b/src/test/java/com/hubspot/jinjava/lib/tag/eager/EagerSetTagTest.java @@ -103,7 +103,7 @@ public void itLimitsLength() { tooLong.append(i); } context.setDeferredExecutionMode(true); - interpreter.render(String.format("{%% set deferred = %s %%}", tooLong)); + interpreter.render(String.format("{%% set deferred = '%s' %%}", tooLong)); assertThat(interpreter.getErrors()).hasSize(1); assertThat(interpreter.getErrors().get(0).getReason()) .isEqualTo(ErrorReason.OUTPUT_TOO_BIG); diff --git a/src/test/java/com/hubspot/jinjava/lib/tag/eager/EagerTagDecoratorTest.java b/src/test/java/com/hubspot/jinjava/lib/tag/eager/EagerTagDecoratorTest.java index b3658edfd..8920fdab2 100644 --- a/src/test/java/com/hubspot/jinjava/lib/tag/eager/EagerTagDecoratorTest.java +++ b/src/test/java/com/hubspot/jinjava/lib/tag/eager/EagerTagDecoratorTest.java @@ -23,6 +23,7 @@ import com.hubspot.jinjava.tree.TagNode; import com.hubspot.jinjava.tree.parse.DefaultTokenScannerSymbols; import com.hubspot.jinjava.tree.parse.TagToken; +import com.hubspot.jinjava.util.ChunkResolver.ResolvedChunks; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -73,14 +74,14 @@ public void itExecutesInChildContextAndTakesNewValue() { ( interpreter1 -> { ((List) interpreter1.getContext().get("foo")).add(1); - return "function return"; + return ResolvedChunks.fromString("function return"); } ), interpreter, true, false ); - assertThat(result.getResult()).isEqualTo("function return"); + assertThat(result.getResult().toString()).isEqualTo("function return"); assertThat(result.getPrefixToPreserveState()).isEqualTo("{% set foo = [1] %}"); assertThat(context.get("foo")).isEqualTo(ImmutableList.of(1)); assertThat(context.getEagerTokens()).isEmpty(); @@ -96,14 +97,14 @@ public void itExecutesInChildContextAndDefersNewValue() { "foo", DeferredValue.instance(interpreter1.getContext().get("foo")) ); - return "function return"; + return ResolvedChunks.fromString("function return"); } ), interpreter, false, false ); - assertThat(result.getResult()).isEqualTo("function return"); + assertThat(result.getResult().toString()).isEqualTo("function return"); assertThat(result.getPrefixToPreserveState()).isEqualTo("{% set foo = [] %}"); assertThat(context.get("foo")).isInstanceOf(DeferredValue.class); assertThat(context.getEagerTokens()).isNotEmpty(); diff --git a/src/test/java/com/hubspot/jinjava/util/ChunkResolverTest.java b/src/test/java/com/hubspot/jinjava/util/ChunkResolverTest.java index c45ed5f16..f9b9773a6 100644 --- a/src/test/java/com/hubspot/jinjava/util/ChunkResolverTest.java +++ b/src/test/java/com/hubspot/jinjava/util/ChunkResolverTest.java @@ -7,10 +7,13 @@ import com.hubspot.jinjava.Jinjava; import com.hubspot.jinjava.JinjavaConfig; import com.hubspot.jinjava.LegacyOverrides; +import com.hubspot.jinjava.el.ext.AbstractCallableMethod; import com.hubspot.jinjava.interpret.Context; import com.hubspot.jinjava.interpret.DeferredValue; import com.hubspot.jinjava.interpret.JinjavaInterpreter; import com.hubspot.jinjava.lib.fn.ELFunctionDefinition; +import com.hubspot.jinjava.mode.EagerExecutionMode; +import com.hubspot.jinjava.objects.collections.PyList; import com.hubspot.jinjava.objects.collections.PyMap; import com.hubspot.jinjava.objects.date.PyishDate; import com.hubspot.jinjava.objects.serialization.PyishObjectMapper; @@ -21,12 +24,14 @@ import java.time.Instant; import java.time.ZoneId; import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import org.junit.After; import org.junit.Before; -import org.junit.Ignore; import org.junit.Test; public class ChunkResolverTest { @@ -45,6 +50,7 @@ private JinjavaInterpreter getInterpreter(boolean evaluateMapKeys) throws Except Jinjava jinjava = new Jinjava( JinjavaConfig .newBuilder() + .withExecutionMode(EagerExecutionMode.instance()) .withLegacyOverrides( LegacyOverrides.newBuilder().withEvaluateMapKeys(evaluateMapKeys).build() ) @@ -88,12 +94,13 @@ private ChunkResolver makeChunkResolver(String string) { public void itResolvesDeferredBoolean() { context.put("foo", "foo_val"); ChunkResolver chunkResolver = makeChunkResolver("(111 == 112) or (foo == deferred)"); - String partiallyResolved = chunkResolver.resolveChunks(); - assertThat(partiallyResolved).isEqualTo("false or ('foo_val' == deferred)"); + String partiallyResolved = chunkResolver.resolveChunks().toString(); + assertThat(partiallyResolved).isEqualTo("false || ('foo_val' == deferred)"); assertThat(chunkResolver.getDeferredWords()).containsExactly("deferred"); context.put("deferred", "foo_val"); - assertThat(makeChunkResolver(partiallyResolved).resolveChunks()).isEqualTo("true"); + assertThat(makeChunkResolver(partiallyResolved).resolveChunks().toString()) + .isEqualTo("true"); assertThat(interpreter.resolveELExpression(partiallyResolved, 1)).isEqualTo(true); } @@ -102,17 +109,19 @@ public void itResolvesDeferredList() { context.put("foo", "foo_val"); context.put("bar", "bar_val"); ChunkResolver chunkResolver = makeChunkResolver("[foo == bar, deferred, bar]"); - assertThat(chunkResolver.resolveChunks()).isEqualTo("[false, deferred, 'bar_val']"); + assertThat(chunkResolver.resolveChunks().toString()) + .isEqualTo("[false, deferred, 'bar_val']"); assertThat(chunkResolver.getDeferredWords()).containsExactlyInAnyOrder("deferred"); context.put("bar", "foo_val"); - assertThat(chunkResolver.resolveChunks()).isEqualTo("[true, deferred, 'foo_val']"); + assertThat(chunkResolver.resolveChunks().toString()) + .isEqualTo("[true, deferred, 'foo_val']"); } @Test public void itResolvesSimpleBoolean() { context.put("foo", true); ChunkResolver chunkResolver = makeChunkResolver("false || (foo), 'bar'"); - String partiallyResolved = chunkResolver.resolveChunks(); + String partiallyResolved = chunkResolver.resolveChunks().toString(); assertThat(partiallyResolved).isEqualTo("true, 'bar'"); assertThat(chunkResolver.getDeferredWords()).isEmpty(); } @@ -121,7 +130,7 @@ public void itResolvesSimpleBoolean() { @SuppressWarnings("unchecked") public void itResolvesRange() { ChunkResolver chunkResolver = makeChunkResolver("range(0,2)"); - String partiallyResolved = chunkResolver.resolveChunks(); + String partiallyResolved = chunkResolver.resolveChunks().toString(); assertThat(partiallyResolved).isEqualTo("[0, 1]"); assertThat(chunkResolver.getDeferredWords()).isEmpty(); // I don't know why this is a list of longs? @@ -136,14 +145,14 @@ public void itResolvesDeferredRange() throws Exception { context.put("foo", 1); context.put("bar", 3); ChunkResolver chunkResolver = makeChunkResolver("range(deferred, foo + bar)"); - String partiallyResolved = chunkResolver.resolveChunks(); + String partiallyResolved = chunkResolver.resolveChunks().toString(); assertThat(partiallyResolved).isEqualTo("range(deferred, 4)"); assertThat(chunkResolver.getDeferredWords()) .containsExactlyInAnyOrder("deferred", "range"); context.put("deferred", 1); - assertThat(makeChunkResolver(partiallyResolved).resolveChunks()) - .isEqualTo(new PyishObjectMapper().getAsPyishString(expectedList)); + assertThat(makeChunkResolver(partiallyResolved).resolveChunks().toString()) + .isEqualTo(PyishObjectMapper.getAsPyishString(expectedList)); // But this is a list of integers assertThat((List) interpreter.resolveELExpression(partiallyResolved, 1)) .isEqualTo(expectedList); @@ -156,7 +165,7 @@ public void itResolvesDictionary() { context.put("the_dictionary", dict); ChunkResolver chunkResolver = makeChunkResolver("[the_dictionary, 1]"); - String partiallyResolved = chunkResolver.resolveChunks(); + String partiallyResolved = chunkResolver.resolveChunks().toString(); assertThat(chunkResolver.getDeferredWords()).isEmpty(); assertThat(interpreter.resolveELExpression(partiallyResolved, 1)) .isEqualTo(ImmutableList.of(dict, 1L)); @@ -169,13 +178,13 @@ public void itResolvesNested() { ChunkResolver chunkResolver = makeChunkResolver( "[foo, range(deferred, bar), range(foo, bar)][0:2]" ); - String partiallyResolved = chunkResolver.resolveChunks(); + String partiallyResolved = chunkResolver.resolveChunks().toString(); assertThat(partiallyResolved).isEqualTo("[1, range(deferred, 3), [1, 2]][0:2]"); assertThat(chunkResolver.getDeferredWords()) .containsExactlyInAnyOrder("deferred", "range"); context.put("deferred", 2); - assertThat(makeChunkResolver(partiallyResolved).resolveChunks()) + assertThat(makeChunkResolver(partiallyResolved).resolveChunks().toString()) .isEqualTo("[1, [2]]"); assertThat(interpreter.resolveELExpression(partiallyResolved, 1)) .isEqualTo(ImmutableList.of(1L, ImmutableList.of(2))); @@ -186,12 +195,12 @@ public void itSplitsOnNonWords() { context.put("foo", 1); context.put("bar", 4); ChunkResolver chunkResolver = makeChunkResolver("range(0,foo) + -deferred/bar"); - String partiallyResolved = chunkResolver.resolveChunks(); - assertThat(partiallyResolved).isEqualTo("[0] + -deferred/4"); + String partiallyResolved = chunkResolver.resolveChunks().toString(); + assertThat(partiallyResolved).isEqualTo("[0] + -deferred / 4"); assertThat(chunkResolver.getDeferredWords()).containsExactly("deferred"); context.put("deferred", 2); - assertThat(makeChunkResolver(partiallyResolved).resolveChunks()) + assertThat(makeChunkResolver(partiallyResolved).resolveChunks().toString()) .isEqualTo("[0, -0.5]"); assertThat(interpreter.resolveELExpression(partiallyResolved, 1)) .isEqualTo(ImmutableList.of(0L, -0.5)); @@ -202,26 +211,26 @@ public void itSplitsAndIndexesOnNonWords() { context.put("foo", 3); context.put("bar", 4); ChunkResolver chunkResolver = makeChunkResolver("range(-2,foo)[-1] + -deferred/bar"); - String partiallyResolved = chunkResolver.resolveChunks(); - assertThat(partiallyResolved).isEqualTo("2 + -deferred/4"); + String partiallyResolved = chunkResolver.resolveChunks().toString(); + assertThat(partiallyResolved).isEqualTo("2 + -deferred / 4"); assertThat(chunkResolver.getDeferredWords()).containsExactly("deferred"); context.put("deferred", 2); - assertThat(makeChunkResolver(partiallyResolved).resolveChunks()).isEqualTo("1.5"); + assertThat(makeChunkResolver(partiallyResolved).resolveChunks().toString()) + .isEqualTo("1.5"); assertThat(interpreter.resolveELExpression(partiallyResolved, 1)).isEqualTo(1.5); } @Test - @Ignore - // TODO support order of operations public void itSupportsOrderOfOperations() { ChunkResolver chunkResolver = makeChunkResolver("[0,1]|reverse + deferred"); - String partiallyResolved = chunkResolver.resolveChunks(); - assertThat(partiallyResolved).isEqualTo("[1,0] + deferred"); + String partiallyResolved = chunkResolver.resolveChunks().toString(); + assertThat(partiallyResolved).isEqualTo("[1, 0] + deferred"); assertThat(chunkResolver.getDeferredWords()).containsExactly("deferred"); - context.put("deferred", 2); - assertThat(makeChunkResolver(partiallyResolved).resolveChunks()).isEqualTo("[1,0,2]"); + context.put("deferred", 2L); + assertThat(makeChunkResolver(partiallyResolved).resolveChunks().toString()) + .isEqualTo("[1, 0, 2]"); assertThat(interpreter.resolveELExpression(partiallyResolved, 1)) .isEqualTo(ImmutableList.of(1L, 0L, 2L)); } @@ -229,40 +238,21 @@ public void itSupportsOrderOfOperations() { @Test public void itCatchesDeferredVariables() { ChunkResolver chunkResolver = makeChunkResolver("range(0, deferred)"); - String partiallyResolved = chunkResolver.resolveChunks(); + String partiallyResolved = chunkResolver.resolveChunks().toString(); assertThat(partiallyResolved).isEqualTo("range(0, deferred)"); // Since the range function is deferred, it is added to deferredWords. assertThat(chunkResolver.getDeferredWords()) .containsExactlyInAnyOrder("range", "deferred"); } - @Test - public void itSplitsChunks() { - ChunkResolver chunkResolver = makeChunkResolver("1, 1 + 1, 1 + 2"); - List miniChunks = chunkResolver.splitChunks(); - assertThat(miniChunks).containsExactly("1", "2", "3"); - assertThat(chunkResolver.getDeferredWords()).isEmpty(); - } - - @Test - public void itProperlySplitsMultiLevelChunks() { - ChunkResolver chunkResolver = makeChunkResolver( - "[5,7], 1 + 1, 1 + range(0 + 1, deferred)" - ); - List miniChunks = chunkResolver.splitChunks(); - assertThat(miniChunks).containsExactly("[5, 7]", "2", "1 + range(1, deferred)"); - assertThat(chunkResolver.getDeferredWords()) - .containsExactlyInAnyOrder("range", "deferred"); - } - @Test public void itDoesntDeferReservedWords() { context.put("foo", 0); ChunkResolver chunkResolver = makeChunkResolver( - "[(foo > 1) or deferred, deferred].append(1)" + "[(foo > 1) || deferred, deferred].append(1)" ); - String partiallyResolved = chunkResolver.resolveChunks(); - assertThat(partiallyResolved).isEqualTo("[false or deferred, deferred].append(1)"); + String partiallyResolved = chunkResolver.resolveChunks().toString(); + assertThat(partiallyResolved).isEqualTo("[false || deferred, deferred].append(1)"); assertThat(chunkResolver.getDeferredWords()).doesNotContain("false", "or"); assertThat(chunkResolver.getDeferredWords()).contains("deferred", ".append"); } @@ -271,7 +261,7 @@ public void itDoesntDeferReservedWords() { public void itEvaluatesDict() { context.put("foo", new PyMap(ImmutableMap.of("bar", 99))); ChunkResolver chunkResolver = makeChunkResolver("foo.bar == deferred.bar"); - String partiallyResolved = chunkResolver.resolveChunks(); + String partiallyResolved = chunkResolver.resolveChunks().toString(); assertThat(partiallyResolved).isEqualTo("99 == deferred.bar"); assertThat(chunkResolver.getDeferredWords()) .containsExactlyInAnyOrder("deferred.bar"); @@ -285,10 +275,9 @@ public void itSerializesDateProperly() { context.put("date", date); ChunkResolver chunkResolver = makeChunkResolver("date"); - // don't prematurely resolve date because of datetime functions. - assertThat(WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks())) - .isEqualTo("date"); - assertThat(WhitespaceUtils.unquoteAndUnescape(interpreter.resolveString("date", -1))) + assertThat( + WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks().toString()) + ) .isEqualTo(date.toString()); } @@ -299,7 +288,9 @@ public void itHandlesSingleQuotes() { ChunkResolver chunkResolver = makeChunkResolver( "foo ~ ' & ' ~ bar ~ ' & ' ~ '\\'\\\"'" ); - assertThat(WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks())) + assertThat( + WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks().toString()) + ) .isEqualTo("' & ' & '\""); } @@ -310,14 +301,18 @@ public void itHandlesNewlines() { ChunkResolver chunkResolver = makeChunkResolver( "foo ~ ' & ' ~ bar ~ ' & ' ~ '\\\\' ~ 'n' ~ ' & \\\\n'" ); - assertThat(WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks())) + assertThat( + WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks().toString()) + ) .isEqualTo("\n & \\n & \\n & \\n"); } @Test public void itOutputsUnknownVariablesAsEmpty() { ChunkResolver chunkResolver = makeChunkResolver("contact.some_odd_property"); - assertThat(WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks())) + assertThat( + WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks().toString()) + ) .isEqualTo(""); } @@ -325,7 +320,9 @@ public void itOutputsUnknownVariablesAsEmpty() { public void itHandlesCancellingSlashes() { context.put("foo", "bar"); ChunkResolver chunkResolver = makeChunkResolver("foo ~ 'foo\\\\' ~ foo ~ 'foo'"); - assertThat(WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks())) + assertThat( + WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks().toString()) + ) .isEqualTo("barfoo\\barfoo"); } @@ -337,7 +334,7 @@ public void itOutputsEmptyForVoidFunctions() throws Exception { .isEmpty(); assertThat( WhitespaceUtils.unquoteAndUnescape( - makeChunkResolver("void_function(2)").resolveChunks() + makeChunkResolver("void_function(2)").resolveChunks().toString() ) ) .isEmpty(); @@ -345,24 +342,28 @@ public void itOutputsEmptyForVoidFunctions() throws Exception { @Test public void itOutputsNullAsEmptyString() { - assertThat(makeChunkResolver("void_function(2)").resolveChunks()).isEqualTo("''"); - assertThat(makeChunkResolver("nothing").resolveChunks()).isEqualTo("''"); + assertThat(makeChunkResolver("void_function(2)").resolveChunks().toString()) + .isEqualTo("''"); + assertThat(makeChunkResolver("nothing").resolveChunks().toString()).isEqualTo("''"); } @Test public void itInterpretsNullAsNull() { - assertThat(makeChunkResolver("is_null(nothing, null)").resolveChunks()) + assertThat(makeChunkResolver("is_null(nothing, null)").resolveChunks().toString()) .isEqualTo("true"); - assertThat(makeChunkResolver("is_null(void_function(2), nothing)").resolveChunks()) + assertThat( + makeChunkResolver("is_null(void_function(2), nothing)").resolveChunks().toString() + ) .isEqualTo("true"); - assertThat(makeChunkResolver("is_null('', nothing)").resolveChunks()) + assertThat(makeChunkResolver("is_null('', nothing)").resolveChunks().toString()) .isEqualTo("false"); } @Test public void itDoesntDeferNull() { ChunkResolver chunkResolver = makeChunkResolver("range(deferred, nothing)"); - assertThat(chunkResolver.resolveChunks()).isEqualTo("range(deferred, null)"); + assertThat(chunkResolver.resolveChunks().toString()) + .isEqualTo("range(deferred, null)"); assertThat(chunkResolver.getDeferredWords()) .containsExactlyInAnyOrder("range", "deferred"); } @@ -371,7 +372,7 @@ public void itDoesntDeferNull() { public void itDoesntSplitOnBar() { context.put("date", new PyishDate(0L)); ChunkResolver chunkResolver = makeChunkResolver("date|datetimeformat('%Y')"); - assertThat(chunkResolver.resolveChunks()).isEqualTo("1970"); + assertThat(chunkResolver.resolveChunks().toString()).isEqualTo("1970"); } @Test @@ -380,8 +381,10 @@ public void itDoesntResolveNonPyishSerializable() { context.put("dict", dict); context.put("foo", new Foo("bar")); context.put("mark", "!"); - ChunkResolver chunkResolver = makeChunkResolver("(dict.update({'foo': foo})"); - assertThat(WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks())) + ChunkResolver chunkResolver = makeChunkResolver("dict.update({'foo': foo})"); + assertThat( + WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks().toString()) + ) .isEqualTo(""); assertThat(dict.get("foo")).isInstanceOf(Foo.class); assertThat(((Foo) dict.get("foo")).bar()).isEqualTo("bar"); @@ -394,7 +397,9 @@ public void itLeavesPaddedZeros() { ZonedDateTime.ofInstant(Instant.ofEpochMilli(0L), ZoneId.systemDefault()) ); ChunkResolver chunkResolver = makeChunkResolver("zero_date.strftime('%d')"); - assertThat(WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks())) + assertThat( + WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks().toString()) + ) .isEqualTo("01"); } @@ -403,7 +408,9 @@ public void itPreservesLengthyDoubleStrings() { // does not convert to scientific notation context.put("small", "0.0000000001"); ChunkResolver chunkResolver = makeChunkResolver("small"); - assertThat(WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks())) + assertThat( + WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks().toString()) + ) .isEqualTo("0.0000000001"); } @@ -412,14 +419,16 @@ public void itConvertsDoubles() { // does convert to scientific notation context.put("small", 0.0000000001); ChunkResolver chunkResolver = makeChunkResolver("small"); - assertThat(WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks())) + assertThat( + WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks().toString()) + ) .isEqualTo("1.0E-10"); } @Test public void itDoesntQuoteFloats() { ChunkResolver chunkResolver = makeChunkResolver("0.4 + 0.1"); - assertThat(chunkResolver.resolveChunks()).isEqualTo("0.5"); + assertThat(chunkResolver.resolveChunks().toString()).isEqualTo("0.5"); } @Test @@ -427,7 +436,9 @@ public void itHandlesWhitespaceAroundPipe() { String lowerFilterString = "'AB' | truncate(1) ~ 'BC' |truncate(1) ~ 'CD'| truncate(1)"; ChunkResolver chunkResolver = makeChunkResolver(lowerFilterString); - assertThat(WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks())) + assertThat( + WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks().toString()) + ) .isEqualTo(interpreter.resolveELExpression(lowerFilterString, 0)); } @@ -435,7 +446,9 @@ public void itHandlesWhitespaceAroundPipe() { public void itHandlesMultipleWhitespaceAroundPipe() { String lowerFilterString = "'AB' | truncate(1)"; ChunkResolver chunkResolver = makeChunkResolver(lowerFilterString); - assertThat(WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks())) + assertThat( + WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks().toString()) + ) .isEqualTo(interpreter.resolveELExpression(lowerFilterString, 0)); } @@ -443,7 +456,9 @@ public void itHandlesMultipleWhitespaceAroundPipe() { public void itEscapesFormFeed() { context.put("foo", "Form feed\f"); ChunkResolver chunkResolver = makeChunkResolver("foo"); - assertThat(WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks())) + assertThat( + WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks().toString()) + ) .isEqualTo("Form feed\f"); } @@ -452,8 +467,10 @@ public void itHandlesUnconventionalSpacing() { ChunkResolver chunkResolver = makeChunkResolver( "( range (0 , 3 ) [ 1] + deferred) ~ 'YES'| lower" ); - String result = WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks()); - assertThat(result).isEqualTo("( 1 + deferred) ~ 'yes'"); + String result = WhitespaceUtils.unquoteAndUnescape( + chunkResolver.resolveChunks().toString() + ); + assertThat(result).isEqualTo("(1 + deferred) ~ 'yes'"); context.put("deferred", 2); assertThat(interpreter.resolveELExpression(result, 0)).isEqualTo("3yes"); } @@ -463,7 +480,9 @@ public void itHandlesDotSpacing() { context.put("bar", "fake"); context.put("foo", ImmutableMap.of("bar", "foobar")); ChunkResolver chunkResolver = makeChunkResolver("foo . bar"); - String result = WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks()); + String result = WhitespaceUtils.unquoteAndUnescape( + chunkResolver.resolveChunks().toString() + ); assertThat(result).isEqualTo("foobar"); } @@ -471,7 +490,9 @@ public void itHandlesDotSpacing() { public void itPreservesLegacyDictionaryCreation() { context.put("foo", "not_foo"); ChunkResolver chunkResolver = makeChunkResolver("{foo: 'bar'}"); - String result = WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks()); + String result = WhitespaceUtils.unquoteAndUnescape( + chunkResolver.resolveChunks().toString() + ); assertThat(result).isEqualTo("{'foo': 'bar'}"); } @@ -481,7 +502,9 @@ public void itHandlesPythonicDictionaryCreation() throws Exception { try { context.put("foo", "not_foo"); ChunkResolver chunkResolver = makeChunkResolver("{foo: 'bar'}"); - String result = WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks()); + String result = WhitespaceUtils.unquoteAndUnescape( + chunkResolver.resolveChunks().toString() + ); assertThat(result).isEqualTo("{'not_foo': 'bar'}"); } finally { JinjavaInterpreter.popCurrent(); @@ -492,7 +515,9 @@ public void itHandlesPythonicDictionaryCreation() throws Exception { public void itKeepsPlusSignPrefix() { context.put("foo", "+12223334444"); ChunkResolver chunkResolver = makeChunkResolver("foo"); - assertThat(WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks())) + assertThat( + WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks().toString()) + ) .isEqualTo("+12223334444"); } @@ -500,7 +525,9 @@ public void itKeepsPlusSignPrefix() { public void itHandlesPhoneNumbers() { context.put("foo", "+1(123)456-7890"); ChunkResolver chunkResolver = makeChunkResolver("foo"); - assertThat(WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks())) + assertThat( + WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks().toString()) + ) .isEqualTo("+1(123)456-7890"); } @@ -508,7 +535,9 @@ public void itHandlesPhoneNumbers() { public void itHandlesNegativeZero() { context.put("foo", "-0"); ChunkResolver chunkResolver = makeChunkResolver("foo"); - assertThat(WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks())) + assertThat( + WhitespaceUtils.unquoteAndUnescape(chunkResolver.resolveChunks().toString()) + ) .isEqualTo("-0"); } @@ -517,17 +546,131 @@ public void itHandlesPyishSerializable() { context.put("foo", new SomethingPyish("yes")); assertThat( interpreter.render( - String.format("{{ %s.name }}", makeChunkResolver("foo").resolveChunks()) + String.format( + "{{ %s.name }}", + makeChunkResolver("foo").resolveChunks().toString() + ) ) ) .isEqualTo("yes"); } + @Test + public void itFinishesResolvingList() { + assertThat(makeChunkResolver("[0 + 1, deferred, 2 + 1]").resolveChunks().toString()) + .isEqualTo("[1, deferred, 3]"); + } + + @Test + public void itHandlesExtraSapces() { + context.put("foo", " foo"); + assertThat(makeChunkResolver("foo").resolveChunks().toString()).isEqualTo("' foo'"); + } + + @Test + public void itHandlesDeferredExpTests() { + context.put("foo", 4); + ChunkResolver chunkResolver = makeChunkResolver("foo is not equalto deferred"); + interpreter.getContext().setThrowInterpreterErrors(true); + String partiallyResolved = chunkResolver.resolveChunks().toString(); + assertThat(partiallyResolved) + .isEqualTo("exptest:equalto.evaluateNegated(4, ____int3rpr3t3r____, deferred)"); + assertThat(chunkResolver.getDeferredWords()) + .containsExactlyInAnyOrder("deferred", "equalto.evaluateNegated"); + context.put("deferred", 4); + assertThat(makeChunkResolver(partiallyResolved).resolveChunks().toString()) + .isEqualTo("false"); + context.put("deferred", 1); + assertThat(makeChunkResolver(partiallyResolved).resolveChunks().toString()) + .isEqualTo("true"); + } + + @Test + public void itHandlesDeferredChoice() { + context.put("foo", "foo"); + context.put("bar", "bar"); + assertThat(makeChunkResolver("deferred ? foo : bar").resolveChunks().toString()) + .isEqualTo("deferred ? 'foo' : 'bar'"); + assertThat(makeChunkResolver("true ? deferred : bar").resolveChunks().toString()) + .isEqualTo("deferred"); + assertThat(makeChunkResolver("false ? foo : deferred").resolveChunks().toString()) + .isEqualTo("deferred"); + } + + @Test + public void itHandlesDeferredNamedParameter() { + context.put("foo", "foo"); + assertThat(makeChunkResolver("x=foo, y=deferred").resolveChunks().toString()) + .isEqualTo("x='foo', y=deferred"); + } + + @Test + public void itHandlesDeferredValueInList() { + context.put("foo", "foo"); + assertThat(makeChunkResolver("[foo, deferred, foo ~ '!']").resolveChunks().toString()) + .isEqualTo("['foo', deferred, 'foo!']"); + } + + @Test + public void itHandlesDeferredValueInTuple() { + context.put("foo", "foo"); + assertThat(makeChunkResolver("(foo, deferred, foo ~ '!')").resolveChunks().toString()) + .isEqualTo("('foo', deferred, 'foo!')"); + } + + @Test + public void itHandlesDeferredMethod() { + context.put("foo", "foo"); + context.put("my_list", new PyList(new ArrayList<>())); + assertThat( + makeChunkResolver("my_list.append(deferred ~ foo)").resolveChunks().toString() + ) + .isEqualTo("my_list.append(deferred ~ 'foo')"); + assertThat(makeChunkResolver("deferred.append(foo)").resolveChunks().toString()) + .isEqualTo("deferred.append('foo')"); + assertThat(makeChunkResolver("deferred[1 + 1] | length").resolveChunks().toString()) + .isEqualTo("filter:length.filter(deferred[2], ____int3rpr3t3r____)"); + } + + @Test + public void itHandlesDeferredBracketMethod() throws NoSuchMethodException { + context.put("zero", 0); + context.put("foo", "foo"); + LinkedHashMap map = new LinkedHashMap<>(); + map.put("string", null); + context.put( + "my_list", + new PyList( + Collections.singletonList( + new AbstractCallableMethod("echo", map) { + + @Override + public Object doEvaluate( + Map argMap, + Map kwargMap, + List varArgs + ) { + return argMap.get("string"); + } + } + ) + ) + ); + assertThat(makeChunkResolver("my_list[zero](foo)").resolveChunks().toString()) + .isEqualTo("'foo'"); + assertThat( + makeChunkResolver("my_list[zero](deferred ~ foo)").resolveChunks().toString() + ) + .isEqualTo("my_list[0](deferred ~ 'foo')"); + } + @Test public void itHandlesOrOperator() { assertThat( WhitespaceUtils.unquoteAndUnescape( - makeChunkResolver("false == true || (true) ? 'yes' : 'no'").resolveChunks() + makeChunkResolver("false == true || (true) ? 'yes' : 'no'") + .resolveChunks() + .toString() ) ) .isEqualTo("yes"); @@ -549,6 +692,10 @@ private class Foo { String bar() { return bar; } + + String echo(String toEcho) { + return toEcho; + } } public class SomethingPyish implements PyishSerializable { diff --git a/src/test/resources/eager/defers-macro-in-expression.expected.jinja b/src/test/resources/eager/defers-macro-in-expression.expected.jinja index 0503f4655..96f2afbbc 100644 --- a/src/test/resources/eager/defers-macro-in-expression.expected.jinja +++ b/src/test/resources/eager/defers-macro-in-expression.expected.jinja @@ -1,3 +1,3 @@ 2 -{% macro plus(foo, add) %}{{ foo + (add|int) }}{% endmacro %}{{ plus(deferred, 1.1) }}{% set deferred = deferred + 2 %} -{% macro plus(foo, add) %}{{ foo + (add|int) }}{% endmacro %}{{ plus(deferred, 3.1) }} +{% macro plus(foo, add) %}{{ foo + (filter:int.filter(add, ____int3rpr3t3r____)) }}{% endmacro %}{{ plus(deferred, 1.1) }}{% set deferred = deferred + 2 %} +{% macro plus(foo, add) %}{{ foo + (filter:int.filter(add, ____int3rpr3t3r____)) }}{% endmacro %}{{ plus(deferred, 3.1) }} \ No newline at end of file diff --git a/src/test/resources/eager/defers-macro-in-for.expected.jinja b/src/test/resources/eager/defers-macro-in-for.expected.jinja index 3276d84df..93d3b72ac 100644 --- a/src/test/resources/eager/defers-macro-in-for.expected.jinja +++ b/src/test/resources/eager/defers-macro-in-for.expected.jinja @@ -1,3 +1,3 @@ -{% set my_list = [] %}{% set my_list = [] %}{% macro macro_append(num) %}{% do my_list.append(num) %}{{ my_list }}{% endmacro %}{% for item in macro_append(deferred)|split(',',2) %} +{% set my_list = [] %}{% set my_list = [] %}{% macro macro_append(num) %}{% do my_list.append(num) %}{{ my_list }}{% endmacro %}{% for item in filter:split.filter(macro_append(deferred), ____int3rpr3t3r____, ',', 2) %} {{ item }} {% endfor %} diff --git a/src/test/resources/eager/defers-macro-in-if.expected.jinja b/src/test/resources/eager/defers-macro-in-if.expected.jinja index 52be01cc7..5e8ad31cf 100644 --- a/src/test/resources/eager/defers-macro-in-if.expected.jinja +++ b/src/test/resources/eager/defers-macro-in-if.expected.jinja @@ -1,3 +1,3 @@ -{% set my_list = [] %}{% set my_list = [] %}{% macro macro_append(num) %}{% do my_list.append(num) %}{{ my_list }}{% endmacro %}{% if [] == macro_append(deferred)|split(',',2) %} +{% set my_list = [] %}{% set my_list = [] %}{% macro macro_append(num) %}{% do my_list.append(num) %}{{ my_list }}{% endmacro %}{% if [] == filter:split.filter(macro_append(deferred), ____int3rpr3t3r____, ',', 2) %} {{ my_list }} {% endif %} diff --git a/src/test/resources/eager/evaluates-non-eager-set.expected.jinja b/src/test/resources/eager/evaluates-non-eager-set.expected.jinja index c160f4638..7c4f02cd9 100644 --- a/src/test/resources/eager/evaluates-non-eager-set.expected.jinja +++ b/src/test/resources/eager/evaluates-non-eager-set.expected.jinja @@ -1,4 +1,4 @@ -{% for item in [0, 1, 2, 3, 4]+deferred %} +{% for item in [0, 1, 2, 3, 4] + deferred %} {% if item == 5 %} It is foo! {% else %} diff --git a/src/test/resources/eager/handles-deferred-cycle-as.expected.jinja b/src/test/resources/eager/handles-deferred-cycle-as.expected.jinja index ca191b17b..4ff70372c 100644 --- a/src/test/resources/eager/handles-deferred-cycle-as.expected.jinja +++ b/src/test/resources/eager/handles-deferred-cycle-as.expected.jinja @@ -1,6 +1,6 @@ -{% set c = [1,deferred] %} -{% if c is iterable %}{{ c[0 % c|length] }}{% else %}{{ c }}{% endif %} -{% set c = [2,deferred] %} -{% if c is iterable %}{{ c[1 % c|length] }}{% else %}{{ c }}{% endif %} -{% set c = [3,deferred] %} -{% if c is iterable %}{{ c[2 % c|length] }}{% else %}{{ c }}{% endif %} +{% set c = [1, deferred] %} +{% if exptest:iterable.evaluate(c, ____int3rpr3t3r____) %}{{ c[0 % filter:length.filter(c, ____int3rpr3t3r____)] }}{% else %}{{ c }}{% endif %} +{% set c = [2, deferred] %} +{% if exptest:iterable.evaluate(c, ____int3rpr3t3r____) %}{{ c[1 % filter:length.filter(c, ____int3rpr3t3r____)] }}{% else %}{{ c }}{% endif %} +{% set c = [3, deferred] %} +{% if exptest:iterable.evaluate(c, ____int3rpr3t3r____) %}{{ c[2 % filter:length.filter(c, ____int3rpr3t3r____)] }}{% else %}{{ c }}{% endif %} \ No newline at end of file diff --git a/src/test/resources/eager/handles-deferred-import-vars.expected.jinja b/src/test/resources/eager/handles-deferred-import-vars.expected.jinja index c0b31b6df..56cfc04fa 100644 --- a/src/test/resources/eager/handles-deferred-import-vars.expected.jinja +++ b/src/test/resources/eager/handles-deferred-import-vars.expected.jinja @@ -5,7 +5,7 @@ Hello {{ myname }} foo: Hello {{ myname }} bar: {{ bar }} ---{% set myname = deferred + 7 %} -{% set bar = myname + 19 %}{% set simple = {} %}{% do simple.update({'bar': bar}) %} +{% set bar = myname + 19 %}{% set simple = {} %}{% do simple.update({"bar": bar}) %} Hello {{ myname }} {% do simple.update({'import_resource_path': 'macro-and-set.jinja'}) %} simple.foo: {% set deferred_import_resource_path = 'macro-and-set.jinja' %}{% macro simple.foo() %}Hello {{ myname }}{% endmacro %}{% set deferred_import_resource_path = null %}{{ simple.foo() }}