Additional changes to ExpressionTransformer

docs/api-reference/expression-estimator.md

@@ -118,12 +118,12 @@ which uses the standard conversion parsing. When converting from a floating poin
 
 | **​Name** | ​ **Meaning** | ​ **Comments** |
 | --- | --- | --- |
-| ​bool | ​convert to BL | The operand must be text or boolean. |
-| ​int | convert to I4 | ​The input may be of any type. |
-| ​long | ​convert to I8 | The input may be of any type. |
-| ​single, float | ​convert to R4 | ​The input may be of any type. |
-| ​double | ​convert to R8 | ​​The input may be of any type. |
-| ​text | ​convert to TX | ​​​The input may be of any type. This produces a default text representation. |
+| ​bool | ​convert to Boolean | The operand must be text or boolean. |
+| ​int | convert to <xref:System.Int32> | ​The input may be of any type. |
+| ​long | ​convert to <xref:System.Int64> | The input may be of any type. |
+| ​single, float | ​convert to <xref:System.Single> | ​The input may be of any type. |
+| ​double | ​convert to <xref:System.Double> | ​​The input may be of any type. |
+| ​text | ​convert to [text](xref:Microsoft.ML.Data.TextDataViewType) | ​​​The input may be of any type. This produces a default text representation. |
 
 The unary functions that require a numeric operand are listed in the following table. The result type is the same as the operand type. An NA operand value produces NA.
 

src/Microsoft.ML.Transforms/Expression/BuiltinFunctions.cs

@@ -950,8 +950,9 @@ public static BL True()
 
         /// <summary>
         /// Raise a to the b power. Special cases:
-        /// * 1^NA => 1
-        /// * NA^0 => 1
+        /// * a^(negative value) => 0
+        /// * In case of overflow, return I4.MinValue or I4.MaxValue, based on whether the result would have been
+        /// negative or positive.
         /// </summary>
         public static I4 Pow(I4 a, I4 b)
         {
@@ -969,11 +970,18 @@ public static I4 Pow(I4 a, I4 b)
             if (a == -1)
                 return (b & 1) == 0 ? 1 : -1;
             if (b < 0)
-                throw Contracts.Except("Cannot raise an integer to a negative power");
+                return 0;
+
+            bool neg = false;
+            if (a < 0)
+            {
+                a = -a;
+                neg = (b & 1) != 0;
+            }
 
             // Since the abs of the base is at least two, the exponent must be less than 31.
             if (b >= 31)
-                throw Contracts.Except("Cannot raise an integer to a power greater than 30");
+                return neg ? I4.MinValue : I4.MaxValue;
 
             if (a == 0)
             {
@@ -982,31 +990,25 @@ public static I4 Pow(I4 a, I4 b)
                 return 0;
             }
 
-            bool neg = false;
-            if (a < 0)
-            {
-                a = -a;
-                neg = (b & 1) != 0;
-            }
             Contracts.Assert(a >= 2);
 
             // Since the exponent is at least three, the base must be <= 1290.
             Contracts.Assert(b >= 3);
             if (a > 1290)
-                throw Contracts.Except($"Base must be at most 1290 when raising to the power of {b}");
+                return neg ? I4.MinValue : I4.MaxValue;
 
             // REVIEW: Should we use a checked context and exception catching like I8 does?
             ulong u = (ulong)(uint)a;
             ulong result = 1;
             for (; ; )
             {
                 if ((b & 1) != 0 && (result *= u) > I4.MaxValue)
-                    throw Contracts.Except("Overflow");
+                    return neg ? I4.MinValue : I4.MaxValue;
                 b >>= 1;
                 if (b == 0)
                     break;
                 if ((u *= u) > I4.MaxValue)
-                    throw Contracts.Except("Overflow");
+                    return neg ? I4.MinValue : I4.MaxValue;
             }
             Contracts.Assert(result <= I4.MaxValue);
 
@@ -1018,8 +1020,9 @@ public static I4 Pow(I4 a, I4 b)
 
         /// <summary>
         /// Raise a to the b power. Special cases:
-        /// * 1^NA => 1
-        /// * NA^0 => 1
+        /// * a^(negative value) => 0
+        /// * In case of overflow, return I8.MinValue or I8.MaxValue, based on whether the result would have been
+        /// negative or positive.
         /// </summary>
         public static I8 Pow(I8 a, I8 b)
         {
@@ -1037,11 +1040,18 @@ public static I8 Pow(I8 a, I8 b)
             if (a == -1)
                 return (b & 1) == 0 ? 1 : -1;
             if (b < 0)
-                throw Contracts.Except("Cannot raise an integer to a negative power");
+                return 0;
+
+            bool neg = false;
+            if (a < 0)
+            {
+                a = -a;
+                neg = (b & 1) != 0;
+            }
 
             // Since the abs of the base is at least two, the exponent must be less than 63.
             if (b >= 63)
-                throw Contracts.Except("Cannot raise an integer to a power greater than 62");
+                return neg ? I8.MinValue : I8.MaxValue;
 
             if (a == 0)
             {
@@ -1050,18 +1060,12 @@ public static I8 Pow(I8 a, I8 b)
                 return 0;
             }
 
-            bool neg = false;
-            if (a < 0)
-            {
-                a = -a;
-                neg = (b & 1) != 0;
-            }
             Contracts.Assert(a >= 2);
 
             // Since the exponent is at least three, the base must be < 2^21.
             Contracts.Assert(b >= 3);
             if (a >= (1L << 21))
-                throw Contracts.Except($"Base must be less than 2^21 when raising to the power of {b}");
+                return neg ? I8.MinValue : I8.MaxValue;
 
             long res = 1;
             long x = a;
@@ -1083,7 +1087,7 @@ public static I8 Pow(I8 a, I8 b)
             }
             catch (OverflowException)
             {
-                throw Contracts.Except("Overflow");
+                return neg ? I8.MinValue : I8.MaxValue;
             }
             Contracts.Assert(res > 0);
 

src/Microsoft.ML.Transforms/Expression/TokKind.cs

@@ -45,45 +45,45 @@ internal enum TokKind
 
         Car, // ^
         CarEqu,
-        Amp, // &
-        AmpAmp,
+        Amp,
+        AmpAmp, // &&
         AmpEqu,
-        Bar, // |
-        BarBar,
+        Bar,
+        BarBar, // ||
         BarEqu,
 
-        Til, // ~
+        Til,
         Bng, // !
-        BngEqu,
+        BngEqu, // !=
 
         Equ, // =
-        EquEqu,
-        EquGrt,
-        Lss,
+        EquEqu, // ==
+        EquGrt, // =>
+        Lss, // <
         LssLss,
-        LssEqu,
-        LssGrt,
+        LssEqu, // <=
+        LssGrt, // <>
         LssLssEqu,
-        Grt,
+        Grt, // >
         GrtGrt,
-        GrtEqu,
+        GrtEqu, // >=
         GrtGrtEqu,
 
         Que, // ?
-        QueQue,
+        QueQue, // ??
 
-        Dot,
-        Comma,
-        Colon,
+        Dot, // .
+        Comma, // ,
+        Colon, // :
         ColonColon,
-        Semi,
+        Semi, // ;
 
         OpenCurly,
-        OpenParen,
+        OpenParen, // (
         OpenSquare,
 
         CloseCurly,
-        CloseParen,
+        CloseParen, // )
         CloseSquare,
 
         // Words - identifier and key words
@@ -95,37 +95,6 @@ internal enum TokKind
         And,
         Or,
 
-        // REVIEW: These are specific to the NetParser. Use a general mechanism.
-        Const,
-        Input,
-        Output,
-        Hidden,
-        Share,
-        Sigmoid,
-        Linear,
-        SoftMax,
-        RectifiedLinear,
-        Square,
-        Sqrt,
-        SoftRectifiedLinear,
-        Tanh,
-        BoundedRectifiedLinear,
-        From,
-        All,
-        Where,
-        Convolve,
-        Pool,
-        Abs,
-        Bittest,
-        Max,
-        Mean,
-        Response,
-        Norm,
-        FloatsFromBytes,
-        Param,
-        Auto,
-
-        // REVIEW: These are specific to the ExprTransform parser. Use a general mechanism.
         With,
     }
 

src/Microsoft.ML.Transforms/ExpressionCatalog.cs

@@ -17,6 +17,13 @@ public static class ExpressionCatalog
         /// This column's data type will be the same as that of the input column.</param>
         /// <param name="expression">The expression to apply to <paramref name="inputColumnNames"/> to create the column <paramref name="outputColumnName"/>.</param>
         /// <param name="inputColumnNames">The names of the input columns.</param>
+        /// <example>
+        /// <format type="text/markdown">
+        /// <![CDATA[
+        /// [!code-csharp[Expression](~/../docs/samples/docs/samples/Microsoft.ML.Samples/Dynamic/Transforms/Expression.cs)]
+        /// ]]>
+        /// </format>
+        /// </example>
         public static ExpressionEstimator Expression(this TransformsCatalog catalog, string outputColumnName, string expression, params string[] inputColumnNames)
             => new ExpressionEstimator(CatalogUtils.GetEnvironment(catalog), new ExpressionEstimator.ColumnOptions(outputColumnName, inputColumnNames, expression));
     }

src/Microsoft.ML.Transforms/ExpressionTransformer.cs

@@ -31,6 +31,7 @@ namespace Microsoft.ML.Transforms
 {
     /// <summary>
     /// This estimator applies a user provided expression (specified as a string) to input column values to produce new output column values.
+    /// </summary>
     /// <remarks>
     /// <format type="text/markdown"><![CDATA[
     ///
@@ -50,7 +51,6 @@ namespace Microsoft.ML.Transforms
     /// ]]></format>
     /// </remarks>
     /// <seealso cref="ExpressionCatalog.Expression(TransformsCatalog, string, string, string[])"/>
-    /// </summary>
     public sealed class ExpressionEstimator : IEstimator<ExpressionTransformer>
     {
         internal sealed class ColumnOptions