[SPARK-32870][DOCS][SQL] Make sure that all expressions have their ExpressionDescription filled

sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/ExpressionInfo.java

@@ -105,8 +105,9 @@ public ExpressionInfo(
             String deprecated) {
         assert name != null;
         assert arguments != null;
+        assert arguments.isEmpty() || arguments.startsWith("\n    Arguments:\n");
         assert examples != null;
-        assert examples.isEmpty() || examples.contains("    Examples:");
+        assert examples.isEmpty() || examples.startsWith("\n    Examples:\n");
         assert note != null;
         assert group != null;
         assert since != null;
@@ -142,9 +143,9 @@ public ExpressionInfo(
                 "got [" + group + "].");
         }
         if (!since.isEmpty()) {
-            if (Integer.parseInt(since.split("\\.")[0]) < 0) {
+            if (!since.matches("[0-9]+\\.[0-9]+\\.[0-9]+")) {
                 throw new IllegalArgumentException("'since' is malformed in the expression [" +
-                    this.name + "]. It should not start with a negative number; however, " +
+                    this.name + "]. It should follow the MAJOR.MINOR.PATCH pattern; however, " +
                     "got [" + since + "].");
             }
             this.extended += "\n    Since: " + since + "\n";

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala

@@ -551,18 +551,18 @@ object FunctionRegistry {
     // cast
     expression[Cast]("cast"),
     // Cast aliases (SPARK-16730)
-    castAlias("boolean", BooleanType),
-    castAlias("tinyint", ByteType),
-    castAlias("smallint", ShortType),
-    castAlias("int", IntegerType),
-    castAlias("bigint", LongType),
-    castAlias("float", FloatType),
-    castAlias("double", DoubleType),
-    castAlias("decimal", DecimalType.USER_DEFAULT),
-    castAlias("date", DateType),
-    castAlias("timestamp", TimestampType),
-    castAlias("binary", BinaryType),
-    castAlias("string", StringType),
+    castAlias("boolean", BooleanType, "1", "true"),
+    castAlias("tinyint", ByteType, "1.0", "1"),
+    castAlias("smallint", ShortType, "1.0", "1"),
+    castAlias("int", IntegerType, "1.0", "1"),
+    castAlias("bigint", LongType, "1.0", "1"),
+    castAlias("float", FloatType, "1", "1.0"),
+    castAlias("double", DoubleType, "1", "1.0"),
+    castAlias("decimal", DecimalType.USER_DEFAULT, "1.0", "1"),
+    castAlias("date", DateType, "'1970-01-01T12'", "1970-01-01"),
+    castAlias("timestamp", TimestampType, "'1970-01-01 12:00'", "1970-01-01 12:00:00"),
+    castAlias("binary", BinaryType, "'spark'", "spark"),
+    castAlias("string", StringType, "1.0", "1.0"),
 
     // csv
     expression[CsvToStructs]("from_csv"),
@@ -651,7 +651,9 @@ object FunctionRegistry {
    */
   private def castAlias(
       name: String,
-      dataType: DataType): (String, (ExpressionInfo, FunctionBuilder)) = {
+      dataType: DataType,
+      exampleIn: String,
+      exampleOut: String): (String, (ExpressionInfo, FunctionBuilder)) = {
     val builder = (args: Seq[Expression]) => {
       if (args.size != 1) {
         throw new AnalysisException(s"Function $name accepts only one argument")
@@ -660,8 +662,13 @@ object FunctionRegistry {
     }
     val clazz = scala.reflect.classTag[Cast].runtimeClass
     val usage = "_FUNC_(expr) - Casts the value `expr` to the target data type `_FUNC_`."
-    val expressionInfo =
-      new ExpressionInfo(clazz.getCanonicalName, null, name, usage, "", "", "", "", "", "")
+    val examples = s"""
+    Examples:
+      > SELECT _FUNC_($exampleIn);
+       $exampleOut
+  """
+    val expressionInfo = new ExpressionInfo(clazz.getCanonicalName, null, name, usage, "",
+      examples, "", "", "2.0.1", "")
     (name, (expressionInfo, builder))
   }
 

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/CallMethodViaReflection.scala

@@ -51,7 +51,8 @@ import org.apache.spark.util.Utils
        c33fb387-8500-4bfa-81d2-6e0e3e930df2
       > SELECT _FUNC_('java.util.UUID', 'fromString', 'a5cf6c42-0c85-418f-af6c-3e4e5b1328f2');
        a5cf6c42-0c85-418f-af6c-3e4e5b1328f2
-  """)
+  """,
+  since = "2.0.0")
 case class CallMethodViaReflection(children: Seq[Expression])
   extends Expression with CodegenFallback {
 

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala

@@ -1747,7 +1747,8 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
     Examples:
       > SELECT _FUNC_('10' as int);
        10
-  """)
+  """,
+  since = "1.0.0")
 case class Cast(child: Expression, dataType: DataType, timeZoneId: Option[String] = None)
   extends CastBase {
 

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/MonotonicallyIncreasingID.scala

@@ -40,7 +40,13 @@ import org.apache.spark.sql.types.{DataType, LongType}
       within each partition. The assumption is that the data frame has less than 1 billion
       partitions, and each partition has less than 8 billion records.
       The function is non-deterministic because its result depends on partition IDs.
-  """)
+  """,
+  examples = """
+    Examples:
+      > SELECT _FUNC_();
+       0
+  """,
+  since = "1.4.0")
 case class MonotonicallyIncreasingID() extends LeafExpression with Stateful {
 
   /**

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/SparkPartitionID.scala

@@ -26,7 +26,13 @@ import org.apache.spark.sql.types.{DataType, IntegerType}
  * Expression that returns the current partition id.
  */
 @ExpressionDescription(
-  usage = "_FUNC_() - Returns the current partition id.")
+  usage = "_FUNC_() - Returns the current partition id.",
+  examples = """
+    Examples:
+      > SELECT _FUNC_();
+       0
+  """,
+  since = "1.4.0")
 case class SparkPartitionID() extends LeafExpression with Nondeterministic {
 
   override def nullable: Boolean = false

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/CountMinSketchAgg.scala

@@ -37,15 +37,22 @@ import org.apache.spark.util.sketch.CountMinSketch
  * @param confidenceExpression confidence, must be positive and less than 1.0
  * @param seedExpression random seed
  */
+// scalastyle:off line.size.limit
 @ExpressionDescription(
   usage = """
     _FUNC_(col, eps, confidence, seed) - Returns a count-min sketch of a column with the given esp,
       confidence and seed. The result is an array of bytes, which can be deserialized to a
       `CountMinSketch` before usage. Count-min sketch is a probabilistic data structure used for
       cardinality estimation using sub-linear space.
   """,
+  examples = """
+    Examples:
+      > SELECT hex(_FUNC_(col, 0.5d, 0.5d, 1)) FROM VALUES (1), (2), (1) AS tab(col);
+       0000000100000000000000030000000100000004000000005D8D6AB90000000000000000000000000000000200000000000000010000000000000000
+  """,
   group = "agg_funcs",
   since = "2.2.0")
+// scalastyle:on line.size.limit
 case class CountMinSketchAgg(
     child: Expression,
     epsExpression: Expression,

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/bitwiseAggregates.scala

@@ -60,6 +60,7 @@ abstract class BitAggregate extends DeclarativeAggregate with ExpectsInputTypes
       > SELECT _FUNC_(col) FROM VALUES (3), (5) AS tab(col);
        1
   """,
+  group = "agg_funcs",
   since = "3.0.0")
 case class BitAndAgg(child: Expression) extends BitAggregate {
 

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala

@@ -32,7 +32,8 @@ import org.apache.spark.unsafe.types.CalendarInterval
     Examples:
       > SELECT _FUNC_(1);
        -1
-  """)
+  """,
+  since = "1.0.0")
 case class UnaryMinus(child: Expression) extends UnaryExpression
     with ExpectsInputTypes with NullIntolerant {
   private val checkOverflow = SQLConf.get.ansiEnabled
@@ -95,7 +96,13 @@ case class UnaryMinus(child: Expression) extends UnaryExpression
 }
 
 @ExpressionDescription(
-  usage = "_FUNC_(expr) - Returns the value of `expr`.")
+  usage = "_FUNC_(expr) - Returns the value of `expr`.",
+  examples = """
+    Examples:
+      > SELECT _FUNC_(1);
+       1
+  """,
+  since = "1.5.0")
 case class UnaryPositive(child: Expression)
     extends UnaryExpression with ExpectsInputTypes with NullIntolerant {
   override def prettyName: String = "positive"
@@ -121,7 +128,8 @@ case class UnaryPositive(child: Expression)
     Examples:
       > SELECT _FUNC_(-1);
        1
-  """)
+  """,
+  since = "1.2.0")
 case class Abs(child: Expression)
     extends UnaryExpression with ExpectsInputTypes with NullIntolerant {
 
@@ -223,7 +231,8 @@ object BinaryArithmetic {
     Examples:
       > SELECT 1 _FUNC_ 2;
        3
-  """)
+  """,
+  since = "1.0.0")
 case class Add(left: Expression, right: Expression) extends BinaryArithmetic {
 
   override def inputType: AbstractDataType = TypeCollection.NumericAndInterval
@@ -255,7 +264,8 @@ case class Add(left: Expression, right: Expression) extends BinaryArithmetic {
     Examples:
       > SELECT 2 _FUNC_ 1;
        1
-  """)
+  """,
+  since = "1.0.0")
 case class Subtract(left: Expression, right: Expression) extends BinaryArithmetic {
 
   override def inputType: AbstractDataType = TypeCollection.NumericAndInterval
@@ -287,7 +297,8 @@ case class Subtract(left: Expression, right: Expression) extends BinaryArithmeti
     Examples:
       > SELECT 2 _FUNC_ 3;
        6
-  """)
+  """,
+  since = "1.0.0")
 case class Multiply(left: Expression, right: Expression) extends BinaryArithmetic {
 
   override def inputType: AbstractDataType = NumericType
@@ -382,7 +393,8 @@ trait DivModLike extends BinaryArithmetic {
        1.5
       > SELECT 2L _FUNC_ 2L;
        1.0
-  """)
+  """,
+  since = "1.0.0")
 // scalastyle:on line.size.limit
 case class Divide(left: Expression, right: Expression) extends DivModLike {
 
@@ -455,7 +467,8 @@ object IntegralDivide {
        0.2
       > SELECT MOD(2, 1.8);
        0.2
-  """)
+  """,
+  since = "1.0.0")
 case class Remainder(left: Expression, right: Expression) extends DivModLike {
 
   override def inputType: AbstractDataType = NumericType
@@ -502,7 +515,8 @@ case class Remainder(left: Expression, right: Expression) extends DivModLike {
        1
       > SELECT _FUNC_(-10, 3);
        2
-  """)
+  """,
+  since = "1.5.0")
 case class Pmod(left: Expression, right: Expression) extends BinaryArithmetic {
 
   override def toString: String = s"pmod($left, $right)"
@@ -658,7 +672,8 @@ case class Pmod(left: Expression, right: Expression) extends BinaryArithmetic {
     Examples:
       > SELECT _FUNC_(10, 9, 2, 4, 3);
        2
-  """)
+  """,
+  since = "1.5.0")
 case class Least(children: Seq[Expression]) extends ComplexTypeMergingExpression {
 
   override def nullable: Boolean = children.forall(_.nullable)
@@ -731,7 +746,8 @@ case class Least(children: Seq[Expression]) extends ComplexTypeMergingExpression
     Examples:
       > SELECT _FUNC_(10, 9, 2, 4, 3);
        10
-  """)
+  """,
+  since = "1.5.0")
 case class Greatest(children: Seq[Expression]) extends ComplexTypeMergingExpression {
 
   override def nullable: Boolean = children.forall(_.nullable)

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/bitwiseExpressions.scala

@@ -32,7 +32,8 @@ import org.apache.spark.sql.types._
     Examples:
       > SELECT 3 _FUNC_ 5;
        1
-  """)
+  """,
+  since = "1.4.0")
 case class BitwiseAnd(left: Expression, right: Expression) extends BinaryArithmetic {
 
   override def inputType: AbstractDataType = IntegralType
@@ -64,7 +65,8 @@ case class BitwiseAnd(left: Expression, right: Expression) extends BinaryArithme
     Examples:
       > SELECT 3 _FUNC_ 5;
        7
-  """)
+  """,
+  since = "1.4.0")
 case class BitwiseOr(left: Expression, right: Expression) extends BinaryArithmetic {
 
   override def inputType: AbstractDataType = IntegralType
@@ -96,7 +98,8 @@ case class BitwiseOr(left: Expression, right: Expression) extends BinaryArithmet
     Examples:
       > SELECT 3 _FUNC_ 5;
        6
-  """)
+  """,
+  since = "1.4.0")
 case class BitwiseXor(left: Expression, right: Expression) extends BinaryArithmetic {
 
   override def inputType: AbstractDataType = IntegralType
@@ -126,7 +129,8 @@ case class BitwiseXor(left: Expression, right: Expression) extends BinaryArithme
     Examples:
       > SELECT _FUNC_ 0;
        -1
-  """)
+  """,
+  since = "1.4.0")
 case class BitwiseNot(child: Expression)
   extends UnaryExpression with ExpectsInputTypes with NullIntolerant {
 

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala

@@ -89,7 +89,8 @@ trait BinaryArrayExpressionWithImplicitCast extends BinaryExpression
        2
       > SELECT _FUNC_(NULL);
        -1
-  """)
+  """,
+  since = "1.5.0")
 case class Size(child: Expression, legacySizeOfNull: Boolean)
   extends UnaryExpression with ExpectsInputTypes {
 
@@ -139,7 +140,8 @@ object Size {
       > SELECT _FUNC_(map(1, 'a', 2, 'b'));
        [1,2]
   """,
-  group = "map_funcs")
+  group = "map_funcs",
+  since = "2.0.0")
 case class MapKeys(child: Expression)
   extends UnaryExpression with ExpectsInputTypes with NullIntolerant {
 
@@ -330,7 +332,8 @@ case class ArraysZip(children: Seq[Expression]) extends Expression with ExpectsI
       > SELECT _FUNC_(map(1, 'a', 2, 'b'));
        ["a","b"]
   """,
-  group = "map_funcs")
+  group = "map_funcs",
+  since = "2.0.0")
 case class MapValues(child: Expression)
   extends UnaryExpression with ExpectsInputTypes with NullIntolerant {
 
@@ -871,7 +874,8 @@ object ArraySortLike {
       > SELECT _FUNC_(array('b', 'd', null, 'c', 'a'), true);
        [null,"a","b","c","d"]
   """,
-  group = "array_funcs")
+  group = "array_funcs",
+  since = "1.5.0")
 // scalastyle:on line.size.limit
 case class SortArray(base: Expression, ascendingOrder: Expression)
   extends BinaryExpression with ArraySortLike with NullIntolerant {
@@ -1086,7 +1090,8 @@ case class Reverse(child: Expression)
       > SELECT _FUNC_(array(1, 2, 3), 2);
        true
   """,
-  group = "array_funcs")
+  group = "array_funcs",
+  since = "1.5.0")
 case class ArrayContains(left: Expression, right: Expression)
   extends BinaryExpression with ImplicitCastInputTypes with NullIntolerant {
 
@@ -2048,7 +2053,8 @@ case class ElementAt(left: Expression, right: Expression)
   note = """
     Concat logic for arrays is available since 2.4.0.
   """,
-  group = "array_funcs")
+  group = "array_funcs",
+  since = "1.5.0")
 case class Concat(children: Seq[Expression]) extends ComplexTypeMergingExpression {
 
   private def allowedTypes: Seq[AbstractDataType] = Seq(StringType, BinaryType, ArrayType)