Query: Adds Weighted RRF capability to LINQ

Microsoft.Azure.Cosmos/src/Linq/BuiltinFunctions/OtherBuiltinSystemFunctions.cs

@@ -24,17 +24,21 @@ public RRFVisit()
                     true,
                     new List<Type[]>()
                     {
-                        new Type[]{typeof(double[])}
+                        new Type[]{typeof(double[])},
+                        new Type[]{typeof(double[]), typeof(double[])}
                     })
             {
             }
 
             protected override SqlScalarExpression VisitImplicit(MethodCallExpression methodCallExpression, TranslationContext context)
             {
-                if (methodCallExpression.Arguments.Count == 1
-                    && methodCallExpression.Arguments[0] is NewArrayExpression argumentsExpressions)
+                if (methodCallExpression.Arguments.Count != 1 && methodCallExpression.Arguments.Count != 2)
+                {
+                    throw new DocumentQueryException("Invalid Argument Count.");
+                }
+
+                if (methodCallExpression.Arguments[0] is NewArrayExpression argumentsExpressions)
                 {
-                    // For RRF, We don't need to care about the first argument, it is the object itself and have no relevance to the computation
                     ReadOnlyCollection<Expression> functionListExpression = argumentsExpressions.Expressions;
                     List<SqlScalarExpression> arguments = new List<SqlScalarExpression>();
                     foreach (Expression argument in functionListExpression)
@@ -65,10 +69,16 @@ protected override SqlScalarExpression VisitImplicit(MethodCallExpression method
                         arguments.Add(ExpressionToSql.VisitNonSubqueryScalarExpression(argument, context));
                     }
 
+                    // Append the weight if exists
+                    if (methodCallExpression.Arguments.Count == 2)
+                    {
+                        arguments.Add(ExpressionToSql.VisitNonSubqueryScalarExpression(methodCallExpression.Arguments[1], context));
+                    }
+
                     return SqlFunctionCallScalarExpression.CreateBuiltin(SqlFunctionCallScalarExpression.Names.RRF, arguments.ToImmutableArray());
                 }
 
-                return null;
+                throw new DocumentQueryException(string.Format(CultureInfo.CurrentCulture, "Method {0} is not supported with the given argument list.", methodCallExpression.Method.Name));
             }
 
             protected override SqlScalarExpression VisitExplicit(MethodCallExpression methodCallExpression, TranslationContext context)

Microsoft.Azure.Cosmos/src/Linq/CosmosLinqExtensions.cs

@@ -470,6 +470,28 @@ public static double RRF(params double[] scoringFunctions)
             throw new NotImplementedException(ClientResources.ExtensionMethodNotImplemented); 
         }
 
+        /// <summary>
+        /// This system function is used to combine two or more scores provided by other scoring functions.
+        /// For more information, see https://learn.microsoft.com/en-us/azure/cosmos-db/nosql/query/rrf.
+        /// This method is to be used in LINQ expressions only and will be evaluated on server.
+        /// There's no implementation provided in the client library.
+        /// </summary>
+        /// <param name="scoringFunctions">the scoring functions to combine. Valid functions are FullTextScore and VectorDistance. </param>
+        /// <param name="weights">the weights to use for scoring functions</param>
+        /// <returns>Returns the the combined scores of the scoring functions.</returns>
+        /// <example>
+        /// <code>
+        /// <![CDATA[
+        /// var matched = documents.OrderByRank(document => document.RRF(document.Name.FullTextScore(<keyword1>), document.Address.FullTextScore(<keyword2>)));
+        /// ]]>
+        /// </code>
+        /// </example>
+        public static double RRF(double[] scoringFunctions, double[] weights)
+        {
+            // The reason for not defining "this" keyword is because this causes undesirable serialization when call Expression.ToString() on this method
+            throw new NotImplementedException(ClientResources.ExtensionMethodNotImplemented);
+        }
+
         /// <summary>
         /// This method generate query definition from LINQ query.
         /// </summary>

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/BaselineTest/TestBaseline/LinqTranslationBaselineTests.TestWeightedRRF.xml

@@ -0,0 +1,77 @@
+<Results>
+  <Result>
+    <Input>
+      <Description><![CDATA[Standard weighted RRF calls]]></Description>
+      <Expression><![CDATA[query.OrderByRank(doc => RRF(new [] {doc.StringField.FullTextScore(new [] {"test1"}), doc.StringField.FullTextScore(new [] {"test1", "text2"})}, new [] {1, 2})).Select(doc => doc.Pk)]]></Expression>
+    </Input>
+    <Output>
+      <SqlQuery><![CDATA[
+SELECT VALUE root["Pk"] 
+FROM root 
+ORDER BY RANK RRF(FullTextScore(root["StringField"], "test1"), FullTextScore(root["StringField"], "test1", "text2"), [1, 2])]]></SqlQuery>
+      <Results><![CDATA[[
+  "Test",
+  "Test"
+]]]></Results>
+    </Output>
+  </Result>
+  <Result>
+    <Input>
+      <Description><![CDATA[Standard weighted RRF calls using anonymous types]]></Description>
+      <Expression><![CDATA[query.OrderByRank(doc => RRF(new [] {doc.StringField.FullTextScore(new [] {"test1"}), doc.StringField.FullTextScore(new [] {"test1", "text2"})}, new [] {1, 2})).Select(doc => doc.Pk)]]></Expression>
+    </Input>
+    <Output>
+      <SqlQuery><![CDATA[
+SELECT VALUE root["Pk"] 
+FROM root 
+ORDER BY RANK RRF(FullTextScore(root["StringField"], "test1"), FullTextScore(root["StringField"], "test1", "text2"), [1, 2])]]></SqlQuery>
+      <Results><![CDATA[[
+  "Test",
+  "Test"
+]]]></Results>
+    </Output>
+  </Result>
+  <Result>
+    <Input>
+      <Description><![CDATA[Weighted RRF with weights and functions not in a list]]></Description>
+      <Expression><![CDATA[query.OrderByRank(doc => RRF(new [] {doc.StringField.FullTextScore(new [] {"test1"}), doc.StringField2.FullTextScore(new [] {"test1", "test2", "test3"}), 1, 2})).Select(doc => doc.Pk)]]></Expression>
+    </Input>
+    <Output>
+      <SqlQuery><![CDATA[]]></SqlQuery>
+      <ErrorMessage><![CDATA[Expressions of type System.Double is not supported as an argument to CosmosLinqExtensions.RRF. Supported expressions are method calls to FullTextScore, VectorDistance.]]></ErrorMessage>
+    </Output>
+  </Result>
+  <Result>
+    <Input>
+      <Description><![CDATA[Weighted RRF with weights array first]]></Description>
+      <Expression><![CDATA[query.OrderByRank(doc => RRF(new [] {1, 2}, new [] {doc.StringField.FullTextScore(new [] {"test1"}), doc.StringField.FullTextScore(new [] {"test1", "text2"})})).Select(doc => doc.Pk)]]></Expression>
+    </Input>
+    <Output>
+      <SqlQuery><![CDATA[]]></SqlQuery>
+      <ErrorMessage><![CDATA[Method RRF is not supported with the given argument list.]]></ErrorMessage>
+    </Output>
+  </Result>
+  <Result>
+    <Input>
+      <Description><![CDATA[Weighted RRF with mixed and matched values/functions in array]]></Description>
+      <Expression><![CDATA[query.OrderByRank(doc => RRF(new [] {1, doc.StringField.FullTextScore(new [] {"test1"})}, new [] {2, doc.StringField.FullTextScore(new [] {"test1", "text2"})})).Select(doc => doc.Pk)]]></Expression>
+    </Input>
+    <Output>
+      <SqlQuery><![CDATA[]]></SqlQuery>
+      <ErrorMessage><![CDATA[Expressions of type System.Double is not supported as an argument to CosmosLinqExtensions.RRF. Supported expressions are method calls to FullTextScore, VectorDistance.]]></ErrorMessage>
+    </Output>
+  </Result>
+  <Result>
+    <Input>
+      <Description><![CDATA[Weighted RRF with mixed and matched values/functions in array 2]]></Description>
+      <Expression><![CDATA[query.OrderByRank(doc => RRF(new [] {doc.StringField.FullTextScore(new [] {"test1"}), doc.StringField.FullTextScore(new [] {"test1"})}, new [] {2, doc.StringField.FullTextScore(new [] {"test1", "text2"})})).Select(doc => doc.Pk)]]></Expression>
+    </Input>
+    <Output>
+      <SqlQuery><![CDATA[
+SELECT VALUE root["Pk"] 
+FROM root 
+ORDER BY RANK RRF(FullTextScore(root["StringField"], "test1"), FullTextScore(root["StringField"], "test1"), [2, FullTextScore(root["StringField"], "test1", "text2")])]]></SqlQuery>
+      <ErrorMessage><![CDATA[Status Code: BadRequest,{"errors":[{"severity":"Error","location":{"start":34,"end":200},"code":"SC2229","message":"The last parameter of the RRF function is an optional array of weights. When present, it must be a literal array of numbers, one for each of the component scores used for the RRF function. The length of this array must be the same as the number of the component scores."}]},0x800A0B00]]></ErrorMessage>
+    </Output>
+  </Result>
+</Results>

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Linq/LinqTranslationBaselineTests.cs

@@ -667,6 +667,95 @@ static DataObject createDataObj(Random random)
             this.ExecuteTestSuite(inputs);
         }
 
+        [TestMethod]
+        public void TestWeightedRRF()
+        {
+            const int Records = 2;
+            const int MaxStringLength = 100;
+            static DataObject createDataObj(Random random)
+            {
+                DataObject obj = new DataObject
+                {
+                    StringField = LinqTestsCommon.RandomString(random, random.Next(MaxStringLength)),
+                    IntField = 1,
+                    Id = Guid.NewGuid().ToString(),
+                    Pk = "Test"
+                };
+                return obj;
+            }
+            Func<bool, IQueryable<DataObject>> getQuery = LinqTestsCommon.GenerateTestCosmosData(createDataObj, Records, testContainer);
+
+            List<LinqTestInput> inputs = new List<LinqTestInput>
+            {
+                // public static double RRF(double[][] scoringFunctions, double[] weights)
+                new LinqTestInput("Standard weighted RRF calls", b => getQuery(b)
+                    .OrderByRank(doc => RRF(new double[]
+                                            {
+                                                doc.StringField.FullTextScore(new string[] { "test1" }),
+                                                doc.StringField.FullTextScore(new string[] { "test1", "text2" })
+                                            },
+                                            new double[] { 1.0, 2.0 } ))
+                    .Select(doc => doc.Pk)),
+
+                new LinqTestInput("Standard weighted RRF calls using anonymous types", b => getQuery(b)
+                    .OrderByRank(doc => RRF(new []
+                                            {
+                                                doc.StringField.FullTextScore(new string[] { "test1" }),
+                                                doc.StringField.FullTextScore(new string[] { "test1", "text2" })
+                                            },
+                                            new [] { 1.0, 2.0 } ))
+                    .Select(doc => doc.Pk)),
+
+                // Negative case: weights are not in an array
+                new LinqTestInput("Weighted RRF with weights and functions not in a list", b => getQuery(b)
+                    .OrderByRank(doc => RRF(doc.StringField.FullTextScore(new string[] { "test1" }),
+                                            doc.StringField2.FullTextScore(new string[] { "test1", "test2", "test3" }),
+                                            1.0,
+                                            2.0))
+                    .Select(doc => doc.Pk)),
+                new LinqTestInput("Weighted RRF with weights array first", b => getQuery(b)
+                    .OrderByRank(doc => RRF(new double[] { 1.0, 2.0 },
+                                            new double[]
+                                            {
+                                                doc.StringField.FullTextScore(new string[] { "test1" }),
+                                                doc.StringField.FullTextScore(new string[] { "test1", "text2" })
+                                            }))
+                    .Select(doc => doc.Pk)),
+                new LinqTestInput("Weighted RRF with mixed and matched values/functions in array", b => getQuery(b)
+                    .OrderByRank(doc => RRF(new double[] { 
+                                                1.0, 
+                                                doc.StringField.FullTextScore(new string[] { "test1" }) },
+                                            new double[]
+                                            {
+                                                2.0,
+                                                doc.StringField.FullTextScore(new string[] { "test1", "text2" })
+                                            }))
+                    .Select(doc => doc.Pk)),
+                new LinqTestInput("Weighted RRF with mixed and matched values/functions in array 2", b => getQuery(b)
+                    .OrderByRank(doc => RRF(new double[] { 
+                                                doc.StringField.FullTextScore(new string[] { "test1" }),
+                                                doc.StringField.FullTextScore(new string[] { "test1" }) },
+                                            new double[]
+                                            {
+                                                2.0,
+                                                doc.StringField.FullTextScore(new string[] { "test1", "text2" })
+                                            }))
+                    .Select(doc => doc.Pk)),
+
+
+            };
+
+            foreach (LinqTestInput input in inputs)
+            {
+                // OrderBy are not supported client side.
+                // Therefore this method is verified with baseline only.
+                input.skipVerification = true;
+                input.serializeOutput = true;
+            }
+
+            this.ExecuteTestSuite(inputs);
+        }
+
         [TestMethod]
         public void TestOrderByRankFunctionComposeWithOtherFunctions()
         {

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Microsoft.Azure.Cosmos.EmulatorTests.csproj

@@ -250,6 +250,9 @@
     <Content Include="BaselineTest\TestBaseline\LinqTranslationBaselineTests.TestRRFOrderByRankFunction.xml">
         <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
+    <Content Include="BaselineTest\TestBaseline\LinqTranslationBaselineTests.TestWeightedRRF.xml">
+        <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
     <Content Include="BaselineTest\TestBaseline\LinqTranslationBaselineTests.TestOrderByRankFunctionComposeWithOtherFunctions.xml">
         <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.Tests/Contracts/DotNetSDKAPI.json

@@ -6366,6 +6366,11 @@
           ],
           "MethodInfo": "Double FullTextScore[TSource](TSource, System.String[]);IsAbstract:False;IsStatic:True;IsVirtual:False;IsGenericMethod:True;IsConstructor:False;IsFinal:False;"
         },
+        "Double RRF(Double[], Double[])": {
+          "Type": "Method",
+          "Attributes": [],
+          "MethodInfo": "Double RRF(Double[], Double[]);IsAbstract:False;IsStatic:True;IsVirtual:False;IsGenericMethod:False;IsConstructor:False;IsFinal:False;"
+        },
         "Double RRF(Double[])": {
           "Type": "Method",
           "Attributes": [],