Change the behaviour of not-in and != when convert standard query to pipeline

Firestore/Swift/Tests/Integration/QueryIntegrationTests.swift

@@ -206,6 +206,9 @@ class QueryIntegrationTests: FSTIntegrationTestCase {
   }
 
   func testMultipleInOps() async throws {
+    try XCTSkipIf(!FSTIntegrationTestCase.isRunningAgainstEmulator(),
+                  "Skip this test if running against production.")
+
     let collRef = collectionRef(
       withDocuments: ["doc1": ["a": 1, "b": 0],
                       "doc2": ["b": 1],

Firestore/Swift/Tests/Integration/QueryToPipelineTests.swift

@@ -37,7 +37,14 @@ class QueryToPipelineTests: FSTIntegrationTestCase {
                              file: StaticString = #file,
                              line: UInt = #line) {
     let results = snapshot.results.map { $0.data as! [String: AnyHashable?] }
-    XCTAssertEqual(results.count, expected.count, "Result count mismatch.", file: file, line: line)
+    guard results.count == expected.count else {
+      XCTFail(
+        "Result count mismatch. Got \(results.count), expected \(expected.count)",
+        file: file,
+        line: line
+      )
+      return
+    }
 
     if enforceOrder {
       for i in 0 ..< expected.count {
@@ -712,7 +719,18 @@ class QueryToPipelineTests: FSTIntegrationTestCase {
     let pipeline = db.pipeline().create(from: query)
     let snapshot = try await pipeline.execute()
 
-    verifyResults(snapshot, [["foo": 3, "bar": 10]])
+    switch FSTIntegrationTestCase.backendEdition() {
+    case .standard:
+      // In Standard, `NOT_IN` requires the field to exist.
+      // So document "2" (with no "bar" field) is filtered out.
+      verifyResults(snapshot, [["foo": 3, "bar": 10]])
+    case .enterprise:
+      // In Enterprise, `NOT_IN` does not require the field to exist.
+      // So document "2" (with no "bar" field) is included.
+      verifyResults(snapshot, [["foo": 2], ["foo": 3, "bar": 10]])
+    @unknown default:
+      XCTFail("Unknown backend edition")
+    }
   }
 
   func testSupportsOrOperator() async throws {
@@ -739,4 +757,77 @@ class QueryToPipelineTests: FSTIntegrationTestCase {
       enforceOrder: true
     )
   }
+
+  private func verifyIDs(_ snapshot: Pipeline.Snapshot,
+                         _ expected: [String],
+                         enforceOrder: Bool = false,
+                         file: StaticString = #file,
+                         line: UInt = #line) {
+    let results = snapshot.results.map { $0.ref!.documentID }
+    if enforceOrder {
+      XCTAssertEqual(results, expected, "Result IDs do not match or are not in order.",
+                     file: file, line: line)
+    } else {
+      XCTAssertEqual(Set(results), Set(expected), "Result ID sets do not match.",
+                     file: file, line: line)
+    }
+  }
+
+  func testNotInRemovesExistenceFilter() async throws {
+    let collRef = collectionRef(withDocuments: [
+      "doc1": ["field": 2],
+      "doc2": ["field": 1],
+      "doc3": [:],
+    ])
+    let db = collRef.firestore
+
+    let query = collRef.whereField("field", notIn: [1])
+    let pipeline = db.pipeline().create(from: query)
+    let snapshot = try await pipeline.execute()
+
+    verifyIDs(snapshot, ["doc1", "doc3"])
+  }
+
+  func testNotEqualRemovesExistenceFilter() async throws {
+    let collRef = collectionRef(withDocuments: [
+      "doc1": ["field": 2],
+      "doc2": ["field": 1],
+      "doc3": [:],
+    ])
+    let db = collRef.firestore
+
+    let query = collRef.whereField("field", isNotEqualTo: 1)
+    let pipeline = db.pipeline().create(from: query)
+    let snapshot = try await pipeline.execute()
+
+    verifyIDs(snapshot, ["doc1", "doc3"])
+  }
+
+  func testInequalityMaintainsExistenceFilter() async throws {
+    let collRef = collectionRef(withDocuments: [
+      "doc1": ["field": 0],
+      "doc2": [:],
+    ])
+    let db = collRef.firestore
+
+    let query = collRef.whereField("field", isLessThan: 1)
+    let pipeline = db.pipeline().create(from: query)
+    let snapshot = try await pipeline.execute()
+
+    verifyIDs(snapshot, ["doc1"])
+  }
+
+  func testExplicitOrderMaintainsExistenceFilter() async throws {
+    let collRef = collectionRef(withDocuments: [
+      "doc1": ["field": 1],
+      "doc2": [:],
+    ])
+    let db = collRef.firestore
+
+    let query = collRef.order(by: "field")
+    let pipeline = db.pipeline().create(from: query)
+    let snapshot = try await pipeline.execute()
+
+    verifyIDs(snapshot, ["doc1"])
+  }
 }

Firestore/core/src/core/pipeline_util.cc

@@ -602,6 +602,10 @@ std::shared_ptr<api::Expr> ToPipelineBooleanExpr(const Filter& filter) {
     comparison_expr = std::make_shared<api::FunctionExpr>(
         func_name,
         std::vector<std::shared_ptr<api::Expr>>{api_field, api_constant});
+    if (op == FieldFilter::Operator::NotIn ||
+        op == FieldFilter::Operator::NotEqual) {
+      return comparison_expr;
+    }
     return std::make_shared<api::FunctionExpr>(
         "and",
         std::vector<std::shared_ptr<api::Expr>>{exists_expr, comparison_expr});
@@ -699,27 +703,33 @@ std::vector<std::shared_ptr<api::EvaluableStage>> ToPipelineStages(
   }
 
   // 3. OrderBy Existence Checks
-  const auto& query_order_bys = query.normalized_order_bys();
-  if (!query_order_bys.empty()) {
+  const auto& query_explicit_order_bys = query.explicit_order_bys();
+  if (!query_explicit_order_bys.empty()) {
     std::vector<std::shared_ptr<api::Expr>> exists_exprs;
-    exists_exprs.reserve(query_order_bys.size());
-    for (const auto& core_order_by : query_order_bys) {
+    exists_exprs.reserve(query_explicit_order_bys.size());
+    for (const auto& core_order_by : query.explicit_order_bys()) {
       exists_exprs.push_back(std::make_shared<api::FunctionExpr>(
           "exists", std::vector<std::shared_ptr<api::Expr>>{
                         std::make_shared<api::Field>(core_order_by.field())}));
     }
-    if (exists_exprs.size() == 1) {
-      stages.push_back(std::make_shared<api::Where>(exists_exprs[0]));
-    } else {
-      stages.push_back(std::make_shared<api::Where>(
-          std::make_shared<api::FunctionExpr>("and", exists_exprs)));
+
+    if (!exists_exprs.empty()) {
+      std::shared_ptr<api::Expr> final_exists_expr;
+      if (exists_exprs.size() == 1) {
+        final_exists_expr = exists_exprs[0];
+      } else {
+        final_exists_expr =
+            std::make_shared<api::FunctionExpr>("and", exists_exprs);
+      }
+      stages.push_back(std::make_shared<api::Where>(final_exists_expr));
     }
   }
 
   // 4. Orderings, Cursors, Limit
   std::vector<api::Ordering> api_orderings;
-  api_orderings.reserve(query_order_bys.size());
-  for (const auto& core_order_by : query_order_bys) {
+  const auto& query_normalized_order_bys = query.normalized_order_bys();
+  api_orderings.reserve(query_normalized_order_bys.size());
+  for (const auto& core_order_by : query_normalized_order_bys) {
     api_orderings.emplace_back(
         std::make_shared<api::Field>(core_order_by.field()),
         core_order_by.direction() == Direction::Ascending