From 54c4f6eaeae9bfc2468cb38a4d5587135a66653e Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 23 Oct 2025 14:14:20 -0700 Subject: [PATCH 01/36] fixed tests --- tests/unit/v1/test_pipeline_expressions.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/unit/v1/test_pipeline_expressions.py b/tests/unit/v1/test_pipeline_expressions.py index 9f06c47b8..0e96d06e6 100644 --- a/tests/unit/v1/test_pipeline_expressions.py +++ b/tests/unit/v1/test_pipeline_expressions.py @@ -370,7 +370,7 @@ def test__from_query_filter_pb_composite_filter_or(self, mock_client): field1 = Field.of("field1") field2 = Field.of("field2") expected_cond1 = expr.And(field1.exists(), field1.equal(Constant("val1"))) - expected_cond2 = expr.And(field2.exists(), field2.equal(Constant(None))) + expected_cond2 = expr.And(field2.exists(), field2.is_null()) expected = expr.Or(expected_cond1, expected_cond2) assert repr(result) == repr(expected) @@ -458,7 +458,7 @@ def test__from_query_filter_pb_composite_filter_nested(self, mock_client): expected_cond1 = expr.And(field1.exists(), field1.equal(Constant("val1"))) expected_cond2 = expr.And(field2.exists(), field2.greater_than(Constant(10))) expected_cond3 = expr.And( - field3.exists(), expr.Not(field3.equal(Constant(None))) + field3.exists(), expr.Not(field3.is_null()) ) expected_inner_and = expr.And(expected_cond2, expected_cond3) expected_outer_or = expr.Or(expected_cond1, expected_inner_and) @@ -495,11 +495,11 @@ def test__from_query_filter_pb_composite_filter_unknown_op(self, mock_client): ), ( query_pb.StructuredQuery.UnaryFilter.Operator.IS_NULL, - lambda f: f.equal(None), + lambda f: f.is_null(), ), ( query_pb.StructuredQuery.UnaryFilter.Operator.IS_NOT_NULL, - lambda f: expr.Not(f.equal(None)), + lambda f: expr.Not(f.is_null()), ), ], ) @@ -836,7 +836,7 @@ def test_is_nan(self): def test_is_null(self): arg1 = self._make_arg("Value") - instance = Expr.is_ull(arg1) + instance = Expr.is_null(arg1) assert instance.name == "is_null" assert instance.params == [arg1] assert repr(instance) == "Value.is_null()" From 107e412cdeddeb3b0b53507fac7e6d8c89d917d6 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 23 Oct 2025 16:23:50 -0700 Subject: [PATCH 02/36] added vector expressions --- google/cloud/firestore_v1/_pipeline_stages.py | 6 +- .../firestore_v1/pipeline_expressions.py | 62 ++++++++ tests/system/pipeline_e2e.yaml | 148 ++++++++++++++++++ tests/system/test_pipeline_acceptance.py | 3 +- tests/unit/v1/test_pipeline_expressions.py | 47 ++++++ 5 files changed, 263 insertions(+), 3 deletions(-) diff --git a/google/cloud/firestore_v1/_pipeline_stages.py b/google/cloud/firestore_v1/_pipeline_stages.py index 7233a8eec..2ada1c90f 100644 --- a/google/cloud/firestore_v1/_pipeline_stages.py +++ b/google/cloud/firestore_v1/_pipeline_stages.py @@ -274,13 +274,15 @@ def __init__( self, field: str | Expr, vector: Sequence[float] | Vector, - distance_measure: "DistanceMeasure", + distance_measure: "DistanceMeasure" | str, options: Optional["FindNearestOptions"] = None, ): super().__init__("find_nearest") self.field: Expr = Field(field) if isinstance(field, str) else field self.vector: Vector = vector if isinstance(vector, Vector) else Vector(vector) - self.distance_measure = distance_measure + self.distance_measure = distance_measure if isinstance( + distance_measure, DistanceMeasure + ) else DistanceMeasure[distance_measure.upper()] self.options = options or FindNearestOptions() def _pb_args(self): diff --git a/google/cloud/firestore_v1/pipeline_expressions.py b/google/cloud/firestore_v1/pipeline_expressions.py index 4639e0f7d..9b510a7e4 100644 --- a/google/cloud/firestore_v1/pipeline_expressions.py +++ b/google/cloud/firestore_v1/pipeline_expressions.py @@ -116,6 +116,14 @@ def _to_pb(self) -> Value: def _cast_to_expr_or_convert_to_constant(o: Any) -> "Expr": return o if isinstance(o, Expr) else Constant(o) + @staticmethod + def _convert_to_vector_expr(o: list[float] | Vector | Expr) -> Expr: + if isinstance(o, list): + o = Vector(o) + elif isinstance(o, Constant) and isinstance(o.value, list): + o = Vector(o.value) + return Expr._cast_to_expr_or_convert_to_constant(o) + class expose_as_static: """ Decorator to mark instance methods to be exposed as static methods as well as instance @@ -864,6 +872,60 @@ def map_get(self, key: str | Constant[str]) -> "Expr": "map_get", [self, Constant.of(key) if isinstance(key, str) else key] ) + @expose_as_static + def cosine_distance(self, other: Expr | list[float] | Vector) -> "Expr": + """Calculates the cosine distance between two vectors. + + Example: + >>> # Calculate the cosine distance between the 'userVector' field and the 'itemVector' field + >>> Field.of("userVector").cosine_distance(Field.of("itemVector")) + >>> # Calculate the Cosine distance between the 'location' field and a target location + >>> Field.of("location").cosine_distance([37.7749, -122.4194]) + + Args: + other: The other vector (represented as an Expr, list of floats, or Vector) to compare against. + + Returns: + A new `Expr` representing the cosine distance between the two vectors. + """ + return Function("cosine_distance", [self, self._convert_to_vector_expr(other)]) + + @expose_as_static + def euclidean_distance(self, other: Expr | list[float] | Vector) -> "Expr": + """Calculates the Euclidean distance between two vectors. + + Example: + >>> # Calculate the Euclidean distance between the 'location' field and a target location + >>> Field.of("location").euclidean_distance([37.7749, -122.4194]) + >>> # Calculate the Euclidean distance between two vector fields: 'pointA' and 'pointB' + >>> Field.of("pointA").euclidean_distance(Field.of("pointB")) + + Args: + other: The other vector (represented as an Expr, list of floats, or Vector) to compare against. + + Returns: + A new `Expr` representing the Euclidean distance between the two vectors. + """ + return Function("euclidean_distance", [self, self._convert_to_vector_expr(other)]) + + @expose_as_static + def dot_product(self, other: Expr | list[float] | Vector) -> "Expr": + """Calculates the dot product between two vectors. + + Example: + >>> # Calculate the dot product between a feature vector and a target vector + >>> Field.of("features").dot_product([0.5, 0.8, 0.2]) + >>> # Calculate the dot product between two document vectors: 'docVector1' and 'docVector2' + >>> Field.of("docVector1").dot_product(Field.of("docVector2")) + + Args: + other: The other vector (represented as an Expr, list of floats, or Vector) to calculate dot product with. + + Returns: + A new `Expr` representing the dot product between the two vectors. + """ + return Function("dot_product", [self, self._convert_to_vector_expr(other)]) + @expose_as_static def vector_length(self) -> "Expr": """Creates an expression that calculates the length (dimension) of a Firestore Vector. diff --git a/tests/system/pipeline_e2e.yaml b/tests/system/pipeline_e2e.yaml index 50cc7c29d..50fc8e6ab 100644 --- a/tests/system/pipeline_e2e.yaml +++ b/tests/system/pipeline_e2e.yaml @@ -136,6 +136,10 @@ data: embedding: [1.0, 2.0, 3.0] vec2: embedding: [4.0, 5.0, 6.0, 7.0] + vec3: + embedding: [5.0, 6.0, 7.0] + vec4: + embedding: [1.0, 2.0, 4.0] tests: - description: "testAggregates - count" pipeline: @@ -1782,6 +1786,8 @@ tests: - Field: embedding_length - ASCENDING assert_results: + - embedding_length: 3 + - embedding_length: 3 - embedding_length: 3 - embedding_length: 4 - description: testTimestampFunctions @@ -1956,3 +1962,145 @@ tests: conditional_field: "Dystopian" - title: "Dune" conditional_field: "Frank Herbert" + - description: testFindNearestEuclidean + pipeline: + - Collection: vectors + - FindNearest: + field: embedding + vector: [1.0, 2.0, 3.0] + distance_measure: EUCLIDEAN + options: + FindNearestOptions: + limit: 2 + distance_field: + Field: distance + - Select: + - distance + assert_results: + - distance: 0.0 + - distance: 1.0 + assert_proto: + pipeline: + stages: + - name: collection + args: + - referenceValue: /vectors + - name: find_nearest + args: + - fieldReferenceValue: embedding + - mapValue: + fields: + __type__: + stringValue: __vector__ + value: + arrayValue: + values: + - doubleValue: 1.0 + - doubleValue: 2.0 + - doubleValue: 3.0 + - stringValue: euclidean + options: + limit: + integerValue: '2' + distance_field: + fieldReferenceValue: distance + - name: select + args: + - mapValue: + fields: + distance: + fieldReferenceValue: distance + - description: testFindNearestDotProduct + pipeline: + - Collection: vectors + - FindNearest: + field: embedding + vector: [1.0, 2.0, 3.0] + distance_measure: DOT_PRODUCT + options: + FindNearestOptions: + limit: 3 + distance_field: + Field: distance + - Select: + - distance + assert_results: + - distance: 38.0 + - distance: 17.0 + - distance: 14.0 + assert_proto: + pipeline: + stages: + - name: collection + args: + - referenceValue: /vectors + - name: find_nearest + args: + - fieldReferenceValue: embedding + - mapValue: + fields: + __type__: + stringValue: __vector__ + value: + arrayValue: + values: + - doubleValue: 1.0 + - doubleValue: 2.0 + - doubleValue: 3.0 + - stringValue: dot_product + options: + limit: + integerValue: '3' + distance_field: + fieldReferenceValue: distance + - name: select + args: + - mapValue: + fields: + distance: + fieldReferenceValue: distance + - description: testDotProductWithConstant + pipeline: + - Collection: vectors + - Where: + - Expr.equal: + - Field: embedding + - Vector: [1.0, 2.0, 3.0] + - Select: + - AliasedExpr: + - Expr.dot_product: + - Field: embedding + - Vector: [1.0, 1.0, 1.0] + - "dot_product_result" + assert_results: + - dot_product_result: 6.0 + - description: testEuclideanDistanceWithConstant + pipeline: + - Collection: vectors + - Where: + - Expr.equal: + - Field: embedding + - Vector: [1.0, 2.0, 3.0] + - Select: + - AliasedExpr: + - Expr.euclidean_distance: + - Field: embedding + - Vector: [1.0, 2.0, 3.0] + - "euclidean_distance_result" + assert_results: + - euclidean_distance_result: 0.0 + - description: testCosineDistanceWithConstant + pipeline: + - Collection: vectors + - Where: + - Expr.equal: + - Field: embedding + - Vector: [1.0, 2.0, 3.0] + - Select: + - AliasedExpr: + - Expr.cosine_distance: + - Field: embedding + - Vector: [1.0, 2.0, 3.0] + - "cosine_distance_result" + assert_results: + - cosine_distance_result: 0.0 \ No newline at end of file diff --git a/tests/system/test_pipeline_acceptance.py b/tests/system/test_pipeline_acceptance.py index d4c654e63..fe1d7659b 100644 --- a/tests/system/test_pipeline_acceptance.py +++ b/tests/system/test_pipeline_acceptance.py @@ -28,6 +28,7 @@ from google.cloud.firestore_v1 import _pipeline_stages as stages from google.cloud.firestore_v1 import pipeline_expressions from google.cloud.firestore_v1.vector import Vector +from google.cloud.firestore_v1 import pipeline_expressions as expr from google.api_core.exceptions import GoogleAPIError from google.cloud.firestore import Client, AsyncClient @@ -218,7 +219,7 @@ def _apply_yaml_args_to_callable(callable_obj, client, yaml_args): """ if isinstance(yaml_args, dict): return callable_obj(**_parse_expressions(client, yaml_args)) - elif isinstance(yaml_args, list): + elif isinstance(yaml_args, list) and not (callable_obj == expr.Constant or callable_obj == Vector): # yaml has an array of arguments. Treat as args return callable_obj(*_parse_expressions(client, yaml_args)) else: diff --git a/tests/unit/v1/test_pipeline_expressions.py b/tests/unit/v1/test_pipeline_expressions.py index 0e96d06e6..4fa3dd8e8 100644 --- a/tests/unit/v1/test_pipeline_expressions.py +++ b/tests/unit/v1/test_pipeline_expressions.py @@ -1097,6 +1097,53 @@ def test_unix_seconds_to_timestamp(self): infix_instance = arg1.unix_seconds_to_timestamp() assert infix_instance == instance + def test_euclidean_distance(self): + arg1 = self._make_arg("Vector1") + arg2 = self._make_arg("Vector2") + instance = Expr.euclidean_distance(arg1, arg2) + assert instance.name == "euclidean_distance" + assert instance.params == [arg1, arg2] + assert repr(instance) == "Vector1.euclidean_distance(Vector2)" + infix_instance = arg1.euclidean_distance(arg2) + assert infix_instance == instance + + def test_cosine_distance(self): + arg1 = self._make_arg("Vector1") + arg2 = self._make_arg("Vector2") + instance = Expr.cosine_distance(arg1, arg2) + assert instance.name == "cosine_distance" + assert instance.params == [arg1, arg2] + assert repr(instance) == "Vector1.cosine_distance(Vector2)" + infix_instance = arg1.cosine_distance(arg2) + assert infix_instance == instance + + def test_dot_product(self): + arg1 = self._make_arg("Vector1") + arg2 = self._make_arg("Vector2") + instance = Expr.dot_product(arg1, arg2) + assert instance.name == "dot_product" + assert instance.params == [arg1, arg2] + assert repr(instance) == "Vector1.dot_product(Vector2)" + infix_instance = arg1.dot_product(arg2) + assert infix_instance == instance + + @pytest.mark.parametrize("method", ["euclidean_distance", "cosine_distance", "dot_product"]) + @pytest.mark.parametrize( + "input", [Vector([1.0, 2.0]), [1, 2], Constant.of(Vector([1.0, 2.0])), Constant.of([1, 2]), []] + ) + def test_vector_ctor(self, method, input): + """ + test constructing various vector expressions with + different inputs + """ + arg1 = self._make_arg("Vector") + instance = getattr(arg1, method)(input) + assert instance.name == method + got_second_param = instance.params[1] + assert isinstance(got_second_param, Constant) + assert isinstance(got_second_param.value, Vector) + + def test_vector_length(self): arg1 = self._make_arg("Array") instance = Expr.vector_length(arg1) From a0c36ca4f5b744e4f8cf9cea9dbf9a54d097e7b4 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 24 Oct 2025 10:44:58 -0700 Subject: [PATCH 03/36] added new math expressions --- .../firestore_v1/pipeline_expressions.py | 129 +++++++++++++ tests/system/pipeline_e2e.yaml | 172 ++++++++++++++++++ tests/system/test_pipeline_acceptance.py | 6 + tests/unit/v1/test_pipeline_expressions.py | 83 +++++++++ 4 files changed, 390 insertions(+) diff --git a/google/cloud/firestore_v1/pipeline_expressions.py b/google/cloud/firestore_v1/pipeline_expressions.py index 9b510a7e4..ecb51f2d0 100644 --- a/google/cloud/firestore_v1/pipeline_expressions.py +++ b/google/cloud/firestore_v1/pipeline_expressions.py @@ -247,6 +247,135 @@ def mod(self, other: Expr | float) -> "Expr": """ return Function("mod", [self, self._cast_to_expr_or_convert_to_constant(other)]) + @expose_as_static + def abs(self) -> "Expr": + """Creates an expression that calculates the absolute value of this expression. + + Example: + >>> # Get the absolute value of the 'change' field. + >>> Field.of("change").abs() + + Returns: + A new `Expr` representing the absolute value. + """ + return Function("abs", [self]) + + @expose_as_static + def ceil(self) -> "Expr": + """Creates an expression that calculates the ceiling of this expression. + + Example: + >>> # Get the ceiling of the 'value' field. + >>> Field.of("value").ceil() + + Returns: + A new `Expr` representing the ceiling value. + """ + return Function("ceil", [self]) + + @expose_as_static + def exp(self) -> "Expr": + """Creates an expression that computes e to the power of this expression. + + Example: + >>> # Compute e to the power of the 'value' field + >>> Field.of("value").exp() + + Returns: + A new `Expr` representing the exponential value. + """ + return Function("exp", [self]) + + @expose_as_static + def floor(self) -> "Expr": + """Creates an expression that calculates the floor of this expression. + + Example: + >>> # Get the floor of the 'value' field. + >>> Field.of("value").floor() + + Returns: + A new `Expr` representing the floor value. + """ + return Function("floor", [self]) + + @expose_as_static + def ln(self) -> "Expr": + """Creates an expression that calculates the natural logarithm of this expression. + + Example: + >>> # Get the natural logarithm of the 'value' field. + >>> Field.of("value").ln() + + Returns: + A new `Expr` representing the natural logarithm. + """ + return Function("ln", [self]) + + @expose_as_static + def log(self, base: Expr | float) -> "Expr": + """Creates an expression that calculates the logarithm of this expression with a given base. + + Example: + >>> # Get the logarithm of 'value' with base 2. + >>> Field.of("value").log(2) + >>> # Get the logarithm of 'value' with base from 'base_field'. + >>> Field.of("value").log(Field.of("base_field")) + + Args: + base: The base of the logarithm. + + Returns: + A new `Expr` representing the logarithm. + """ + return Function( + "log", [self, self._cast_to_expr_or_convert_to_constant(base)] + ) + + @expose_as_static + def pow(self, exponent: Expr | float) -> "Expr": + """Creates an expression that calculates this expression raised to the power of the exponent. + + Example: + >>> # Raise 'base_val' to the power of 2. + >>> Field.of("base_val").pow(2) + >>> # Raise 'base_val' to the power of 'exponent_val'. + >>> Field.of("base_val").pow(Field.of("exponent_val")) + + Args: + exponent: The exponent. + + Returns: + A new `Expr` representing the power operation. + """ + return Function("pow", [self, self._cast_to_expr_or_convert_to_constant(exponent)]) + + @expose_as_static + def round(self) -> "Expr": + """Creates an expression that rounds this expression to the nearest integer. + + Example: + >>> # Round the 'value' field. + >>> Field.of("value").round() + + Returns: + A new `Expr` representing the rounded value. + """ + return Function("round", [self]) + + @expose_as_static + def sqrt(self) -> "Expr": + """Creates an expression that calculates the square root of this expression. + + Example: + >>> # Get the square root of the 'area' field. + >>> Field.of("area").sqrt() + + Returns: + A new `Expr` representing the square root. + """ + return Function("sqrt", [self]) + @expose_as_static def logical_maximum(self, other: Expr | CONSTANT_TYPE) -> "Expr": """Creates an expression that returns the larger value between this expression diff --git a/tests/system/pipeline_e2e.yaml b/tests/system/pipeline_e2e.yaml index 50fc8e6ab..f50efbc23 100644 --- a/tests/system/pipeline_e2e.yaml +++ b/tests/system/pipeline_e2e.yaml @@ -2010,6 +2010,178 @@ tests: fields: distance: fieldReferenceValue: distance + - description: testMathExpressions + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: To Kill a Mockingbird + - Select: + - AliasedExpr: + - Expr.abs: + - Field: rating + - "abs_rating" + - AliasedExpr: + - Expr.ceil: + - Field: rating + - "ceil_rating" + - AliasedExpr: + - Expr.exp: + - Field: rating + - "exp_rating" + - AliasedExpr: + - Expr.floor: + - Field: rating + - "floor_rating" + - AliasedExpr: + - Expr.ln: + - Field: rating + - "ln_rating" + - AliasedExpr: + - Expr.log: + - Field: rating + - Constant: 10 + - "log_rating_base10" + - AliasedExpr: + - Expr.pow: + - Field: rating + - Constant: 2 + - "pow_rating" + - AliasedExpr: + - Expr.sqrt: + - Field: rating + - "sqrt_rating" + assert_results_approximate: + - abs_rating: 4.2 + ceil_rating: 5.0 + exp_rating: 66.686331 + floor_rating: 4.0 + ln_rating: 1.4350845 + log_rating_base10: 0.623249 + pow_rating: 17.64 + sqrt_rating: 2.049390 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: To Kill a Mockingbird + name: equal + name: where + - args: + - mapValue: + fields: + abs_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: abs + ceil_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: ceil + exp_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: exp + floor_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: floor + ln_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: ln + log_rating_base10: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '10' + name: log + pow_rating: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '2' + name: pow + sqrt_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: sqrt + name: select + - description: testRoundExpressions + pipeline: + - Collection: books + - Where: + - Expr.equal_any: + - Field: title + - - Constant: "To Kill a Mockingbird" # rating 4.2 + - Constant: "Pride and Prejudice" # rating 4.5 + - Constant: "The Lord of the Rings" # rating 4.7 + - Select: + - title + - AliasedExpr: + - Expr.round: + - Field: rating + - "round_rating" + - Sort: + - Ordering: + - Field: title + - ASCENDING + assert_results: + - title: "Pride and Prejudice" + round_rating: 5.0 + - title: "The Lord of the Rings" + round_rating: 5.0 + - title: "To Kill a Mockingbird" + round_rating: 4.0 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - arrayValue: + values: + - stringValue: "To Kill a Mockingbird" + - stringValue: "Pride and Prejudice" + - stringValue: "The Lord of the Rings" + name: equal_any + name: where + - args: + - mapValue: + fields: + title: + fieldReferenceValue: title + round_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: round + name: select + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: title + name: sort - description: testFindNearestDotProduct pipeline: - Collection: vectors diff --git a/tests/system/test_pipeline_acceptance.py b/tests/system/test_pipeline_acceptance.py index fe1d7659b..ab63afd37 100644 --- a/tests/system/test_pipeline_acceptance.py +++ b/tests/system/test_pipeline_acceptance.py @@ -95,12 +95,15 @@ def test_pipeline_results(test_dict, client): Ensure pipeline returns expected results """ expected_results = _parse_yaml_types(test_dict.get("assert_results", None)) + expected_approximate_results = _parse_yaml_types(test_dict.get("assert_results_approximate", None)) expected_count = test_dict.get("assert_count", None) pipeline = parse_pipeline(client, test_dict["pipeline"]) # check if server responds as expected got_results = [snapshot.data() for snapshot in pipeline.stream()] if expected_results: assert got_results == expected_results + if expected_approximate_results: + assert got_results == pytest.approximate(expected_approximate_results) if expected_count is not None: assert len(got_results) == expected_count @@ -136,12 +139,15 @@ async def test_pipeline_results_async(test_dict, async_client): Ensure pipeline returns expected results """ expected_results = _parse_yaml_types(test_dict.get("assert_results", None)) + expected_approximate_results = _parse_yaml_types(test_dict.get("assert_results_approximate", None)) expected_count = test_dict.get("assert_count", None) pipeline = parse_pipeline(async_client, test_dict["pipeline"]) # check if server responds as expected got_results = [snapshot.data() async for snapshot in pipeline.stream()] if expected_results: assert got_results == expected_results + if expected_approximate_results: + assert got_results == pytest.approximate(expected_approximate_results) if expected_count is not None: assert len(got_results) == expected_count diff --git a/tests/unit/v1/test_pipeline_expressions.py b/tests/unit/v1/test_pipeline_expressions.py index 4fa3dd8e8..d49dcea13 100644 --- a/tests/unit/v1/test_pipeline_expressions.py +++ b/tests/unit/v1/test_pipeline_expressions.py @@ -1163,6 +1163,89 @@ def test_add(self): infix_instance = arg1.add(arg2) assert infix_instance == instance + def test_abs(self): + arg1 = self._make_arg("Value") + instance = Expr.abs(arg1) + assert instance.name == "abs" + assert instance.params == [arg1] + assert repr(instance) == "Value.abs()" + infix_instance = arg1.abs() + assert infix_instance == instance + + def test_ceil(self): + arg1 = self._make_arg("Value") + instance = Expr.ceil(arg1) + assert instance.name == "ceil" + assert instance.params == [arg1] + assert repr(instance) == "Value.ceil()" + infix_instance = arg1.ceil() + assert infix_instance == instance + + def test_exp(self): + arg1 = self._make_arg("Value") + instance = Expr.exp(arg1) + assert instance.name == "exp" + assert instance.params == [arg1] + assert repr(instance) == "Value.exp()" + infix_instance = arg1.exp() + assert infix_instance == instance + + def test_floor(self): + arg1 = self._make_arg("Value") + instance = Expr.floor(arg1) + assert instance.name == "floor" + assert instance.params == [arg1] + assert repr(instance) == "Value.floor()" + infix_instance = arg1.floor() + assert infix_instance == instance + + def test_ln(self): + arg1 = self._make_arg("Value") + instance = Expr.ln(arg1) + assert instance.name == "ln" + assert instance.params == [arg1] + assert repr(instance) == "Value.ln()" + infix_instance = arg1.ln() + assert infix_instance == instance + + def test_log(self): + arg1 = self._make_arg("Value") + arg2 = self._make_arg("Base") + instance = Expr.log(arg1, arg2) + assert instance.name == "log" + assert instance.params == [arg1, arg2] + assert repr(instance) == "Value.log(Base)" + infix_instance = arg1.log(arg2) + assert infix_instance == instance + + def test_pow(self): + arg1 = self._make_arg("Value") + arg2 = self._make_arg("Exponent") + instance = Expr.pow(arg1, arg2) + assert instance.name == "pow" + assert instance.params == [arg1, arg2] + assert repr(instance) == "Value.pow(Exponent)" + infix_instance = arg1.pow(arg2) + assert infix_instance == instance + + def test_round(self): + arg1 = self._make_arg("Value") + instance = Expr.round(arg1) + assert instance.name == "round" + assert instance.params == [arg1] + assert repr(instance) == "Value.round()" + infix_instance = arg1.round() + assert infix_instance == instance + + def test_sqrt(self): + arg1 = self._make_arg("Value") + instance = Expr.sqrt(arg1) + assert instance.name == "sqrt" + assert instance.params == [arg1] + assert repr(instance) == "Value.sqrt()" + infix_instance = arg1.sqrt() + assert infix_instance == instance + def test_array_length(self): arg1 = self._make_arg("Array") instance = Expr.array_length(arg1) From dcd6af1c69b54884de23e901c874d5a820692e5b Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 24 Oct 2025 13:24:40 -0700 Subject: [PATCH 04/36] added string manipulation expressions --- .../firestore_v1/pipeline_expressions.py | 93 ++++++ tests/system/pipeline_e2e.yaml | 270 +++++++++++++++++- tests/unit/v1/test_pipeline_expressions.py | 67 +++++ 3 files changed, 429 insertions(+), 1 deletion(-) diff --git a/google/cloud/firestore_v1/pipeline_expressions.py b/google/cloud/firestore_v1/pipeline_expressions.py index ecb51f2d0..d83d27e88 100644 --- a/google/cloud/firestore_v1/pipeline_expressions.py +++ b/google/cloud/firestore_v1/pipeline_expressions.py @@ -983,6 +983,99 @@ def string_concat(self, *elements: Expr | CONSTANT_TYPE) -> "Expr": [self] + [self._cast_to_expr_or_convert_to_constant(el) for el in elements], ) + @expose_as_static + def to_lower(self) -> "Expr": + """Creates an expression that converts a string to lowercase. + + Example: + >>> # Convert the 'name' field to lowercase + >>> Field.of("name").to_lower() + + Returns: + A new `Expr` representing the lowercase string. + """ + return Function("to_lower", [self]) + + @expose_as_static + def to_upper(self) -> "Expr": + """Creates an expression that converts a string to uppercase. + + Example: + >>> # Convert the 'title' field to uppercase + >>> Field.of("title").to_upper() + + Returns: + A new `Expr` representing the uppercase string. + """ + return Function("to_upper", [self]) + + @expose_as_static + def trim(self) -> "Expr": + """Creates an expression that removes leading and trailing whitespace from a string. + + Example: + >>> # Trim whitespace from the 'userInput' field + >>> Field.of("userInput").trim() + + Returns: + A new `Expr` representing the trimmed string. + """ + return Function("trim", [self]) + + @expose_as_static + def string_reverse(self) -> "Expr": + """Creates an expression that reverses a string. + + Example: + >>> # Reverse the 'userInput' field + >>> Field.of("userInput").reverse() + + Returns: + A new `Expr` representing the reversed string. + """ + return Function("string_reverse", [self]) + + @expose_as_static + def substring(self, position: Expr | int, length: Expr | int | None=None) -> "Expr": + """Creates an expression that returns a substring of the results of this expression. + + + Example: + >>> Field.of("description").substring(5, 10) + >>> Field.of("description").substring(5) + + Args: + position: the index of the first character of the substring. + length: the length of the substring. If not provided the substring + will end at the end of the input. + + Returns: + A new `Expr` representing the extracted substring. + """ + args = [self, self._cast_to_expr_or_convert_to_constant(position)] + if length is not None: + args.append(self._cast_to_expr_or_convert_to_constant(length)) + return Function("substring", args) + + @expose_as_static + def join(self, delimeter: Expr | str) -> "Expr": + """Creates an expression that joins the elements of an array into a string + + + Example: + >>> Field.of("tags").join(", ") + + Args: + delimiter: The delimiter to add between the elements of the array. + + Returns: + A new `Expr` representing the joined string. + """ + return Function( + "join", [self, self._cast_to_expr_or_convert_to_constant(delimeter)] + ) + + @expose_as_static def map_get(self, key: str | Constant[str]) -> "Expr": """Accesses a value from the map produced by evaluating this expression. diff --git a/tests/system/pipeline_e2e.yaml b/tests/system/pipeline_e2e.yaml index f50efbc23..10a72476c 100644 --- a/tests/system/pipeline_e2e.yaml +++ b/tests/system/pipeline_e2e.yaml @@ -2275,4 +2275,272 @@ tests: - Vector: [1.0, 2.0, 3.0] - "cosine_distance_result" assert_results: - - cosine_distance_result: 0.0 \ No newline at end of file + - cosine_distance_result: 0.0 + - description: testStringFunctions - ToLower + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: author + - Constant: "Douglas Adams" + - Select: + - AliasedExpr: + - Expr.to_lower: + - Field: title + - "lower_title" + assert_results: + - lower_title: "the hitchhiker's guide to the galaxy" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: author + - stringValue: Douglas Adams + name: equal + name: where + - args: + - mapValue: + fields: + lower_title: + functionValue: + args: + - fieldReferenceValue: title + name: to_lower + name: select + - description: testStringFunctions - ToUpper + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: author + - Constant: "Douglas Adams" + - Select: + - AliasedExpr: + - Expr.to_upper: + - Field: title + - "upper_title" + assert_results: + - upper_title: "THE HITCHHIKER'S GUIDE TO THE GALAXY" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: author + - stringValue: Douglas Adams + name: equal + name: where + - args: + - mapValue: + fields: + upper_title: + functionValue: + args: + - fieldReferenceValue: title + name: to_upper + name: select + - description: testStringFunctions - Trim + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: author + - Constant: "Douglas Adams" + - Select: + - AliasedExpr: + - Expr.trim: + - Expr.string_concat: + - Constant: " " + - Field: title + - Constant: " " + - "trimmed_title" + assert_results: + - trimmed_title: "The Hitchhiker's Guide to the Galaxy" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: author + - stringValue: Douglas Adams + name: equal + name: where + - args: + - mapValue: + fields: + trimmed_title: + functionValue: + args: + - functionValue: + args: + - stringValue: " " + - fieldReferenceValue: title + - stringValue: " " + name: string_concat + name: trim + name: select + - description: testStringFunctions - StringReverse + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: author + - Constant: "Jane Austen" + - Select: + - AliasedExpr: + - Expr.string_reverse: + - Field: title + - "reversed_title" + assert_results: + - reversed_title: "ecidujerP dna edirP" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: author + - stringValue: "Jane Austen" + name: equal + name: where + - args: + - mapValue: + fields: + reversed_title: + functionValue: + args: + - fieldReferenceValue: title + name: string_reverse + name: select + - description: testStringFunctions - Substring + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: author + - Constant: "Douglas Adams" + - Select: + - AliasedExpr: + - Expr.substring: + - Field: title + - Constant: 4 + - Constant: 11 + - "substring_title" + assert_results: + - substring_title: "Hitchhiker'" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: author + - stringValue: "Douglas Adams" + name: equal + name: where + - args: + - mapValue: + fields: + substring_title: + functionValue: + args: + - fieldReferenceValue: title + - integerValue: '4' + - integerValue: '11' + name: substring + name: select + - description: testStringFunctions - Substring without length + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: author + - Constant: "Fyodor Dostoevsky" + - Select: + - AliasedExpr: + - Expr.substring: + - Field: title + - Constant: 10 + - "substring_title" + assert_results: + - substring_title: "Punishment" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: author + - stringValue: "Fyodor Dostoevsky" + name: equal + name: where + - args: + - mapValue: + fields: + substring_title: + functionValue: + args: + - fieldReferenceValue: title + - integerValue: '10' + name: substring + name: select + - description: testStringFunctions - Join + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: author + - Constant: "Douglas Adams" + - Select: + - AliasedExpr: + - Expr.join: + - Field: tags + - Constant: ", " + - "joined_tags" + assert_results: + - joined_tags: "comedy, space, adventure" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: author + - stringValue: "Douglas Adams" + name: equal + name: where + - args: + - mapValue: + fields: + joined_tags: + functionValue: + args: + - fieldReferenceValue: tags + - stringValue: ", " + name: join + name: select \ No newline at end of file diff --git a/tests/unit/v1/test_pipeline_expressions.py b/tests/unit/v1/test_pipeline_expressions.py index d49dcea13..15cb86eb0 100644 --- a/tests/unit/v1/test_pipeline_expressions.py +++ b/tests/unit/v1/test_pipeline_expressions.py @@ -970,6 +970,73 @@ def test_logical_minimum(self): infix_instance = arg1.logical_minimum(arg2) assert infix_instance == instance + def test_to_lower(self): + arg1 = self._make_arg("Input") + instance = Expr.to_lower(arg1) + assert instance.name == "to_lower" + assert instance.params == [arg1] + assert repr(instance) == "Input.to_lower()" + infix_instance = arg1.to_lower() + assert infix_instance == instance + + def test_to_upper(self): + arg1 = self._make_arg("Input") + instance = Expr.to_upper(arg1) + assert instance.name == "to_upper" + assert instance.params == [arg1] + assert repr(instance) == "Input.to_upper()" + infix_instance = arg1.to_upper() + assert infix_instance == instance + + def test_trim(self): + arg1 = self._make_arg("Input") + instance = Expr.trim(arg1) + assert instance.name == "trim" + assert instance.params == [arg1] + assert repr(instance) == "Input.trim()" + infix_instance = arg1.trim() + assert infix_instance == instance + + def test_string_reverse(self): + arg1 = self._make_arg("Input") + instance = Expr.string_reverse(arg1) + assert instance.name == "string_reverse" + assert instance.params == [arg1] + assert repr(instance) == "Input.string_reverse()" + infix_instance = arg1.string_reverse() + assert infix_instance == instance + + def test_substring(self): + arg1 = self._make_arg("Input") + arg2 = self._make_arg("Position") + instance = Expr.substring(arg1, arg2) + assert instance.name == "substring" + assert instance.params == [arg1, arg2] + assert repr(instance) == "Input.substring(Position)" + infix_instance = arg1.substring(arg2) + assert infix_instance == instance + + def test_substring_w_length(self): + arg1 = self._make_arg("Input") + arg2 = self._make_arg("Position") + arg3 = self._make_arg("Length") + instance = Expr.substring(arg1, arg2, arg3) + assert instance.name == "substring" + assert instance.params == [arg1, arg2, arg3] + assert repr(instance) == "Input.substring(Position, Length)" + infix_instance = arg1.substring(arg2, arg3) + assert infix_instance == instance + + def test_join(self): + arg1 = self._make_arg("Array") + arg2 = self._make_arg("Separator") + instance = Expr.join(arg1, arg2) + assert instance.name == "join" + assert instance.params == [arg1, arg2] + assert repr(instance) == "Array.join(Separator)" + infix_instance = arg1.join(arg2) + assert infix_instance == instance + def test_map_get(self): arg1 = self._make_arg("Map") arg2 = "key" From 71308dc030483eb1a5061626682cb22dac11242f Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 24 Oct 2025 13:41:55 -0700 Subject: [PATCH 05/36] added not_nan, not_null, and is_absent --- .../firestore_v1/pipeline_expressions.py | 45 ++++++++++++++- tests/system/pipeline_e2e.yaml | 57 +++++++++++++++++++ tests/unit/v1/test_pipeline_expressions.py | 35 ++++++++++-- 3 files changed, 130 insertions(+), 7 deletions(-) diff --git a/google/cloud/firestore_v1/pipeline_expressions.py b/google/cloud/firestore_v1/pipeline_expressions.py index d83d27e88..bd0e17547 100644 --- a/google/cloud/firestore_v1/pipeline_expressions.py +++ b/google/cloud/firestore_v1/pipeline_expressions.py @@ -711,6 +711,20 @@ def array_reverse(self) -> "Expr": """ return Function("array_reverse", [self]) + @expose_as_static + def is_absent(self) -> "BooleanExpr": + """Creates an expression that returns true if a value is absent. Otherwise, returns false even if + the value is null. + + Example: + >>> # Check if the 'email' field is absent. + >>> Field.of("email").is_absent() + + Returns: + A new `BooleanExpression` representing the isAbsent operation. + """ + return BooleanExpr("is_absent", [self]) + @expose_as_static def is_nan(self) -> "BooleanExpr": """Creates an expression that checks if this expression evaluates to 'NaN' (Not a Number). @@ -724,9 +738,22 @@ def is_nan(self) -> "BooleanExpr": """ return BooleanExpr("is_nan", [self]) + @expose_as_static + def is_not_nan(self) -> "BooleanExpr": + """Creates an expression that checks if this expression evaluates to a non-'NaN' (Not a Number) value. + + Example: + >>> # Check if the result of a calculation is not NaN + >>> Field.of("value").divide(1).is_not_nan() + + Returns: + A new `Expr` representing the 'is not NaN' check. + """ + return BooleanExpr("is_not_nan", [self]) + @expose_as_static def is_null(self) -> "BooleanExpr": - """Creates an expression that checks if this expression evaluates to 'Null'. + """Creates an expression that checks if the value of a field is 'Null'. Example: >>> Field.of("value").is_null() @@ -736,6 +763,18 @@ def is_null(self) -> "BooleanExpr": """ return BooleanExpr("is_null", [self]) + @expose_as_static + def is_not_null(self) -> "BooleanExpr": + """Creates an expression that checks if the value of a field is not 'Null'. + + Example: + >>> Field.of("value").is_not_null() + + Returns: + A new `Expr` representing the 'isNotNull' check. + """ + return BooleanExpr("is_not_null", [self]) + @expose_as_static def exists(self) -> "BooleanExpr": """Creates an expression that checks if a field exists in the document. @@ -1607,11 +1646,11 @@ def _from_query_filter_pb(filter_pb, client): if filter_pb.op == Query_pb.UnaryFilter.Operator.IS_NAN: return And(field.exists(), field.is_nan()) elif filter_pb.op == Query_pb.UnaryFilter.Operator.IS_NOT_NAN: - return And(field.exists(), Not(field.is_nan())) + return And(field.exists(), field.is_not_nan()) elif filter_pb.op == Query_pb.UnaryFilter.Operator.IS_NULL: return And(field.exists(), field.is_null()) elif filter_pb.op == Query_pb.UnaryFilter.Operator.IS_NOT_NULL: - return And(field.exists(), Not(field.is_null())) + return And(field.exists(), field.is_not_null()) else: raise TypeError(f"Unexpected UnaryFilter operator type: {filter_pb.op}") elif isinstance(filter_pb, Query_pb.FieldFilter): diff --git a/tests/system/pipeline_e2e.yaml b/tests/system/pipeline_e2e.yaml index 10a72476c..83429fb70 100644 --- a/tests/system/pipeline_e2e.yaml +++ b/tests/system/pipeline_e2e.yaml @@ -1418,6 +1418,63 @@ tests: - args: - integerValue: '1' name: limit + - description: testIsNotNull + pipeline: + - Collection: books + - Where: + - Expr.is_not_null: + - Field: rating + assert_count: 10 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: rating + name: is_not_null + name: where + - description: testIsNotNaN + pipeline: + - Collection: books + - Where: + - Expr.is_not_nan: + - Field: rating + assert_count: 10 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: rating + name: is_not_nan + name: where + - description: testIsAbsent + pipeline: + - Collection: books + - Where: + - Expr.is_absent: + - Field: awards.pulitzer + assert_count: 9 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: awards.pulitzer + name: is_absent + name: where - description: testLogicalMinMax pipeline: - Collection: books diff --git a/tests/unit/v1/test_pipeline_expressions.py b/tests/unit/v1/test_pipeline_expressions.py index 15cb86eb0..567960808 100644 --- a/tests/unit/v1/test_pipeline_expressions.py +++ b/tests/unit/v1/test_pipeline_expressions.py @@ -458,7 +458,7 @@ def test__from_query_filter_pb_composite_filter_nested(self, mock_client): expected_cond1 = expr.And(field1.exists(), field1.equal(Constant("val1"))) expected_cond2 = expr.And(field2.exists(), field2.greater_than(Constant(10))) expected_cond3 = expr.And( - field3.exists(), expr.Not(field3.is_null()) + field3.exists(), field3.is_not_null() ) expected_inner_and = expr.And(expected_cond2, expected_cond3) expected_outer_or = expr.Or(expected_cond1, expected_inner_and) @@ -491,15 +491,15 @@ def test__from_query_filter_pb_composite_filter_unknown_op(self, mock_client): (query_pb.StructuredQuery.UnaryFilter.Operator.IS_NAN, Expr.is_nan), ( query_pb.StructuredQuery.UnaryFilter.Operator.IS_NOT_NAN, - lambda f: expr.Not(f.is_nan()), + Expr.is_not_nan, ), ( query_pb.StructuredQuery.UnaryFilter.Operator.IS_NULL, - lambda f: f.is_null(), + Expr.is_null, ), ( query_pb.StructuredQuery.UnaryFilter.Operator.IS_NOT_NULL, - lambda f: expr.Not(f.is_null()), + Expr.is_not_null, ), ], ) @@ -825,6 +825,15 @@ def test_not_equal_any(self): infix_instance = arg1.not_equal_any([arg2, arg3]) assert infix_instance == instance + def test_is_absent(self): + arg1 = self._make_arg("Field") + instance = Expr.is_absent(arg1) + assert instance.name == "is_absent" + assert instance.params == [arg1] + assert repr(instance) == "Field.is_absent()" + infix_instance = arg1.is_absent() + assert infix_instance == instance + def test_is_nan(self): arg1 = self._make_arg("Value") instance = Expr.is_nan(arg1) @@ -834,6 +843,15 @@ def test_is_nan(self): infix_instance = arg1.is_nan() assert infix_instance == instance + def test_is_not_nan(self): + arg1 = self._make_arg("Value") + instance = Expr.is_not_nan(arg1) + assert instance.name == "is_not_nan" + assert instance.params == [arg1] + assert repr(instance) == "Value.is_not_nan()" + infix_instance = arg1.is_not_nan() + assert infix_instance == instance + def test_is_null(self): arg1 = self._make_arg("Value") instance = Expr.is_null(arg1) @@ -843,6 +861,15 @@ def test_is_null(self): infix_instance = arg1.is_null() assert infix_instance == instance + def test_is_not_null(self): + arg1 = self._make_arg("Value") + instance = Expr.is_not_null(arg1) + assert instance.name == "is_not_null" + assert instance.params == [arg1] + assert repr(instance) == "Value.is_not_null()" + infix_instance = arg1.is_not_null() + assert infix_instance == instance + def test_not(self): arg1 = self._make_arg("Condition") instance = expr.Not(arg1) From 026001447e6a5d86a4089d77cb4cf4a37b2e482e Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 24 Oct 2025 15:53:41 -0700 Subject: [PATCH 06/36] added new Array type --- .../firestore_v1/pipeline_expressions.py | 112 +++++---- tests/system/pipeline_e2e.yaml | 227 +++++++++++++++++- tests/system/test_pipeline_acceptance.py | 2 +- tests/unit/v1/test_pipeline_expressions.py | 125 +++++----- 4 files changed, 346 insertions(+), 120 deletions(-) diff --git a/google/cloud/firestore_v1/pipeline_expressions.py b/google/cloud/firestore_v1/pipeline_expressions.py index bd0e17547..25544dfb5 100644 --- a/google/cloud/firestore_v1/pipeline_expressions.py +++ b/google/cloud/firestore_v1/pipeline_expressions.py @@ -113,16 +113,18 @@ def _to_pb(self) -> Value: raise NotImplementedError @staticmethod - def _cast_to_expr_or_convert_to_constant(o: Any) -> "Expr": - return o if isinstance(o, Expr) else Constant(o) - - @staticmethod - def _convert_to_vector_expr(o: list[float] | Vector | Expr) -> Expr: + def _cast_to_expr_or_convert_to_constant(o: Any, include_vector=False, include_array=False) -> "Expr": + """Convert arbitrary object to an Expr.""" + if isinstance(o, Constant) and isinstance(o.value, list): + o = o.value + if isinstance(o, Expr): + return o if isinstance(o, list): - o = Vector(o) - elif isinstance(o, Constant) and isinstance(o.value, list): - o = Vector(o.value) - return Expr._cast_to_expr_or_convert_to_constant(o) + if include_vector and all([isinstance(i, (float, int)) for i in o]): + return Constant(Vector(o)) + elif include_array: + return Array(o) + return Constant(o) class expose_as_static: """ @@ -140,6 +142,10 @@ def __init__(self, instance_func): self.instance_func = instance_func def static_func(self, first_arg, *other_args, **kwargs): + if not isinstance(first_arg, (Expr, str)): + raise TypeError( + f"`expressions must be called on an Expr or a string representing a field name. got {type(first_arg)}." + ) first_expr = ( Field.of(first_arg) if not isinstance(first_arg, Expr) else first_arg ) @@ -557,7 +563,7 @@ def less_than_or_equal(self, other: Expr | CONSTANT_TYPE) -> "BooleanExpr": ) @expose_as_static - def equal_any(self, array: Sequence[Expr | CONSTANT_TYPE]) -> "BooleanExpr": + def equal_any(self, array: Array | Sequence[Expr | CONSTANT_TYPE] | Expr) -> "BooleanExpr": """Creates an expression that checks if this expression is equal to any of the provided values or expressions. @@ -575,14 +581,12 @@ def equal_any(self, array: Sequence[Expr | CONSTANT_TYPE]) -> "BooleanExpr": "equal_any", [ self, - _ListOfExprs( - [self._cast_to_expr_or_convert_to_constant(v) for v in array] - ), + self._cast_to_expr_or_convert_to_constant(array, include_array=True), ], ) @expose_as_static - def not_equal_any(self, array: Sequence[Expr | CONSTANT_TYPE]) -> "BooleanExpr": + def not_equal_any(self, array: Array | list[Expr | CONSTANT_TYPE] | Expr) -> "BooleanExpr": """Creates an expression that checks if this expression is not equal to any of the provided values or expressions. @@ -600,9 +604,7 @@ def not_equal_any(self, array: Sequence[Expr | CONSTANT_TYPE]) -> "BooleanExpr": "not_equal_any", [ self, - _ListOfExprs( - [self._cast_to_expr_or_convert_to_constant(v) for v in array] - ), + self._cast_to_expr_or_convert_to_constant(array, include_array=True), ], ) @@ -629,7 +631,7 @@ def array_contains(self, element: Expr | CONSTANT_TYPE) -> "BooleanExpr": @expose_as_static def array_contains_all( self, - elements: Sequence[Expr | CONSTANT_TYPE], + elements: Array | list[Expr | CONSTANT_TYPE] | Expr, ) -> "BooleanExpr": """Creates an expression that checks if an array contains all the specified elements. @@ -649,16 +651,14 @@ def array_contains_all( "array_contains_all", [ self, - _ListOfExprs( - [self._cast_to_expr_or_convert_to_constant(e) for e in elements] - ), + self._cast_to_expr_or_convert_to_constant(elements, include_array=True), ], ) @expose_as_static def array_contains_any( self, - elements: Sequence[Expr | CONSTANT_TYPE], + elements: Array | list[Expr | CONSTANT_TYPE] | Expr, ) -> "BooleanExpr": """Creates an expression that checks if an array contains any of the specified elements. @@ -679,9 +679,7 @@ def array_contains_any( "array_contains_any", [ self, - _ListOfExprs( - [self._cast_to_expr_or_convert_to_constant(e) for e in elements] - ), + self._cast_to_expr_or_convert_to_constant(elements, include_array=True), ], ) @@ -711,6 +709,26 @@ def array_reverse(self) -> "Expr": """ return Function("array_reverse", [self]) + @expose_as_static + def array_concat(self, *other_arrays: Array | list[Expr | CONSTANT_TYPE] | Expr) -> "Expr": + """Creates an expression that concatenates an array expression with another array. + + Example: + >>> # Combine the 'tags' array with a new array and an array field + >>> Field.of("tags").array_concat(["newTag1", "newTag2", Field.of("otherTag")]) + + Args: + array: The list of constants or expressions to concat with. + + Returns: + A new `Expr` representing the concatenated array. + """ + return Function( + "array_concat", [self] + [ + self._cast_to_expr_or_convert_to_constant(arr, include_array=True) for arr in other_arrays + ], + ) + @expose_as_static def is_absent(self) -> "BooleanExpr": """Creates an expression that returns true if a value is absent. Otherwise, returns false even if @@ -1149,7 +1167,7 @@ def cosine_distance(self, other: Expr | list[float] | Vector) -> "Expr": Returns: A new `Expr` representing the cosine distance between the two vectors. """ - return Function("cosine_distance", [self, self._convert_to_vector_expr(other)]) + return Function("cosine_distance", [self, self._cast_to_expr_or_convert_to_constant(other, include_vector=True)]) @expose_as_static def euclidean_distance(self, other: Expr | list[float] | Vector) -> "Expr": @@ -1167,7 +1185,7 @@ def euclidean_distance(self, other: Expr | list[float] | Vector) -> "Expr": Returns: A new `Expr` representing the Euclidean distance between the two vectors. """ - return Function("euclidean_distance", [self, self._convert_to_vector_expr(other)]) + return Function("euclidean_distance", [self, self._cast_to_expr_or_convert_to_constant(other, include_vector=True)]) @expose_as_static def dot_product(self, other: Expr | list[float] | Vector) -> "Expr": @@ -1185,7 +1203,7 @@ def dot_product(self, other: Expr | list[float] | Vector) -> "Expr": Returns: A new `Expr` representing the dot product between the two vectors. """ - return Function("dot_product", [self, self._convert_to_vector_expr(other)]) + return Function("dot_product", [self, self._cast_to_expr_or_convert_to_constant(other, include_vector=True)]) @expose_as_static def vector_length(self) -> "Expr": @@ -1430,25 +1448,6 @@ def _to_pb(self) -> Value: return encode_value(self.value) -class _ListOfExprs(Expr): - """Represents a list of expressions, typically used as an argument to functions like 'in' or array functions.""" - - def __init__(self, exprs: Sequence[Expr]): - self.exprs: list[Expr] = list(exprs) - - def __eq__(self, other): - if not isinstance(other, _ListOfExprs): - return False - else: - return other.exprs == self.exprs - - def __repr__(self): - return repr(self.exprs) - - def _to_pb(self): - return Value(array_value={"values": [e._to_pb() for e in self.exprs]}) - - class Function(Expr): """A base class for expressions that represent function calls.""" @@ -1689,6 +1688,25 @@ def _from_query_filter_pb(filter_pb, client): else: raise TypeError(f"Unexpected filter type: {type(filter_pb)}") +class Array(Function): + """ + Creates an expression that creates a Firestore array value from an input list. + + Example: + >>> Expr.array(["bar", Field.of("baz")]) + + Args: + elements: THe input list to evaluate in the expression + """ + + def __init__(self, elements: list[Expr | CONSTANT_TYPE]): + if not isinstance(elements, list): + raise TypeError("Array must be constructed with a list") + converted_elements = [self._cast_to_expr_or_convert_to_constant(el) for el in elements] + super().__init__("array", converted_elements) + + def __repr__(self): + return f"Array({self.params})" class And(BooleanExpr): """ diff --git a/tests/system/pipeline_e2e.yaml b/tests/system/pipeline_e2e.yaml index 83429fb70..2efd95c50 100644 --- a/tests/system/pipeline_e2e.yaml +++ b/tests/system/pipeline_e2e.yaml @@ -701,10 +701,11 @@ tests: - functionValue: args: - fieldReferenceValue: tags - - arrayValue: - values: + - functionValue: + args: - stringValue: comedy - stringValue: classic + name: array name: array_contains_any name: where - args: @@ -743,10 +744,11 @@ tests: - functionValue: args: - fieldReferenceValue: tags - - arrayValue: - values: + - functionValue: + args: - stringValue: adventure - stringValue: magic + name: array name: array_contains_all name: where - args: @@ -1791,6 +1793,218 @@ tests: - adventure - space - comedy + - description: testArrayConcat + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: "The Hitchhiker's Guide to the Galaxy" + - Select: + - AliasedExpr: + - Expr.array_concat: + - Field: tags + - Constant: ["new_tag", "another_tag"] + - "concatenatedTags" + assert_results: + - concatenatedTags: + - comedy + - space + - adventure + - new_tag + - another_tag + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: "The Hitchhiker's Guide to the Galaxy" + name: equal + name: where + - args: + - mapValue: + fields: + concatenatedTags: + functionValue: + args: + - fieldReferenceValue: tags + - functionValue: + args: + - stringValue: "new_tag" + - stringValue: "another_tag" + name: array + name: array_concat + name: select + - description: testArrayConcatMultiple + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: "Dune" + - Select: + - AliasedExpr: + - Expr.array_concat: + - Field: tags + - Constant: ["sci-fi"] + - Constant: ["classic", "epic"] + - "concatenatedTags" + assert_results: + - concatenatedTags: + - politics + - desert + - ecology + - sci-fi + - classic + - epic + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: "Dune" + name: equal + name: where + - args: + - mapValue: + fields: + concatenatedTags: + functionValue: + args: + - fieldReferenceValue: tags + - functionValue: + args: + - stringValue: "sci-fi" + name: array + - functionValue: + args: + - stringValue: "classic" + - stringValue: "epic" + name: array + name: array_concat + name: select + - description: testArrayContainsAnyWithField + pipeline: + - Collection: books + - AddFields: + - AliasedExpr: + - Expr.array_concat: + - Field: tags + - Constant: ["Dystopian"] + - "new_tags" + - Where: + - Expr.array_contains_any: + - Field: new_tags + - - Constant: non_existent_tag + - Field: genre + - Select: + - title + - genre + - Sort: + - Ordering: + - Field: title + - ASCENDING + assert_results: + - title: "1984" + genre: "Dystopian" + - title: "The Handmaid's Tale" + genre: "Dystopian" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + new_tags: + functionValue: + args: + - fieldReferenceValue: tags + - functionValue: + args: + - stringValue: "Dystopian" + name: array + name: array_concat + name: add_fields + - args: + - functionValue: + args: + - fieldReferenceValue: new_tags + - functionValue: + args: + - stringValue: "non_existent_tag" + - fieldReferenceValue: genre + name: array + name: array_contains_any + name: where + - args: + - mapValue: + fields: + title: + fieldReferenceValue: title + genre: + fieldReferenceValue: genre + name: select + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: title + name: sort + - description: testArrayConcatLiterals + pipeline: + - Collection: books + - Limit: 1 + - Select: + - AliasedExpr: + - Expr.array_concat: + - Array: [1, 2, 3] + - Array: [4, 5] + - "concatenated" + assert_results: + - concatenated: [1, 2, 3, 4, 5] + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - integerValue: '1' + name: limit + - args: + - mapValue: + fields: + concatenated: + functionValue: + args: + - functionValue: + args: + - integerValue: '1' + - integerValue: '2' + - integerValue: '3' + name: array + - functionValue: + args: + - integerValue: '4' + - integerValue: '5' + name: array + name: array_concat + name: select - description: testExists pipeline: - Collection: books @@ -2213,11 +2427,12 @@ tests: - functionValue: args: - fieldReferenceValue: title - - arrayValue: - values: + - functionValue: + args: - stringValue: "To Kill a Mockingbird" - stringValue: "Pride and Prejudice" - stringValue: "The Lord of the Rings" + name: array name: equal_any name: where - args: diff --git a/tests/system/test_pipeline_acceptance.py b/tests/system/test_pipeline_acceptance.py index ab63afd37..3c41c0227 100644 --- a/tests/system/test_pipeline_acceptance.py +++ b/tests/system/test_pipeline_acceptance.py @@ -225,7 +225,7 @@ def _apply_yaml_args_to_callable(callable_obj, client, yaml_args): """ if isinstance(yaml_args, dict): return callable_obj(**_parse_expressions(client, yaml_args)) - elif isinstance(yaml_args, list) and not (callable_obj == expr.Constant or callable_obj == Vector): + elif isinstance(yaml_args, list) and not (callable_obj == expr.Constant or callable_obj == Vector or callable_obj == expr.Array): # yaml has an array of arguments. Treat as args return callable_obj(*_parse_expressions(client, yaml_args)) else: diff --git a/tests/unit/v1/test_pipeline_expressions.py b/tests/unit/v1/test_pipeline_expressions.py index 567960808..21dec6e4c 100644 --- a/tests/unit/v1/test_pipeline_expressions.py +++ b/tests/unit/v1/test_pipeline_expressions.py @@ -24,7 +24,6 @@ from google.cloud.firestore_v1._helpers import GeoPoint import google.cloud.firestore_v1.pipeline_expressions as expr from google.cloud.firestore_v1.pipeline_expressions import BooleanExpr -from google.cloud.firestore_v1.pipeline_expressions import _ListOfExprs from google.cloud.firestore_v1.pipeline_expressions import Expr from google.cloud.firestore_v1.pipeline_expressions import Constant from google.cloud.firestore_v1.pipeline_expressions import Field @@ -173,57 +172,6 @@ def test_equality(self, first, second, expected): assert (first == second) is expected -class TestListOfExprs: - def test_to_pb(self): - instance = _ListOfExprs([Constant(1), Constant(2)]) - result = instance._to_pb() - assert len(result.array_value.values) == 2 - assert result.array_value.values[0].integer_value == 1 - assert result.array_value.values[1].integer_value == 2 - - def test_empty_to_pb(self): - instance = _ListOfExprs([]) - result = instance._to_pb() - assert len(result.array_value.values) == 0 - - def test_repr(self): - instance = _ListOfExprs([Constant(1), Constant(2)]) - repr_string = repr(instance) - assert repr_string == "[Constant.of(1), Constant.of(2)]" - empty_instance = _ListOfExprs([]) - empty_repr_string = repr(empty_instance) - assert empty_repr_string == "[]" - - @pytest.mark.parametrize( - "first,second,expected", - [ - (_ListOfExprs([]), _ListOfExprs([]), True), - (_ListOfExprs([]), _ListOfExprs([Constant(1)]), False), - (_ListOfExprs([Constant(1)]), _ListOfExprs([]), False), - ( - _ListOfExprs([Constant(1)]), - _ListOfExprs([Constant(1)]), - True, - ), - ( - _ListOfExprs([Constant(1)]), - _ListOfExprs([Constant(2)]), - False, - ), - ( - _ListOfExprs([Constant(1), Constant(2)]), - _ListOfExprs([Constant(1), Constant(2)]), - True, - ), - (_ListOfExprs([Constant(1)]), [Constant(1)], False), - (_ListOfExprs([Constant(1)]), [1], False), - (_ListOfExprs([Constant(1)]), object(), False), - ], - ) - def test_equality(self, first, second, expected): - assert (first == second) is expected - - class TestSelectable: """ contains tests for each Expr class that derives from Selectable @@ -643,6 +591,31 @@ def test__from_query_filter_pb_unknown_filter_type(self, mock_client): BooleanExpr._from_query_filter_pb(document_pb.Value(), mock_client) +class TestArray: + """Tests for the array class""" + def test_array(self): + arg1 = Field.of("field1") + instance = expr.Array([arg1]) + assert instance.name == "array" + assert instance.params == [arg1] + assert repr(instance) == "Array([Field.of('field1')])" + + def test_empty_array(self): + instance = expr.Array([]) + assert instance.name == "array" + assert instance.params == [] + assert repr(instance) == "Array([])" + + def test_array_w_primitives(self): + a = expr.Array([1, Constant.of(2), "3"]) + assert a.name == "array" + assert a.params == [Constant.of(1), Constant.of(2), Constant.of("3")] + assert repr(a) == "Array([Constant.of(1), Constant.of(2), Constant.of('3')])" + + def test_array_w_non_list(self): + with pytest.raises(TypeError): + expr.Array(1) + class TestExpressionMethods: """ contains test methods for each Expr method @@ -723,10 +696,10 @@ def test_array_contains_any(self): arg3 = self._make_arg("Element2") instance = Expr.array_contains_any(arg1, [arg2, arg3]) assert instance.name == "array_contains_any" - assert isinstance(instance.params[1], _ListOfExprs) + assert isinstance(instance.params[1], expr.Array) assert instance.params[0] == arg1 - assert instance.params[1].exprs == [arg2, arg3] - assert repr(instance) == "ArrayField.array_contains_any([Element1, Element2])" + assert instance.params[1].params == [arg2, arg3] + assert repr(instance) == "ArrayField.array_contains_any(Array([Element1, Element2]))" infix_instance = arg1.array_contains_any([arg2, arg3]) assert infix_instance == instance @@ -805,10 +778,10 @@ def test_equal_any(self): arg3 = self._make_arg("Value2") instance = Expr.equal_any(arg1, [arg2, arg3]) assert instance.name == "equal_any" - assert isinstance(instance.params[1], _ListOfExprs) + assert isinstance(instance.params[1], expr.Array) assert instance.params[0] == arg1 - assert instance.params[1].exprs == [arg2, arg3] - assert repr(instance) == "Field.equal_any([Value1, Value2])" + assert instance.params[1].params == [arg2, arg3] + assert repr(instance) == "Field.equal_any(Array([Value1, Value2]))" infix_instance = arg1.equal_any([arg2, arg3]) assert infix_instance == instance @@ -818,10 +791,10 @@ def test_not_equal_any(self): arg3 = self._make_arg("Value2") instance = Expr.not_equal_any(arg1, [arg2, arg3]) assert instance.name == "not_equal_any" - assert isinstance(instance.params[1], _ListOfExprs) + assert isinstance(instance.params[1], expr.Array) assert instance.params[0] == arg1 - assert instance.params[1].exprs == [arg2, arg3] - assert repr(instance) == "Field.not_equal_any([Value1, Value2])" + assert instance.params[1].params == [arg2, arg3] + assert repr(instance) == "Field.not_equal_any(Array([Value1, Value2]))" infix_instance = arg1.not_equal_any([arg2, arg3]) assert infix_instance == instance @@ -883,10 +856,10 @@ def test_array_contains_all(self): arg3 = self._make_arg("Element2") instance = Expr.array_contains_all(arg1, [arg2, arg3]) assert instance.name == "array_contains_all" - assert isinstance(instance.params[1], _ListOfExprs) + assert isinstance(instance.params[1], expr.Array) assert instance.params[0] == arg1 - assert instance.params[1].exprs == [arg2, arg3] - assert repr(instance) == "ArrayField.array_contains_all([Element1, Element2])" + assert instance.params[1].params == [arg2, arg3] + assert repr(instance) == "ArrayField.array_contains_all(Array([Element1, Element2]))" infix_instance = arg1.array_contains_all([arg2, arg3]) assert infix_instance == instance @@ -1223,14 +1196,14 @@ def test_dot_product(self): @pytest.mark.parametrize("method", ["euclidean_distance", "cosine_distance", "dot_product"]) @pytest.mark.parametrize( - "input", [Vector([1.0, 2.0]), [1, 2], Constant.of(Vector([1.0, 2.0])), Constant.of([1, 2]), []] + "input", [Vector([1.0, 2.0]), [1, 2], Constant.of(Vector([1.0, 2.0])), []] ) def test_vector_ctor(self, method, input): """ test constructing various vector expressions with different inputs """ - arg1 = self._make_arg("Vector") + arg1 = self._make_arg("VectorRef") instance = getattr(arg1, method)(input) assert instance.name == method got_second_param = instance.params[1] @@ -1358,6 +1331,26 @@ def test_array_reverse(self): infix_instance = arg1.array_reverse() assert infix_instance == instance + def test_array_concat(self): + arg1 = self._make_arg("ArrayRef1") + arg2 = self._make_arg("ArrayRef2") + instance = Expr.array_concat(arg1, arg2) + assert instance.name == "array_concat" + assert instance.params == [arg1, arg2] + assert repr(instance) == "ArrayRef1.array_concat(ArrayRef2)" + infix_instance = arg1.array_concat(arg2) + assert infix_instance == instance + + def test_array_concat_multiple(self): + arg1 = expr.Array([Constant.of(0)]) + arg2 = Field.of("ArrayRef2") + arg3 = Field.of("ArrayRef3") + arg4 = [self._make_arg("Constant")] + instance = arg1.array_concat(arg2, arg3, arg4) + assert instance.name == "array_concat" + assert instance.params == [arg1, arg2, arg3, expr.Array(arg4)] + assert repr(instance) == "Array([Constant.of(0)]).array_concat(Field.of('ArrayRef2'), Field.of('ArrayRef3'), Array([Constant]))" + def test_byte_length(self): arg1 = self._make_arg("Expr") instance = Expr.byte_length(arg1) From 1b69435e07ab61d20d5df0d59e33f325c03aceea Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 24 Oct 2025 16:43:44 -0700 Subject: [PATCH 07/36] added map and related expressions --- .../firestore_v1/pipeline_expressions.py | 67 ++++++- tests/system/pipeline_e2e.yaml | 164 ++++++++++++++++++ tests/unit/v1/test_pipeline_expressions.py | 51 ++++++ 3 files changed, 279 insertions(+), 3 deletions(-) diff --git a/google/cloud/firestore_v1/pipeline_expressions.py b/google/cloud/firestore_v1/pipeline_expressions.py index 25544dfb5..f9cfd8fe0 100644 --- a/google/cloud/firestore_v1/pipeline_expressions.py +++ b/google/cloud/firestore_v1/pipeline_expressions.py @@ -119,6 +119,8 @@ def _cast_to_expr_or_convert_to_constant(o: Any, include_vector=False, include_a o = o.value if isinstance(o, Expr): return o + if isinstance(o, dict): + return Map(o) if isinstance(o, list): if include_vector and all([isinstance(i, (float, int)) for i in o]): return Constant(Vector(o)) @@ -1132,13 +1134,12 @@ def join(self, delimeter: Expr | str) -> "Expr": "join", [self, self._cast_to_expr_or_convert_to_constant(delimeter)] ) - @expose_as_static def map_get(self, key: str | Constant[str]) -> "Expr": """Accesses a value from the map produced by evaluating this expression. Example: - >>> Expr.map({"city": "London"}).map_get("city") + >>> Map({"city": "London"}).map_get("city") >>> Field.of("address").map_get("city") Args: @@ -1148,7 +1149,42 @@ def map_get(self, key: str | Constant[str]) -> "Expr": A new `Expr` representing the value associated with the given key in the map. """ return Function( - "map_get", [self, Constant.of(key) if isinstance(key, str) else key] + "map_get", [self, self._cast_to_expr_or_convert_to_constant(key)] + ) + + @expose_as_static + def map_remove(self, key: str | Constant[str]) -> "Expr": + """Remove a key from a the map produced by evaluating this expression. + + Example: + >>> Map({"city": "London"}).map_remove("city") + >>> Field.of("address").map_remove("city") + + Args: + key: The key to ewmove in the map. + + Returns: + A new `Expr` representing the map_remove operation. + """ + return Function("map_remove", [self, self._cast_to_expr_or_convert_to_constant(key)]) + + @expose_as_static + def map_merge(self, *other_maps: Map | dict[str | Constant[str], Expr | CONSTANT_TYPE] | Expr)-> "Expr": + """Creates an expression that merges one or more dicts into a single map. + + Example: + >>> Map({"city": "London"}).map_merge({"country": "UK"}, {"isCapital": True}) + >>> Field.of("settings").map_merge({"enabled":True}, Function.conditional(Field.of('isAdmin'), {"admin":True}, {}}) + + Args: + *other_maps: Sequence of maps to merge into the resulting map. + + Returns: + A new `Expr` representing the value associated with the given key in the map. + """ + return Function("map_merge", [self] + [ + self._cast_to_expr_or_convert_to_constant(m) for m in other_maps + ], ) @expose_as_static @@ -1688,6 +1724,7 @@ def _from_query_filter_pb(filter_pb, client): else: raise TypeError(f"Unexpected filter type: {type(filter_pb)}") + class Array(Function): """ Creates an expression that creates a Firestore array value from an input list. @@ -1708,6 +1745,30 @@ def __init__(self, elements: list[Expr | CONSTANT_TYPE]): def __repr__(self): return f"Array({self.params})" + +class Map(Function): + """ + Creates an expression that creates a Firestore map value from an input dict. + + Example: + >>> Expr.map({"foo": "bar", "baz": Field.of("baz")}) + + Args: + elements: THe input dict to evaluate in the expression + """ + + def __init__(self, elements: dict[str | Constant[str], Expr | CONSTANT_TYPE]): + element_list = [] + for k,v in elements.items(): + element_list.append(self._cast_to_expr_or_convert_to_constant(k)) + element_list.append(self._cast_to_expr_or_convert_to_constant(v)) + super().__init__("map", element_list) + + def __repr__(self): + d = {a.value : b for a, b in zip(self.params[::2], self.params[1::2])} + return f"Map({d})" + + class And(BooleanExpr): """ Represents an expression that performs a logical 'AND' operation on multiple filter conditions. diff --git a/tests/system/pipeline_e2e.yaml b/tests/system/pipeline_e2e.yaml index 2efd95c50..4540a281e 100644 --- a/tests/system/pipeline_e2e.yaml +++ b/tests/system/pipeline_e2e.yaml @@ -1583,6 +1583,153 @@ tests: - booleanValue: true name: equal name: where + - description: testMapGetWithField + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: "Dune" + - AddFields: + - AliasedExpr: + - Constant: "hugo" + - "award_name" + - Select: + - AliasedExpr: + - Expr.map_get: + - Field: awards + - Field: award_name + - "hugoAward" + - Field: title + assert_results: + - hugoAward: true + title: Dune + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: "Dune" + name: equal + name: where + - args: + - mapValue: + fields: + award_name: + stringValue: "hugo" + name: add_fields + - args: + - mapValue: + fields: + hugoAward: + functionValue: + name: map_get + args: + - fieldReferenceValue: awards + - fieldReferenceValue: award_name + title: + fieldReferenceValue: title + name: select + - description: testMapRemove + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: "Dune" + - Select: + - AliasedExpr: + - Expr.map_remove: + - Field: awards + - "nebula" + - "awards_removed" + assert_results: + - awards_removed: + hugo: true + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: "Dune" + name: equal + name: where + - args: + - mapValue: + fields: + awards_removed: + functionValue: + name: map_remove + args: + - fieldReferenceValue: awards + - stringValue: "nebula" + name: select + - description: testMapMerge + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: "Dune" + - Select: + - AliasedExpr: + - Expr.map_merge: + - Field: awards + - Map: + elements: {"new_award": true, "hugo": false} + - Map: + elements: {"another_award": "yes"} + - "awards_merged" + assert_results: + - awards_merged: + hugo: false + nebula: true + new_award: true + another_award: "yes" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: "Dune" + name: equal + name: where + - args: + - mapValue: + fields: + awards_merged: + functionValue: + name: map_merge + args: + - fieldReferenceValue: awards + - functionValue: + name: map + args: + - stringValue: "new_award" + - booleanValue: true + - stringValue: "hugo" + - booleanValue: false + - functionValue: + name: map + args: + - stringValue: "another_award" + - stringValue: "yes" + name: select - description: testNestedFields pipeline: - Collection: books @@ -1893,6 +2040,23 @@ tests: name: array name: array_concat name: select + - description: testMapMergeLiterals + pipeline: + - Collection: books + - Limit: 1 + - Select: + - AliasedExpr: + - Expr.map_merge: + - Map: + elements: {"a": "orig", "b": "orig"} + - Map: + elements: {"b": "new", "c": "new"} + - "merged" + assert_results: + - merged: + a: "orig" + b: "new" + c: "new" - description: testArrayContainsAnyWithField pipeline: - Collection: books diff --git a/tests/unit/v1/test_pipeline_expressions.py b/tests/unit/v1/test_pipeline_expressions.py index 21dec6e4c..5472fbeae 100644 --- a/tests/unit/v1/test_pipeline_expressions.py +++ b/tests/unit/v1/test_pipeline_expressions.py @@ -616,6 +616,36 @@ def test_array_w_non_list(self): with pytest.raises(TypeError): expr.Array(1) + +class TestMap: + """Tests for the map class""" + def test_map(self): + instance = expr.Map({Constant.of("a"): Constant.of("b")}) + assert instance.name == "map" + assert instance.params == [Constant.of("a"), Constant.of("b")] + assert repr(instance) == "Map({'a': Constant.of('b')})" + + def test_map_w_primitives(self): + instance = expr.Map({"a": "b", "0": 0, "bool": True}) + assert instance.params == [ + Constant.of("a"), Constant.of("b"), + Constant.of("0"), Constant.of(0), + Constant.of("bool"), Constant.of(True) + ] + assert repr(instance) == "Map({'a': Constant.of('b'), '0': Constant.of(0), 'bool': Constant.of(True)})" + + def test_empty_map(self): + instance = expr.Map({}) + assert instance.name == "map" + assert instance.params == [] + assert repr(instance) == "Map({})" + + def test_w_exprs(self): + instance = expr.Map({Constant.of("a"): expr.Array([1,2,3])}) + assert instance.params == [Constant.of("a"), expr.Array([1,2,3])] + assert repr(instance) == "Map({'a': Array([Constant.of(1), Constant.of(2), Constant.of(3)])})" + + class TestExpressionMethods: """ contains test methods for each Expr method @@ -1047,6 +1077,27 @@ def test_map_get(self): infix_instance = arg1.map_get(Constant.of(arg2)) assert infix_instance == instance + def test_map_remove(self): + arg1 = self._make_arg("Map") + arg2 = "key" + instance = Expr.map_remove(arg1, arg2) + assert instance.name == "map_remove" + assert instance.params == [arg1, Constant.of(arg2)] + assert repr(instance) == "Map.map_remove(Constant.of('key'))" + infix_instance = arg1.map_remove(Constant.of(arg2)) + assert infix_instance == instance + + def test_map_merge(self): + arg1 = expr.Map({"a": 1}) + arg2 = expr.Map({"b": 2}) + arg3 = {"c": 3} + instance = Expr.map_merge(arg1, arg2, arg3) + assert instance.name == "map_merge" + assert instance.params == [arg1, arg2, expr.Map(arg3)] + assert repr(instance) == "Map({'a': Constant.of(1)}).map_merge(Map({'b': Constant.of(2)}), Map({'c': Constant.of(3)}))" + infix_instance = arg1.map_merge(arg2, arg3) + assert infix_instance == instance + def test_mod(self): arg1 = self._make_arg("Left") arg2 = self._make_arg("Right") From be749e0c79fd4632fcf0fde97fc69af729560240 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 24 Oct 2025 16:53:18 -0700 Subject: [PATCH 08/36] remove dict and list from constant types --- .../cloud/firestore_v1/pipeline_expressions.py | 16 +++++++--------- tests/system/pipeline_e2e.yaml | 2 +- tests/unit/v1/test_pipeline_expressions.py | 7 ------- 3 files changed, 8 insertions(+), 17 deletions(-) diff --git a/google/cloud/firestore_v1/pipeline_expressions.py b/google/cloud/firestore_v1/pipeline_expressions.py index f9cfd8fe0..0e43b879b 100644 --- a/google/cloud/firestore_v1/pipeline_expressions.py +++ b/google/cloud/firestore_v1/pipeline_expressions.py @@ -41,8 +41,6 @@ bytes, GeoPoint, Vector, - list, - Dict[str, Any], None, ) @@ -113,7 +111,7 @@ def _to_pb(self) -> Value: raise NotImplementedError @staticmethod - def _cast_to_expr_or_convert_to_constant(o: Any, include_vector=False, include_array=False) -> "Expr": + def _cast_to_expr_or_convert_to_constant(o: Any, include_vector=False) -> "Expr": """Convert arbitrary object to an Expr.""" if isinstance(o, Constant) and isinstance(o.value, list): o = o.value @@ -124,7 +122,7 @@ def _cast_to_expr_or_convert_to_constant(o: Any, include_vector=False, include_a if isinstance(o, list): if include_vector and all([isinstance(i, (float, int)) for i in o]): return Constant(Vector(o)) - elif include_array: + else: return Array(o) return Constant(o) @@ -583,7 +581,7 @@ def equal_any(self, array: Array | Sequence[Expr | CONSTANT_TYPE] | Expr) -> "Bo "equal_any", [ self, - self._cast_to_expr_or_convert_to_constant(array, include_array=True), + self._cast_to_expr_or_convert_to_constant(array), ], ) @@ -606,7 +604,7 @@ def not_equal_any(self, array: Array | list[Expr | CONSTANT_TYPE] | Expr) -> "Bo "not_equal_any", [ self, - self._cast_to_expr_or_convert_to_constant(array, include_array=True), + self._cast_to_expr_or_convert_to_constant(array), ], ) @@ -653,7 +651,7 @@ def array_contains_all( "array_contains_all", [ self, - self._cast_to_expr_or_convert_to_constant(elements, include_array=True), + self._cast_to_expr_or_convert_to_constant(elements), ], ) @@ -681,7 +679,7 @@ def array_contains_any( "array_contains_any", [ self, - self._cast_to_expr_or_convert_to_constant(elements, include_array=True), + self._cast_to_expr_or_convert_to_constant(elements), ], ) @@ -727,7 +725,7 @@ def array_concat(self, *other_arrays: Array | list[Expr | CONSTANT_TYPE] | Expr) """ return Function( "array_concat", [self] + [ - self._cast_to_expr_or_convert_to_constant(arr, include_array=True) for arr in other_arrays + self._cast_to_expr_or_convert_to_constant(arr) for arr in other_arrays ], ) diff --git a/tests/system/pipeline_e2e.yaml b/tests/system/pipeline_e2e.yaml index 4540a281e..82247e02b 100644 --- a/tests/system/pipeline_e2e.yaml +++ b/tests/system/pipeline_e2e.yaml @@ -2064,7 +2064,7 @@ tests: - AliasedExpr: - Expr.array_concat: - Field: tags - - Constant: ["Dystopian"] + - Array: ["Dystopian"] - "new_tags" - Where: - Expr.array_contains_any: diff --git a/tests/unit/v1/test_pipeline_expressions.py b/tests/unit/v1/test_pipeline_expressions.py index 5472fbeae..36fdf6c8c 100644 --- a/tests/unit/v1/test_pipeline_expressions.py +++ b/tests/unit/v1/test_pipeline_expressions.py @@ -96,13 +96,6 @@ class TestConstant: Value(timestamp_value={"seconds": 1747008000}), ), (GeoPoint(1, 2), Value(geo_point_value={"latitude": 1, "longitude": 2})), - ( - [0.0, 1.0, 2.0], - Value( - array_value={"values": [Value(double_value=i) for i in range(3)]} - ), - ), - ({"a": "b"}, Value(map_value={"fields": {"a": Value(string_value="b")}})), ( Vector([1.0, 2.0]), Value( From 789f29cdf90eadb299d215151ab8aabe4612182f Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 24 Oct 2025 17:10:17 -0700 Subject: [PATCH 09/36] Fixed lint --- google/cloud/firestore_v1/_pipeline_stages.py | 8 +- .../firestore_v1/pipeline_expressions.py | 85 +++++++++++++------ tests/system/test_pipeline_acceptance.py | 14 ++- tests/system/test_system.py | 9 +- tests/system/test_system_async.py | 5 +- tests/unit/v1/test_aggregation.py | 2 +- tests/unit/v1/test_async_aggregation.py | 2 +- tests/unit/v1/test_pipeline_expressions.py | 50 +++++++---- 8 files changed, 119 insertions(+), 56 deletions(-) diff --git a/google/cloud/firestore_v1/_pipeline_stages.py b/google/cloud/firestore_v1/_pipeline_stages.py index 2ada1c90f..c63b748ac 100644 --- a/google/cloud/firestore_v1/_pipeline_stages.py +++ b/google/cloud/firestore_v1/_pipeline_stages.py @@ -280,9 +280,11 @@ def __init__( super().__init__("find_nearest") self.field: Expr = Field(field) if isinstance(field, str) else field self.vector: Vector = vector if isinstance(vector, Vector) else Vector(vector) - self.distance_measure = distance_measure if isinstance( - distance_measure, DistanceMeasure - ) else DistanceMeasure[distance_measure.upper()] + self.distance_measure = ( + distance_measure + if isinstance(distance_measure, DistanceMeasure) + else DistanceMeasure[distance_measure.upper()] + ) self.options = options or FindNearestOptions() def _pb_args(self): diff --git a/google/cloud/firestore_v1/pipeline_expressions.py b/google/cloud/firestore_v1/pipeline_expressions.py index 0e43b879b..3fae95494 100644 --- a/google/cloud/firestore_v1/pipeline_expressions.py +++ b/google/cloud/firestore_v1/pipeline_expressions.py @@ -17,7 +17,6 @@ Any, Generic, TypeVar, - Dict, Sequence, ) from abc import ABC @@ -334,9 +333,7 @@ def log(self, base: Expr | float) -> "Expr": Returns: A new `Expr` representing the logarithm. """ - return Function( - "log", [self, self._cast_to_expr_or_convert_to_constant(base)] - ) + return Function("log", [self, self._cast_to_expr_or_convert_to_constant(base)]) @expose_as_static def pow(self, exponent: Expr | float) -> "Expr": @@ -354,7 +351,9 @@ def pow(self, exponent: Expr | float) -> "Expr": Returns: A new `Expr` representing the power operation. """ - return Function("pow", [self, self._cast_to_expr_or_convert_to_constant(exponent)]) + return Function( + "pow", [self, self._cast_to_expr_or_convert_to_constant(exponent)] + ) @expose_as_static def round(self) -> "Expr": @@ -563,7 +562,9 @@ def less_than_or_equal(self, other: Expr | CONSTANT_TYPE) -> "BooleanExpr": ) @expose_as_static - def equal_any(self, array: Array | Sequence[Expr | CONSTANT_TYPE] | Expr) -> "BooleanExpr": + def equal_any( + self, array: Array | Sequence[Expr | CONSTANT_TYPE] | Expr + ) -> "BooleanExpr": """Creates an expression that checks if this expression is equal to any of the provided values or expressions. @@ -586,7 +587,9 @@ def equal_any(self, array: Array | Sequence[Expr | CONSTANT_TYPE] | Expr) -> "Bo ) @expose_as_static - def not_equal_any(self, array: Array | list[Expr | CONSTANT_TYPE] | Expr) -> "BooleanExpr": + def not_equal_any( + self, array: Array | list[Expr | CONSTANT_TYPE] | Expr + ) -> "BooleanExpr": """Creates an expression that checks if this expression is not equal to any of the provided values or expressions. @@ -710,7 +713,9 @@ def array_reverse(self) -> "Expr": return Function("array_reverse", [self]) @expose_as_static - def array_concat(self, *other_arrays: Array | list[Expr | CONSTANT_TYPE] | Expr) -> "Expr": + def array_concat( + self, *other_arrays: Array | list[Expr | CONSTANT_TYPE] | Expr + ) -> "Expr": """Creates an expression that concatenates an array expression with another array. Example: @@ -724,9 +729,9 @@ def array_concat(self, *other_arrays: Array | list[Expr | CONSTANT_TYPE] | Expr) A new `Expr` representing the concatenated array. """ return Function( - "array_concat", [self] + [ - self._cast_to_expr_or_convert_to_constant(arr) for arr in other_arrays - ], + "array_concat", + [self] + + [self._cast_to_expr_or_convert_to_constant(arr) for arr in other_arrays], ) @expose_as_static @@ -1093,7 +1098,9 @@ def string_reverse(self) -> "Expr": return Function("string_reverse", [self]) @expose_as_static - def substring(self, position: Expr | int, length: Expr | int | None=None) -> "Expr": + def substring( + self, position: Expr | int, length: Expr | int | None = None + ) -> "Expr": """Creates an expression that returns a substring of the results of this expression. @@ -1164,10 +1171,14 @@ def map_remove(self, key: str | Constant[str]) -> "Expr": Returns: A new `Expr` representing the map_remove operation. """ - return Function("map_remove", [self, self._cast_to_expr_or_convert_to_constant(key)]) + return Function( + "map_remove", [self, self._cast_to_expr_or_convert_to_constant(key)] + ) @expose_as_static - def map_merge(self, *other_maps: Map | dict[str | Constant[str], Expr | CONSTANT_TYPE] | Expr)-> "Expr": + def map_merge( + self, *other_maps: Map | dict[str | Constant[str], Expr | CONSTANT_TYPE] | Expr + ) -> "Expr": """Creates an expression that merges one or more dicts into a single map. Example: @@ -1180,9 +1191,9 @@ def map_merge(self, *other_maps: Map | dict[str | Constant[str], Expr | CONSTANT Returns: A new `Expr` representing the value associated with the given key in the map. """ - return Function("map_merge", [self] + [ - self._cast_to_expr_or_convert_to_constant(m) for m in other_maps - ], + return Function( + "map_merge", + [self] + [self._cast_to_expr_or_convert_to_constant(m) for m in other_maps], ) @expose_as_static @@ -1201,7 +1212,13 @@ def cosine_distance(self, other: Expr | list[float] | Vector) -> "Expr": Returns: A new `Expr` representing the cosine distance between the two vectors. """ - return Function("cosine_distance", [self, self._cast_to_expr_or_convert_to_constant(other, include_vector=True)]) + return Function( + "cosine_distance", + [ + self, + self._cast_to_expr_or_convert_to_constant(other, include_vector=True), + ], + ) @expose_as_static def euclidean_distance(self, other: Expr | list[float] | Vector) -> "Expr": @@ -1219,7 +1236,13 @@ def euclidean_distance(self, other: Expr | list[float] | Vector) -> "Expr": Returns: A new `Expr` representing the Euclidean distance between the two vectors. """ - return Function("euclidean_distance", [self, self._cast_to_expr_or_convert_to_constant(other, include_vector=True)]) + return Function( + "euclidean_distance", + [ + self, + self._cast_to_expr_or_convert_to_constant(other, include_vector=True), + ], + ) @expose_as_static def dot_product(self, other: Expr | list[float] | Vector) -> "Expr": @@ -1237,7 +1260,13 @@ def dot_product(self, other: Expr | list[float] | Vector) -> "Expr": Returns: A new `Expr` representing the dot product between the two vectors. """ - return Function("dot_product", [self, self._cast_to_expr_or_convert_to_constant(other, include_vector=True)]) + return Function( + "dot_product", + [ + self, + self._cast_to_expr_or_convert_to_constant(other, include_vector=True), + ], + ) @expose_as_static def vector_length(self) -> "Expr": @@ -1737,7 +1766,9 @@ class Array(Function): def __init__(self, elements: list[Expr | CONSTANT_TYPE]): if not isinstance(elements, list): raise TypeError("Array must be constructed with a list") - converted_elements = [self._cast_to_expr_or_convert_to_constant(el) for el in elements] + converted_elements = [ + self._cast_to_expr_or_convert_to_constant(el) for el in elements + ] super().__init__("array", converted_elements) def __repr__(self): @@ -1757,13 +1788,16 @@ class Map(Function): def __init__(self, elements: dict[str | Constant[str], Expr | CONSTANT_TYPE]): element_list = [] - for k,v in elements.items(): + for k, v in elements.items(): element_list.append(self._cast_to_expr_or_convert_to_constant(k)) element_list.append(self._cast_to_expr_or_convert_to_constant(v)) super().__init__("map", element_list) def __repr__(self): - d = {a.value : b for a, b in zip(self.params[::2], self.params[1::2])} + formatted_params = [ + a.value if isinstance(a, Constant) else a for a in self.params + ] + d = {a: b for a, b in zip(formatted_params[::2], formatted_params[1::2])} return f"Map({d})" @@ -1854,6 +1888,7 @@ def __init__(self, condition: BooleanExpr, then_expr: Expr, else_expr: Expr): "conditional", [condition, then_expr, else_expr], use_infix_repr=False ) + class Count(AggregateFunction): """ Represents an aggregation that counts the number of stage inputs with valid evaluations of the @@ -1871,6 +1906,4 @@ class Count(AggregateFunction): def __init__(self, expression: Expr | None = None): expression_list = [expression] if expression else [] - super().__init__( - "count", expression_list, use_infix_repr=bool(expression_list) - ) + super().__init__("count", expression_list, use_infix_repr=bool(expression_list)) diff --git a/tests/system/test_pipeline_acceptance.py b/tests/system/test_pipeline_acceptance.py index 3c41c0227..313b9d673 100644 --- a/tests/system/test_pipeline_acceptance.py +++ b/tests/system/test_pipeline_acceptance.py @@ -95,7 +95,9 @@ def test_pipeline_results(test_dict, client): Ensure pipeline returns expected results """ expected_results = _parse_yaml_types(test_dict.get("assert_results", None)) - expected_approximate_results = _parse_yaml_types(test_dict.get("assert_results_approximate", None)) + expected_approximate_results = _parse_yaml_types( + test_dict.get("assert_results_approximate", None) + ) expected_count = test_dict.get("assert_count", None) pipeline = parse_pipeline(client, test_dict["pipeline"]) # check if server responds as expected @@ -139,7 +141,9 @@ async def test_pipeline_results_async(test_dict, async_client): Ensure pipeline returns expected results """ expected_results = _parse_yaml_types(test_dict.get("assert_results", None)) - expected_approximate_results = _parse_yaml_types(test_dict.get("assert_results_approximate", None)) + expected_approximate_results = _parse_yaml_types( + test_dict.get("assert_results_approximate", None) + ) expected_count = test_dict.get("assert_count", None) pipeline = parse_pipeline(async_client, test_dict["pipeline"]) # check if server responds as expected @@ -225,7 +229,11 @@ def _apply_yaml_args_to_callable(callable_obj, client, yaml_args): """ if isinstance(yaml_args, dict): return callable_obj(**_parse_expressions(client, yaml_args)) - elif isinstance(yaml_args, list) and not (callable_obj == expr.Constant or callable_obj == Vector or callable_obj == expr.Array): + elif isinstance(yaml_args, list) and not ( + callable_obj == expr.Constant + or callable_obj == Vector + or callable_obj == expr.Array + ): # yaml has an array of arguments. Treat as args return callable_obj(*_parse_expressions(client, yaml_args)) else: diff --git a/tests/system/test_system.py b/tests/system/test_system.py index a8f94e2ba..c2bd93ef8 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -109,9 +109,11 @@ def _clean_results(results): if isinstance(query, BaseAggregationQuery): # aggregation queries return a list of lists of aggregation results query_results = _clean_results( - list(itertools.chain.from_iterable( - [[a._to_dict() for a in s] for s in query.get()] - )) + list( + itertools.chain.from_iterable( + [[a._to_dict() for a in s] for s in query.get()] + ) + ) ) else: # other qureies return a simple list of results @@ -1531,6 +1533,7 @@ def test_query_stream_or_get_w_no_explain_options(query_docs, database, method): results.get_explain_metrics() verify_pipeline(query) + @pytest.mark.skipif( FIRESTORE_EMULATOR, reason="Query profile not supported in emulator." ) diff --git a/tests/system/test_system_async.py b/tests/system/test_system_async.py index b78a77786..d053cbd7a 100644 --- a/tests/system/test_system_async.py +++ b/tests/system/test_system_async.py @@ -208,7 +208,9 @@ def _clean_results(results): await pipeline.execute() else: # ensure results match query - pipeline_results = _clean_results([s.data() async for s in pipeline.stream()]) + pipeline_results = _clean_results( + [s.data() async for s in pipeline.stream()] + ) assert query_results == pipeline_results except FailedPrecondition as e: # if testing against a non-enterprise db, skip this check @@ -216,7 +218,6 @@ def _clean_results(results): raise e - @pytest.fixture(scope="module") def event_loop(): """Change event_loop fixture to module level.""" diff --git a/tests/unit/v1/test_aggregation.py b/tests/unit/v1/test_aggregation.py index 5064e87ae..9a20fd386 100644 --- a/tests/unit/v1/test_aggregation.py +++ b/tests/unit/v1/test_aggregation.py @@ -1136,7 +1136,7 @@ def test_aggreation_to_pipeline_count_increment(): assert len(aggregate_stage.accumulators) == n for i in range(n): assert isinstance(aggregate_stage.accumulators[i].expr, Count) - assert aggregate_stage.accumulators[i].alias == f"field_{i+1}" + assert aggregate_stage.accumulators[i].alias == f"field_{i + 1}" def test_aggreation_to_pipeline_complex(): diff --git a/tests/unit/v1/test_async_aggregation.py b/tests/unit/v1/test_async_aggregation.py index fdd4a1450..701feab5b 100644 --- a/tests/unit/v1/test_async_aggregation.py +++ b/tests/unit/v1/test_async_aggregation.py @@ -810,7 +810,7 @@ def test_aggreation_to_pipeline_count_increment(): assert len(aggregate_stage.accumulators) == n for i in range(n): assert isinstance(aggregate_stage.accumulators[i].expr, Count) - assert aggregate_stage.accumulators[i].alias == f"field_{i+1}" + assert aggregate_stage.accumulators[i].alias == f"field_{i + 1}" def test_async_aggreation_to_pipeline_complex(): diff --git a/tests/unit/v1/test_pipeline_expressions.py b/tests/unit/v1/test_pipeline_expressions.py index 36fdf6c8c..bfd8a8270 100644 --- a/tests/unit/v1/test_pipeline_expressions.py +++ b/tests/unit/v1/test_pipeline_expressions.py @@ -398,9 +398,7 @@ def test__from_query_filter_pb_composite_filter_nested(self, mock_client): field3 = Field.of("field3") expected_cond1 = expr.And(field1.exists(), field1.equal(Constant("val1"))) expected_cond2 = expr.And(field2.exists(), field2.greater_than(Constant(10))) - expected_cond3 = expr.And( - field3.exists(), field3.is_not_null() - ) + expected_cond3 = expr.And(field3.exists(), field3.is_not_null()) expected_inner_and = expr.And(expected_cond2, expected_cond3) expected_outer_or = expr.Or(expected_cond1, expected_inner_and) @@ -586,6 +584,7 @@ def test__from_query_filter_pb_unknown_filter_type(self, mock_client): class TestArray: """Tests for the array class""" + def test_array(self): arg1 = Field.of("field1") instance = expr.Array([arg1]) @@ -612,20 +611,24 @@ def test_array_w_non_list(self): class TestMap: """Tests for the map class""" + def test_map(self): instance = expr.Map({Constant.of("a"): Constant.of("b")}) assert instance.name == "map" assert instance.params == [Constant.of("a"), Constant.of("b")] - assert repr(instance) == "Map({'a': Constant.of('b')})" + assert repr(instance) == "Map({'a': 'b'})" def test_map_w_primitives(self): instance = expr.Map({"a": "b", "0": 0, "bool": True}) assert instance.params == [ - Constant.of("a"), Constant.of("b"), - Constant.of("0"), Constant.of(0), - Constant.of("bool"), Constant.of(True) + Constant.of("a"), + Constant.of("b"), + Constant.of("0"), + Constant.of(0), + Constant.of("bool"), + Constant.of(True), ] - assert repr(instance) == "Map({'a': Constant.of('b'), '0': Constant.of(0), 'bool': Constant.of(True)})" + assert repr(instance) == "Map({'a': 'b', '0': 0, 'bool': True})" def test_empty_map(self): instance = expr.Map({}) @@ -634,9 +637,12 @@ def test_empty_map(self): assert repr(instance) == "Map({})" def test_w_exprs(self): - instance = expr.Map({Constant.of("a"): expr.Array([1,2,3])}) - assert instance.params == [Constant.of("a"), expr.Array([1,2,3])] - assert repr(instance) == "Map({'a': Array([Constant.of(1), Constant.of(2), Constant.of(3)])})" + instance = expr.Map({Constant.of("a"): expr.Array([1, 2, 3])}) + assert instance.params == [Constant.of("a"), expr.Array([1, 2, 3])] + assert ( + repr(instance) + == "Map({'a': Array([Constant.of(1), Constant.of(2), Constant.of(3)])})" + ) class TestExpressionMethods: @@ -722,7 +728,10 @@ def test_array_contains_any(self): assert isinstance(instance.params[1], expr.Array) assert instance.params[0] == arg1 assert instance.params[1].params == [arg2, arg3] - assert repr(instance) == "ArrayField.array_contains_any(Array([Element1, Element2]))" + assert ( + repr(instance) + == "ArrayField.array_contains_any(Array([Element1, Element2]))" + ) infix_instance = arg1.array_contains_any([arg2, arg3]) assert infix_instance == instance @@ -882,7 +891,10 @@ def test_array_contains_all(self): assert isinstance(instance.params[1], expr.Array) assert instance.params[0] == arg1 assert instance.params[1].params == [arg2, arg3] - assert repr(instance) == "ArrayField.array_contains_all(Array([Element1, Element2]))" + assert ( + repr(instance) + == "ArrayField.array_contains_all(Array([Element1, Element2]))" + ) infix_instance = arg1.array_contains_all([arg2, arg3]) assert infix_instance == instance @@ -1087,7 +1099,7 @@ def test_map_merge(self): instance = Expr.map_merge(arg1, arg2, arg3) assert instance.name == "map_merge" assert instance.params == [arg1, arg2, expr.Map(arg3)] - assert repr(instance) == "Map({'a': Constant.of(1)}).map_merge(Map({'b': Constant.of(2)}), Map({'c': Constant.of(3)}))" + assert repr(instance) == "Map({'a': 1}).map_merge(Map({'b': 2}), Map({'c': 3}))" infix_instance = arg1.map_merge(arg2, arg3) assert infix_instance == instance @@ -1238,7 +1250,9 @@ def test_dot_product(self): infix_instance = arg1.dot_product(arg2) assert infix_instance == instance - @pytest.mark.parametrize("method", ["euclidean_distance", "cosine_distance", "dot_product"]) + @pytest.mark.parametrize( + "method", ["euclidean_distance", "cosine_distance", "dot_product"] + ) @pytest.mark.parametrize( "input", [Vector([1.0, 2.0]), [1, 2], Constant.of(Vector([1.0, 2.0])), []] ) @@ -1254,7 +1268,6 @@ def test_vector_ctor(self, method, input): assert isinstance(got_second_param, Constant) assert isinstance(got_second_param.value, Vector) - def test_vector_length(self): arg1 = self._make_arg("Array") instance = Expr.vector_length(arg1) @@ -1393,7 +1406,10 @@ def test_array_concat_multiple(self): instance = arg1.array_concat(arg2, arg3, arg4) assert instance.name == "array_concat" assert instance.params == [arg1, arg2, arg3, expr.Array(arg4)] - assert repr(instance) == "Array([Constant.of(0)]).array_concat(Field.of('ArrayRef2'), Field.of('ArrayRef3'), Array([Constant]))" + assert ( + repr(instance) + == "Array([Constant.of(0)]).array_concat(Field.of('ArrayRef2'), Field.of('ArrayRef3'), Array([Constant]))" + ) def test_byte_length(self): arg1 = self._make_arg("Expr") From 64be10db7fa558a87bf096fad49f8d920c43b384 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 27 Oct 2025 14:27:38 -0700 Subject: [PATCH 10/36] added count_if and count_distinct --- .../firestore_v1/pipeline_expressions.py | 29 ++++++++++ tests/system/pipeline_e2e.yaml | 58 +++++++++++++++++++ tests/unit/v1/test_pipeline_expressions.py | 18 ++++++ 3 files changed, 105 insertions(+) diff --git a/google/cloud/firestore_v1/pipeline_expressions.py b/google/cloud/firestore_v1/pipeline_expressions.py index 3fae95494..516349041 100644 --- a/google/cloud/firestore_v1/pipeline_expressions.py +++ b/google/cloud/firestore_v1/pipeline_expressions.py @@ -852,6 +852,35 @@ def count(self) -> "Expr": """ return AggregateFunction("count", [self]) + @expose_as_static + def count_if(self) -> "Expr": + """Creates an aggregation that counts the number of values of the provided field or expression + that evaluate to True. + + Example: + >>> # Count the number of adults + >>> Field.of("age").greater_than(18).count_if().as_("totalAdults") + + + Returns: + A new `AggregateFunction` representing the 'count_if' aggregation. + """ + return AggregateFunction("count_if", [self]) + + @expose_as_static + def count_distinct(self) -> "Expr": + """Creates an aggregation that counts the number of distinct values of the + provided field or expression. + + Example: + >>> # Count the total number of countries in the data + >>> Field.of("country").count_distinct().as_("totalCountries") + + Returns: + A new `AggregateFunction` representing the 'count_distinct' aggregation. + """ + return AggregateFunction("count_distinct", [self]) + @expose_as_static def minimum(self) -> "Expr": """Creates an aggregation that finds the minimum value of a field across multiple stage inputs. diff --git a/tests/system/pipeline_e2e.yaml b/tests/system/pipeline_e2e.yaml index 82247e02b..93e02f3a2 100644 --- a/tests/system/pipeline_e2e.yaml +++ b/tests/system/pipeline_e2e.yaml @@ -167,6 +167,64 @@ tests: - fieldReferenceValue: rating - mapValue: {} name: aggregate + - description: "testAggregates - count_if" + pipeline: + - Collection: books + - Aggregate: + - AliasedExpr: + - Expr.count_if: + - Expr.greater_than: + - Field: rating + - Constant: 4.2 + - "count_if_rating_gt_4_2" + assert_results: + - count_if_rating_gt_4_2: 5 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + count_if_rating_gt_4_2: + functionValue: + name: count_if + args: + - functionValue: + name: greater_than + args: + - fieldReferenceValue: rating + - doubleValue: 4.2 + - mapValue: {} + name: aggregate + - description: "testAggregates - count_distinct" + pipeline: + - Collection: books + - Aggregate: + - AliasedExpr: + - Expr.count_distinct: + - Field: genre + - "distinct_genres" + assert_results: + - distinct_genres: 8 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + distinct_genres: + functionValue: + name: count_distinct + args: + - fieldReferenceValue: genre + - mapValue: {} + name: aggregate - description: "testAggregates - avg, count, max" pipeline: - Collection: books diff --git a/tests/unit/v1/test_pipeline_expressions.py b/tests/unit/v1/test_pipeline_expressions.py index bfd8a8270..45d26dca6 100644 --- a/tests/unit/v1/test_pipeline_expressions.py +++ b/tests/unit/v1/test_pipeline_expressions.py @@ -1471,6 +1471,24 @@ def test_base_count(self): assert instance.params == [] assert repr(instance) == "Count()" + def test_count_if(self): + arg1 = self._make_arg("Value") + instance = Expr.count_if(arg1) + assert instance.name == "count_if" + assert instance.params == [arg1] + assert repr(instance) == "Value.count_if()" + infix_instance = arg1.count_if() + assert infix_instance == instance + + def test_count_distinct(self): + arg1 = self._make_arg("Value") + instance = Expr.count_distinct(arg1) + assert instance.name == "count_distinct" + assert instance.params == [arg1] + assert repr(instance) == "Value.count_distinct()" + infix_instance = arg1.count_distinct() + assert infix_instance == instance + def test_minimum(self): arg1 = self._make_arg("Value") instance = Expr.minimum(arg1) From 5d4f8783473095408c735fa8a7d1b55bbf296965 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 27 Oct 2025 15:34:22 -0700 Subject: [PATCH 11/36] added misc expressions --- .../firestore_v1/pipeline_expressions.py | 61 +++++++ tests/system/pipeline_e2e.yaml | 157 +++++++++++++++++- tests/system/test_pipeline_acceptance.py | 12 +- tests/unit/v1/test_pipeline_expressions.py | 46 ++++- 4 files changed, 267 insertions(+), 9 deletions(-) diff --git a/google/cloud/firestore_v1/pipeline_expressions.py b/google/cloud/firestore_v1/pipeline_expressions.py index 516349041..7a582f2f2 100644 --- a/google/cloud/firestore_v1/pipeline_expressions.py +++ b/google/cloud/firestore_v1/pipeline_expressions.py @@ -335,6 +335,18 @@ def log(self, base: Expr | float) -> "Expr": """ return Function("log", [self, self._cast_to_expr_or_convert_to_constant(base)]) + @expose_as_static + def log10(self) -> "Expr": + """Creates an expression that calculates the base 10 logarithm of this expression. + + Example: + >>> Field.of("value").log10() + + Returns: + A new `Expr` representing the logarithm. + """ + return Function("log10", [self]) + @expose_as_static def pow(self, exponent: Expr | float) -> "Expr": """Creates an expression that calculates this expression raised to the power of the exponent. @@ -734,6 +746,32 @@ def array_concat( + [self._cast_to_expr_or_convert_to_constant(arr) for arr in other_arrays], ) + @expose_as_static + def concat(self, *others: Expr | CONSTANT_TYPE) -> "Expr": + """Creates an expression that concatenates expressions together + + Args: + *others: The expressions to concatenate. + + Returns: + A new `Expr` representing the concatenated value. + """ + return Function("concat", [self] + [self._cast_to_expr_or_convert_to_constant(o) for o in others]) + + @expose_as_static + def length(self) -> "Expr": + """ + Creates an expression that calculates the length of the expression if it is a string, array, map, or blob. + + Example: + >>> # Get the length of the 'name' field. + >>> Field.of("name").length() + + Returns: + A new `Expr` representing the length of the expression. + """ + return Function("length", [self]) + @expose_as_static def is_absent(self) -> "BooleanExpr": """Creates an expression that returns true if a value is absent. Otherwise, returns false even if @@ -1467,6 +1505,19 @@ def collection_id(self): """ return Function("collection_id", [self]) + @expose_as_static + def document_id(self): + """Creates an expression that returns the document ID from a path. + + Example: + >>> # Get the document ID from a path. + >>> Field.of("__name__").document_id() + + Returns: + A new `Expr` representing the document ID. + """ + return Function("document_id", [self]) + def ascending(self) -> Ordering: """Creates an `Ordering` that sorts documents in ascending order based on this expression. @@ -1936,3 +1987,13 @@ class Count(AggregateFunction): def __init__(self, expression: Expr | None = None): expression_list = [expression] if expression else [] super().__init__("count", expression_list, use_infix_repr=bool(expression_list)) + +class CurrentTimestamp(Function): + """Creates an expression that returns the current timestamp + + Returns: + A new `Expr` representing the current timestamp. + """ + + def __init__(self): + super().__init__("current_timestamp", [], use_infix_repr=False) \ No newline at end of file diff --git a/tests/system/pipeline_e2e.yaml b/tests/system/pipeline_e2e.yaml index 93e02f3a2..215f761fd 100644 --- a/tests/system/pipeline_e2e.yaml +++ b/tests/system/pipeline_e2e.yaml @@ -993,7 +993,57 @@ tests: expression: fieldReferenceValue: title name: sort + - description: testConcat + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: "The Hitchhiker's Guide to the Galaxy" + - Select: + - AliasedExpr: + - Expr.concat: + - Field: author + - Constant: ": " + - Field: title + - "author_title" + - AliasedExpr: + - Expr.concat: + - Field: tags + - - Constant: "new_tag" + - "concatenatedTags" + assert_results: + - author_title: "Douglas Adams: The Hitchhiker's Guide to the Galaxy" + concatenatedTags: + - comedy + - space + - adventure + - new_tag - description: testLength + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: "The Hitchhiker's Guide to the Galaxy" + - Select: + - AliasedExpr: + - Expr.length: + - Field: title + - "titleLength" + - AliasedExpr: + - Expr.length: + - Field: tags + - "tagsLength" + - AliasedExpr: + - Expr.length: + - Field: awards + - "awardsLength" + assert_results: + - titleLength: 36 + tagsLength: 3 + awardsLength: 2 + - description: testCharLength pipeline: - Collection: books - Select: @@ -1998,6 +2048,95 @@ tests: - adventure - space - comedy + - description: testDocumentId + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: "The Hitchhiker's Guide to the Galaxy" + - Select: + - AliasedExpr: + - Expr.document_id: + - Field: __name__ + - "doc_id" + assert_results: + - doc_id: "book1" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: "The Hitchhiker's Guide to the Galaxy" + name: equal + name: where + - args: + - mapValue: + fields: + doc_id: + functionValue: + name: document_id + args: + - fieldReferenceValue: __name__ + name: select + - description: testCurrentTimestamp + pipeline: + - Collection: books + - Limit: 1 + - Select: + - AliasedExpr: + - And: + - Expr.greater_than_or_equal: + - CurrentTimestamp: [] + - Expr.unix_seconds_to_timestamp: + - Constant: 1735689600 # 2025-01-01 + - Expr.less_than: + - CurrentTimestamp: [] + - Expr.unix_seconds_to_timestamp: + - Constant: 4892438400 # 2125-01-01 + - "is_between_2025_and_2125" + assert_results: + - is_between_2025_and_2125: true + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - integerValue: '1' + name: limit + - args: + - mapValue: + fields: + is_between_2025_and_2125: + functionValue: + name: and + args: + - functionValue: + name: greater_than_or_equal + args: + - functionValue: + name: current_timestamp + - functionValue: + name: unix_seconds_to_timestamp + args: + - integerValue: '1735689600' + - functionValue: + name: less_than + args: + - functionValue: + name: current_timestamp + - functionValue: + name: unix_seconds_to_timestamp + args: + - integerValue: '4892438400' + name: select - description: testArrayConcat pipeline: - Collection: books @@ -2532,10 +2671,14 @@ tests: - Field: rating - "ln_rating" - AliasedExpr: - - Expr.log: + - Expr.log10: - Field: rating - - Constant: 10 - "log_rating_base10" + - AliasedExpr: + - Expr.log: + - Field: rating + - Constant: 2 + - "log_rating_base2" - AliasedExpr: - Expr.pow: - Field: rating @@ -2552,6 +2695,7 @@ tests: floor_rating: 4.0 ln_rating: 1.4350845 log_rating_base10: 0.623249 + log_rating_base2: 2.0704 pow_rating: 17.64 sqrt_rating: 2.049390 assert_proto: @@ -2599,7 +2743,12 @@ tests: functionValue: args: - fieldReferenceValue: rating - - integerValue: '10' + name: log10 + log_rating_base2: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '2' name: log pow_rating: functionValue: @@ -3037,4 +3186,4 @@ tests: - fieldReferenceValue: tags - stringValue: ", " name: join - name: select \ No newline at end of file + name: select diff --git a/tests/system/test_pipeline_acceptance.py b/tests/system/test_pipeline_acceptance.py index 313b9d673..3b3e6189d 100644 --- a/tests/system/test_pipeline_acceptance.py +++ b/tests/system/test_pipeline_acceptance.py @@ -87,7 +87,7 @@ def test_pipeline_expected_errors(test_dict, client): @pytest.mark.parametrize( "test_dict", - [t for t in yaml_loader() if "assert_results" in t or "assert_count" in t], + [t for t in yaml_loader() if "assert_results" in t or "assert_count" in t or "assert_results_approximate" in t], ids=lambda x: f"{x.get('description', '')}", ) def test_pipeline_results(test_dict, client): @@ -105,7 +105,9 @@ def test_pipeline_results(test_dict, client): if expected_results: assert got_results == expected_results if expected_approximate_results: - assert got_results == pytest.approximate(expected_approximate_results) + assert len(got_results) == len(expected_approximate_results), "got unexpected result count" + for idx in range(len(got_results)): + assert got_results[idx] == pytest.approx(expected_approximate_results[idx], abs=1e-4) if expected_count is not None: assert len(got_results) == expected_count @@ -132,7 +134,7 @@ async def test_pipeline_expected_errors_async(test_dict, async_client): @pytest.mark.parametrize( "test_dict", - [t for t in yaml_loader() if "assert_results" in t or "assert_count" in t], + [t for t in yaml_loader() if "assert_results" in t or "assert_count" in t or "assert_results_approximate" in t], ids=lambda x: f"{x.get('description', '')}", ) @pytest.mark.asyncio @@ -151,7 +153,9 @@ async def test_pipeline_results_async(test_dict, async_client): if expected_results: assert got_results == expected_results if expected_approximate_results: - assert got_results == pytest.approximate(expected_approximate_results) + assert len(got_results) == len(expected_approximate_results), "got unexpected result count" + for idx in range(len(got_results)): + assert got_results[idx] == pytest.approx(expected_approximate_results[idx], abs=1e-4) if expected_count is not None: assert len(got_results) == expected_count diff --git a/tests/unit/v1/test_pipeline_expressions.py b/tests/unit/v1/test_pipeline_expressions.py index 45d26dca6..28a5973de 100644 --- a/tests/unit/v1/test_pipeline_expressions.py +++ b/tests/unit/v1/test_pipeline_expressions.py @@ -1144,6 +1144,12 @@ def test_subtract(self): infix_instance = arg1.subtract(arg2) assert infix_instance == instance + def test_current_timestamp(self): + instance = expr.CurrentTimestamp() + assert instance.name == "current_timestamp" + assert instance.params == [] + assert repr(instance) == "CurrentTimestamp()" + def test_timestamp_add(self): arg1 = self._make_arg("Timestamp") arg2 = self._make_arg("Unit") @@ -1342,6 +1348,15 @@ def test_log(self): infix_instance = arg1.log(arg2) assert infix_instance == instance + def test_log10(self): + arg1 = self._make_arg("Value") + instance = Expr.log10(arg1) + assert instance.name == "log10" + assert instance.params == [arg1] + assert repr(instance) == "Value.log10()" + infix_instance = arg1.log10() + assert infix_instance == instance + def test_pow(self): arg1 = self._make_arg("Value") arg2 = self._make_arg("Exponent") @@ -1429,6 +1444,26 @@ def test_char_length(self): infix_instance = arg1.char_length() assert infix_instance == instance + def test_concat(self): + arg1 = self._make_arg("First") + arg2 = self._make_arg("Second") + arg3 = "Third" + instance = Expr.concat(arg1, arg2, arg3) + assert instance.name == "concat" + assert instance.params == [arg1, arg2, Constant.of(arg3)] + assert repr(instance) == "First.concat(Second, Constant.of('Third'))" + infix_instance = arg1.concat(arg2, arg3) + assert infix_instance == instance + + def test_length(self): + arg1 = self._make_arg("Expr") + instance = Expr.length(arg1) + assert instance.name == "length" + assert instance.params == [arg1] + assert repr(instance) == "Expr.length()" + infix_instance = arg1.length() + assert infix_instance == instance + def test_collection_id(self): arg1 = self._make_arg("Value") instance = Expr.collection_id(arg1) @@ -1438,6 +1473,15 @@ def test_collection_id(self): infix_instance = arg1.collection_id() assert infix_instance == instance + def test_document_id(self): + arg1 = self._make_arg("Value") + instance = Expr.document_id(arg1) + assert instance.name == "document_id" + assert instance.params == [arg1] + assert repr(instance) == "Value.document_id()" + infix_instance = arg1.document_id() + assert infix_instance == instance + def test_sum(self): arg1 = self._make_arg("Value") instance = Expr.sum(arg1) @@ -1505,4 +1549,4 @@ def test_maximum(self): assert instance.params == [arg1] assert repr(instance) == "Value.maximum()" infix_instance = arg1.maximum() - assert infix_instance == instance + assert infix_instance == instance \ No newline at end of file From 6d6c57f651cd2c5d9509ec8a25a0d874328e42e3 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 27 Oct 2025 16:05:23 -0700 Subject: [PATCH 12/36] added error functions --- .../firestore_v1/pipeline_expressions.py | 51 ++++++++ tests/system/pipeline_e2e.yaml | 114 ++++++++++++++++++ tests/unit/v1/test_pipeline_expressions.py | 30 +++++ 3 files changed, 195 insertions(+) diff --git a/google/cloud/firestore_v1/pipeline_expressions.py b/google/cloud/firestore_v1/pipeline_expressions.py index 7a582f2f2..5fc425642 100644 --- a/google/cloud/firestore_v1/pipeline_expressions.py +++ b/google/cloud/firestore_v1/pipeline_expressions.py @@ -786,6 +786,24 @@ def is_absent(self) -> "BooleanExpr": """ return BooleanExpr("is_absent", [self]) + @expose_as_static + def if_absent(self, default_value: Expr | CONSTANT_TYPE) -> "Expr": + """Creates an expression that returns a default value if an expression evaluates to an absent value. + + Example: + >>> # Return the value of the 'email' field, or "N/A" if it's absent. + >>> Field.of("email").if_absent("N/A") + + Args: + default_value: The expression or constant value to return if this expression is absent. + + Returns: + A new `Expr` representing the ifAbsent operation. + """ + return Function( + "if_absent", [self, self._cast_to_expr_or_convert_to_constant(default_value)] + ) + @expose_as_static def is_nan(self) -> "BooleanExpr": """Creates an expression that checks if this expression evaluates to 'NaN' (Not a Number). @@ -836,6 +854,38 @@ def is_not_null(self) -> "BooleanExpr": """ return BooleanExpr("is_not_null", [self]) + @expose_as_static + def is_error(self): + """Creates an expression that checks if a given expression produces an error + + Example: + >>> # Resolves to True if an expression produces an error + >>> Field.of("value").divide("string").is_error() + + Returns: + A new `Expr` representing the isError operation. + """ + return Function("is_error", [self]) + + @expose_as_static + def if_error(self, then_value: Expr | CONSTANT_TYPE) -> "Expr": + """Creates an expression that returns ``then_value`` if this expression evaluates to an error. + Otherwise, returns the value of this expression. + + Example: + >>> # Resolves to 0 if an expression produces an error + >>> Field.of("value").divide("string").if_error(0) + + Args: + then_value: The value to return if this expression evaluates to an error. + + Returns: + A new `Expr` representing the ifError operation. + """ + return Function( + "if_error", [self, self._cast_to_expr_or_convert_to_constant(then_value)] + ) + @expose_as_static def exists(self) -> "BooleanExpr": """Creates an expression that checks if a field exists in the document. @@ -1988,6 +2038,7 @@ def __init__(self, expression: Expr | None = None): expression_list = [expression] if expression else [] super().__init__("count", expression_list, use_infix_repr=bool(expression_list)) + class CurrentTimestamp(Function): """Creates an expression that returns the current timestamp diff --git a/tests/system/pipeline_e2e.yaml b/tests/system/pipeline_e2e.yaml index 215f761fd..38595224a 100644 --- a/tests/system/pipeline_e2e.yaml +++ b/tests/system/pipeline_e2e.yaml @@ -1585,6 +1585,120 @@ tests: - fieldReferenceValue: awards.pulitzer name: is_absent name: where + - description: testIfAbsent + pipeline: + - Collection: books + - Select: + - AliasedExpr: + - Expr.if_absent: + - Field: awards.pulitzer + - Constant: false + - "pulitzer_award" + - title + - Where: + - Expr.equal: + - Field: pulitzer_award + - Constant: true + assert_results: + - pulitzer_award: true + title: To Kill a Mockingbird + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + pulitzer_award: + functionValue: + name: if_absent + args: + - fieldReferenceValue: awards.pulitzer + - booleanValue: false + title: + fieldReferenceValue: title + name: select + - args: + - functionValue: + args: + - fieldReferenceValue: pulitzer_award + - booleanValue: true + name: equal + name: where + - description: testIsError + pipeline: + - Collection: books + - Select: + - AliasedExpr: + - Expr.is_error: + - Expr.divide: + - Field: rating + - Constant: "string" + - "is_error_result" + - Limit: 1 + assert_results: + - is_error_result: true + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + is_error_result: + functionValue: + name: is_error + args: + - functionValue: + name: divide + args: + - fieldReferenceValue: rating + - stringValue: "string" + name: select + - args: + - integerValue: '1' + name: limit + - description: testIfError + pipeline: + - Collection: books + - Select: + - AliasedExpr: + - Expr.if_error: + - Expr.divide: + - Field: rating + - Field: genre + - Constant: "An error occurred" + - "if_error_result" + - Limit: 1 + assert_results: + - if_error_result: "An error occurred" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + if_error_result: + functionValue: + name: if_error + args: + - functionValue: + name: divide + args: + - fieldReferenceValue: rating + - fieldReferenceValue: genre + - stringValue: "An error occurred" + name: select + - args: + - integerValue: '1' + name: limit - description: testLogicalMinMax pipeline: - Collection: books diff --git a/tests/unit/v1/test_pipeline_expressions.py b/tests/unit/v1/test_pipeline_expressions.py index 28a5973de..2c3b97259 100644 --- a/tests/unit/v1/test_pipeline_expressions.py +++ b/tests/unit/v1/test_pipeline_expressions.py @@ -839,6 +839,17 @@ def test_is_absent(self): infix_instance = arg1.is_absent() assert infix_instance == instance + + def test_if_absent(self): + arg1 = self._make_arg("Field") + arg2 = self._make_arg("ThenExpr") + instance = Expr.if_absent(arg1, arg2) + assert instance.name == "if_absent" + assert instance.params == [arg1, arg2] + assert repr(instance) == "Field.if_absent(ThenExpr)" + infix_instance = arg1.if_absent(arg2) + assert infix_instance == instance + def test_is_nan(self): arg1 = self._make_arg("Value") instance = Expr.is_nan(arg1) @@ -875,6 +886,25 @@ def test_is_not_null(self): infix_instance = arg1.is_not_null() assert infix_instance == instance + def test_is_error(self): + arg1 = self._make_arg("Value") + instance = Expr.is_error(arg1) + assert instance.name == "is_error" + assert instance.params == [arg1] + assert repr(instance) == "Value.is_error()" + infix_instance = arg1.is_error() + assert infix_instance == instance + + def test_if_error(self): + arg1 = self._make_arg("Value") + arg2 = self._make_arg("ThenExpr") + instance = Expr.if_error(arg1, arg2) + assert instance.name == "if_error" + assert instance.params == [arg1, arg2] + assert repr(instance) == "Value.if_error(ThenExpr)" + infix_instance = arg1.if_error(arg2) + assert infix_instance == instance + def test_not(self): arg1 = self._make_arg("Condition") instance = expr.Not(arg1) From f1690d84df4dd596cc873c0035e4f8a1ac78fe24 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 27 Oct 2025 16:09:21 -0700 Subject: [PATCH 13/36] fixed lint --- .../firestore_v1/pipeline_expressions.py | 12 ++++--- tests/system/test_pipeline_acceptance.py | 32 +++++++++++++++---- tests/unit/v1/test_pipeline_expressions.py | 3 +- 3 files changed, 35 insertions(+), 12 deletions(-) diff --git a/google/cloud/firestore_v1/pipeline_expressions.py b/google/cloud/firestore_v1/pipeline_expressions.py index 5fc425642..93ceca265 100644 --- a/google/cloud/firestore_v1/pipeline_expressions.py +++ b/google/cloud/firestore_v1/pipeline_expressions.py @@ -756,7 +756,10 @@ def concat(self, *others: Expr | CONSTANT_TYPE) -> "Expr": Returns: A new `Expr` representing the concatenated value. """ - return Function("concat", [self] + [self._cast_to_expr_or_convert_to_constant(o) for o in others]) + return Function( + "concat", + [self] + [self._cast_to_expr_or_convert_to_constant(o) for o in others], + ) @expose_as_static def length(self) -> "Expr": @@ -801,7 +804,8 @@ def if_absent(self, default_value: Expr | CONSTANT_TYPE) -> "Expr": A new `Expr` representing the ifAbsent operation. """ return Function( - "if_absent", [self, self._cast_to_expr_or_convert_to_constant(default_value)] + "if_absent", + [self, self._cast_to_expr_or_convert_to_constant(default_value)], ) @expose_as_static @@ -957,7 +961,7 @@ def count_if(self) -> "Expr": @expose_as_static def count_distinct(self) -> "Expr": - """Creates an aggregation that counts the number of distinct values of the + """Creates an aggregation that counts the number of distinct values of the provided field or expression. Example: @@ -2047,4 +2051,4 @@ class CurrentTimestamp(Function): """ def __init__(self): - super().__init__("current_timestamp", [], use_infix_repr=False) \ No newline at end of file + super().__init__("current_timestamp", [], use_infix_repr=False) diff --git a/tests/system/test_pipeline_acceptance.py b/tests/system/test_pipeline_acceptance.py index 3b3e6189d..682fe5e23 100644 --- a/tests/system/test_pipeline_acceptance.py +++ b/tests/system/test_pipeline_acceptance.py @@ -87,7 +87,13 @@ def test_pipeline_expected_errors(test_dict, client): @pytest.mark.parametrize( "test_dict", - [t for t in yaml_loader() if "assert_results" in t or "assert_count" in t or "assert_results_approximate" in t], + [ + t + for t in yaml_loader() + if "assert_results" in t + or "assert_count" in t + or "assert_results_approximate" in t + ], ids=lambda x: f"{x.get('description', '')}", ) def test_pipeline_results(test_dict, client): @@ -105,9 +111,13 @@ def test_pipeline_results(test_dict, client): if expected_results: assert got_results == expected_results if expected_approximate_results: - assert len(got_results) == len(expected_approximate_results), "got unexpected result count" + assert len(got_results) == len( + expected_approximate_results + ), "got unexpected result count" for idx in range(len(got_results)): - assert got_results[idx] == pytest.approx(expected_approximate_results[idx], abs=1e-4) + assert got_results[idx] == pytest.approx( + expected_approximate_results[idx], abs=1e-4 + ) if expected_count is not None: assert len(got_results) == expected_count @@ -134,7 +144,13 @@ async def test_pipeline_expected_errors_async(test_dict, async_client): @pytest.mark.parametrize( "test_dict", - [t for t in yaml_loader() if "assert_results" in t or "assert_count" in t or "assert_results_approximate" in t], + [ + t + for t in yaml_loader() + if "assert_results" in t + or "assert_count" in t + or "assert_results_approximate" in t + ], ids=lambda x: f"{x.get('description', '')}", ) @pytest.mark.asyncio @@ -153,9 +169,13 @@ async def test_pipeline_results_async(test_dict, async_client): if expected_results: assert got_results == expected_results if expected_approximate_results: - assert len(got_results) == len(expected_approximate_results), "got unexpected result count" + assert len(got_results) == len( + expected_approximate_results + ), "got unexpected result count" for idx in range(len(got_results)): - assert got_results[idx] == pytest.approx(expected_approximate_results[idx], abs=1e-4) + assert got_results[idx] == pytest.approx( + expected_approximate_results[idx], abs=1e-4 + ) if expected_count is not None: assert len(got_results) == expected_count diff --git a/tests/unit/v1/test_pipeline_expressions.py b/tests/unit/v1/test_pipeline_expressions.py index 2c3b97259..aec721e7d 100644 --- a/tests/unit/v1/test_pipeline_expressions.py +++ b/tests/unit/v1/test_pipeline_expressions.py @@ -839,7 +839,6 @@ def test_is_absent(self): infix_instance = arg1.is_absent() assert infix_instance == instance - def test_if_absent(self): arg1 = self._make_arg("Field") arg2 = self._make_arg("ThenExpr") @@ -1579,4 +1578,4 @@ def test_maximum(self): assert instance.params == [arg1] assert repr(instance) == "Value.maximum()" infix_instance = arg1.maximum() - assert infix_instance == instance \ No newline at end of file + assert infix_instance == instance From 559ad800aa3d28cc1dd2b144a1687f7ee7fbdf17 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 27 Oct 2025 16:19:39 -0700 Subject: [PATCH 14/36] removed AliasedAggregate in favor of generics --- google/cloud/firestore_v1/_pipeline_stages.py | 3 +- google/cloud/firestore_v1/base_pipeline.py | 10 +++--- .../firestore_v1/pipeline_expressions.py | 31 ------------------- 3 files changed, 6 insertions(+), 38 deletions(-) diff --git a/google/cloud/firestore_v1/_pipeline_stages.py b/google/cloud/firestore_v1/_pipeline_stages.py index c63b748ac..1cad362b1 100644 --- a/google/cloud/firestore_v1/_pipeline_stages.py +++ b/google/cloud/firestore_v1/_pipeline_stages.py @@ -25,7 +25,6 @@ from google.cloud.firestore_v1.pipeline_expressions import ( AggregateFunction, Expr, - AliasedAggregate, AliasedExpr, Field, BooleanExpr, @@ -166,7 +165,7 @@ class Aggregate(Stage): def __init__( self, *args: AliasedExpr[AggregateFunction], - accumulators: Sequence[AliasedAggregate] = (), + accumulators: Sequence[AliasedExpr[AggregateFunction]] = (), groups: Sequence[str | Selectable] = (), ): super().__init__() diff --git a/google/cloud/firestore_v1/base_pipeline.py b/google/cloud/firestore_v1/base_pipeline.py index 01f48ee78..67bffd41e 100644 --- a/google/cloud/firestore_v1/base_pipeline.py +++ b/google/cloud/firestore_v1/base_pipeline.py @@ -23,7 +23,8 @@ from google.cloud.firestore_v1.types.firestore import ExecutePipelineRequest from google.cloud.firestore_v1.pipeline_result import PipelineResult from google.cloud.firestore_v1.pipeline_expressions import ( - AliasedAggregate, + AggregateFunction, + AliasedExpr, Expr, Field, BooleanExpr, @@ -530,7 +531,7 @@ def limit(self, limit: int) -> "_BasePipeline": def aggregate( self, - *accumulators: AliasedAggregate, + *accumulators: AliasedExpr[AggregateFunction], groups: Sequence[str | Selectable] = (), ) -> "_BasePipeline": """ @@ -546,7 +547,6 @@ def aggregate( - **Groups:** Optionally specify fields (by name or `Selectable`) to group the documents by. Aggregations are then performed within each distinct group. If no groups are provided, the aggregation is performed over the entire input. - Example: >>> from google.cloud.firestore_v1.pipeline_expressions import Field >>> pipeline = client.pipeline().collection("books") @@ -568,8 +568,8 @@ def aggregate( Args: - *accumulators: One or more `AliasedAggregate` expressions defining - the aggregations to perform and their output names. + *accumulators: One or more expressions defining the aggregations to perform and their + corresponding output names. groups: An optional sequence of field names (str) or `Selectable` expressions to group by before aggregating. diff --git a/google/cloud/firestore_v1/pipeline_expressions.py b/google/cloud/firestore_v1/pipeline_expressions.py index 93ceca265..38c5fdd4b 100644 --- a/google/cloud/firestore_v1/pipeline_expressions.py +++ b/google/cloud/firestore_v1/pipeline_expressions.py @@ -1695,20 +1695,6 @@ def _to_pb(self): class AggregateFunction(Function): """A base class for aggregation functions that operate across multiple inputs.""" - def as_(self, alias: str) -> "AliasedAggregate": - """Assigns an alias to this expression. - - Aliases are useful for renaming fields in the output of a stage or for giving meaningful - names to calculated values. - - Args: - alias: The alias to assign to this expression. - - Returns: A new AliasedAggregate that wraps this expression and associates it with the - provided alias. - """ - return AliasedAggregate(self, alias) - class Selectable(Expr): """Base class for expressions that can be selected or aliased in projection stages.""" @@ -1766,23 +1752,6 @@ def _to_pb(self): return Value(map_value={"fields": {self.alias: self.expr._to_pb()}}) -class AliasedAggregate: - """Wraps an aggregate with an alias""" - - def __init__(self, expr: AggregateFunction, alias: str): - self.expr = expr - self.alias = alias - - def _to_map(self): - return self.alias, self.expr._to_pb() - - def __repr__(self): - return f"{self.expr}.as_('{self.alias}')" - - def _to_pb(self): - return Value(map_value={"fields": {self.alias: self.expr._to_pb()}}) - - class Field(Selectable): """Represents a reference to a field within a document.""" From 86ad1437a1926b68de2e7645b9a0adc8899397cf Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 27 Oct 2025 19:13:11 -0700 Subject: [PATCH 15/36] improved e2e tests --- google/cloud/firestore_v1/_pipeline_stages.py | 6 +- tests/system/pipeline_e2e.yaml | 369 +++++++++++++++++- 2 files changed, 370 insertions(+), 5 deletions(-) diff --git a/google/cloud/firestore_v1/_pipeline_stages.py b/google/cloud/firestore_v1/_pipeline_stages.py index c63b748ac..1e427d4b4 100644 --- a/google/cloud/firestore_v1/_pipeline_stages.py +++ b/google/cloud/firestore_v1/_pipeline_stages.py @@ -112,11 +112,11 @@ class UnnestOptions: storing the original 0-based index of the element within the array. """ - def __init__(self, index_field: str): - self.index_field = index_field + def __init__(self, index_field: Field | str): + self.index_field = index_field if isinstance(index_field, Field) else Field.of(index_field) def __repr__(self): - return f"{self.__class__.__name__}(index_field={self.index_field!r})" + return f"{self.__class__.__name__}(index_field={self.index_field.path!r})" class Stage(ABC): diff --git a/tests/system/pipeline_e2e.yaml b/tests/system/pipeline_e2e.yaml index 38595224a..d7636eef1 100644 --- a/tests/system/pipeline_e2e.yaml +++ b/tests/system/pipeline_e2e.yaml @@ -2035,23 +2035,75 @@ tests: - description: testUnion pipeline: - Collection: books + - Where: + - Expr.equal: + - Field: genre + - Constant: Romance - Union: - Pipeline: - - Collection: books - assert_count: 20 # Results will be duplicated + - Collection: books + - Where: + - Expr.equal: + - Field: genre + - Constant: Dystopian + - Select: + - title + - Sort: + - Ordering: + - Field: title + - ASCENDING + assert_results: + - title: "1984" + - title: Pride and Prejudice + - title: "The Handmaid's Tale" assert_proto: pipeline: stages: - args: - referenceValue: /books name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: genre + - stringValue: Romance + name: equal + name: where - args: - pipelineValue: stages: - args: - referenceValue: /books name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: genre + - stringValue: Dystopian + name: equal + name: where name: union + - args: + - mapValue: + fields: + title: + fieldReferenceValue: title + name: select + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: title + name: sort + - description: testUnionFullCollection + pipeline: + - Collection: books + - Union: + - Pipeline: + - Collection: books + assert_count: 20 # Results will be duplicated - description: testUnnest pipeline: - Collection: books @@ -2090,6 +2142,57 @@ tests: tags_alias: fieldReferenceValue: tags_alias name: select + - description: testUnnestWithOptions + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: The Hitchhiker's Guide to the Galaxy + - Unnest: + field: tags + alias: tags_alias + options: + UnnestOptions: + - index + - Select: + - tags_alias + - index + assert_results: + - tags_alias: comedy + index: 0 + - tags_alias: space + index: 1 + - tags_alias: adventure + index: 2 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: The Hitchhiker's Guide to the Galaxy + name: equal + name: where + - args: + - fieldReferenceValue: tags + - fieldReferenceValue: tags_alias + name: unnest + options: + index_field: + fieldReferenceValue: index + - args: + - mapValue: + fields: + tags_alias: + fieldReferenceValue: tags_alias + index: + fieldReferenceValue: index + name: select - description: testGreaterThanOrEqual pipeline: - Collection: books @@ -3301,3 +3404,265 @@ tests: - stringValue: ", " name: join name: select + - description: testCollectionGroup + pipeline: + - CollectionGroup: books + - Select: + - title + - Distinct: + - title + - Sort: + - Ordering: + - Field: title + - ASCENDING + assert_results: + - title: "1984" + - title: "Crime and Punishment" + - title: "Dune" + - title: "One Hundred Years of Solitude" + - title: "Pride and Prejudice" + - title: "The Great Gatsby" + - title: "The Handmaid's Tale" + - title: "The Hitchhiker's Guide to the Galaxy" + - title: "The Lord of the Rings" + - title: "To Kill a Mockingbird" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: '' + - stringValue: books + name: collection_group + - args: + - mapValue: + fields: + title: + fieldReferenceValue: title + name: select + - args: + - mapValue: + fields: + title: + fieldReferenceValue: title + name: distinct + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: title + name: sort + + - description: testDistinct + pipeline: + - Collection: books + - Distinct: + - genre + - Sort: + - Ordering: + - Field: genre + - ASCENDING + assert_results: + - genre: Dystopian + - genre: Fantasy + - genre: Magical Realism + - genre: Modernist + - genre: Psychological Thriller + - genre: Romance + - genre: Science Fiction + - genre: Southern Gothic + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + genre: + fieldReferenceValue: genre + name: distinct + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: genre + name: sort + + - description: testDocuments + pipeline: + - Documents: + - /books/book1 + - /books/book10 + - Select: + - title + - Sort: + - Ordering: + - Field: title + - ASCENDING + assert_results: + - title: "Dune" + - title: "The Hitchhiker's Guide to the Galaxy" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books/book1 + - referenceValue: /books/book10 + name: documents + - args: + - mapValue: + fields: + title: + fieldReferenceValue: title + name: select + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: title + name: sort + + - description: testSampleWithLimit + pipeline: + - Collection: books + - Sample: + limit_or_options: 3 + assert_count: 3 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - integerValue: '3' + - stringValue: documents + name: sample + + - description: testSampleWithPercentage + pipeline: + - Collection: books + - Sample: + limit_or_options: + SampleOptions: + value: 1 + mode: "percent" + assert_count: 10 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - doubleValue: 1.0 + - stringValue: percent + name: sample + + - description: testDatabase + pipeline: + - Database + - Select: + - title + - Distinct: + - title + - Aggregate: + - AliasedExpr: + - Count: [] + - count + - Select: + - AliasedExpr: + - Conditional: + - Expr.greater_than_or_equal: + - Field: count + - Constant: 10 + - Constant: True + - Constant: False + - result + assert_results: + - result: True + assert_proto: + pipeline: + stages: + - name: database + - args: + - mapValue: + fields: + title: + fieldReferenceValue: title + name: select + - args: + - mapValue: + fields: + title: + fieldReferenceValue: title + name: distinct + - args: + - mapValue: + fields: + count: + functionValue: + name: count + - mapValue: {} + name: aggregate + - name: select + args: + - mapValue: + fields: + result: + functionValue: + args: + - functionValue: + args: + - fieldReferenceValue: count + - integerValue: '10' + name: greater_than_or_equal + - booleanValue: true + - booleanValue: false + name: conditional + - description: testGenericStage + pipeline: + - GenericStage: + - "collection" + - Value: + reference_value: "/books" + - GenericStage: + - "where" + - Expr.equal: + - Field: title + - Constant: The Hitchhiker's Guide to the Galaxy + - GenericStage: + - "select" + - Value: + map_value: + fields: + author: + field_reference_value: author + assert_results: + - author: Douglas Adams + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: The Hitchhiker's Guide to the Galaxy + name: equal + name: where + - args: + - mapValue: + fields: + author: + fieldReferenceValue: author + name: select \ No newline at end of file From f2697ca75ae37dc5fd38adb51966c8c851c0748d Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 27 Oct 2025 19:26:33 -0700 Subject: [PATCH 16/36] fixed broken stages --- google/cloud/firestore_v1/_pipeline_stages.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/google/cloud/firestore_v1/_pipeline_stages.py b/google/cloud/firestore_v1/_pipeline_stages.py index 1e427d4b4..fcab2b612 100644 --- a/google/cloud/firestore_v1/_pipeline_stages.py +++ b/google/cloud/firestore_v1/_pipeline_stages.py @@ -258,13 +258,7 @@ def of(*documents: "BaseDocumentReference") -> "Documents": return Documents(*doc_paths) def _pb_args(self): - return [ - Value( - array_value={ - "values": [Value(string_value=path) for path in self.paths] - } - ) - ] + return [Value(reference_value=path) for path in self.paths] class FindNearest(Stage): @@ -437,7 +431,7 @@ def _pb_args(self): def _pb_options(self): options = {} if self.options is not None: - options["index_field"] = Value(string_value=self.options.index_field) + options["index_field"] = self.options.index_field._to_pb() return options From d18e1f9a1c90028137e1ea7043ba1bf42152e104 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 27 Oct 2025 19:26:47 -0700 Subject: [PATCH 17/36] added options to generic stage --- google/cloud/firestore_v1/_pipeline_stages.py | 8 +++++++- tests/unit/v1/test_pipeline_stages.py | 12 +++++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/google/cloud/firestore_v1/_pipeline_stages.py b/google/cloud/firestore_v1/_pipeline_stages.py index fcab2b612..f47ec3fc4 100644 --- a/google/cloud/firestore_v1/_pipeline_stages.py +++ b/google/cloud/firestore_v1/_pipeline_stages.py @@ -300,15 +300,21 @@ def _pb_options(self) -> dict[str, Value]: class GenericStage(Stage): """Represents a generic, named stage with parameters.""" - def __init__(self, name: str, *params: Expr | Value): + def __init__(self, name: str, *params: Expr | Value, options: dict[str, Expr | Value] = {}): super().__init__(name) self.params: list[Value] = [ p._to_pb() if isinstance(p, Expr) else p for p in params ] + self.options: dict[str, Value] = { + k: v._to_pb() if isinstance(v, Expr) else v for k, v in options.items() + } def _pb_args(self): return self.params + def _pb_options(self): + return self.options + def __repr__(self): return f"{self.__class__.__name__}(name='{self.name}')" diff --git a/tests/unit/v1/test_pipeline_stages.py b/tests/unit/v1/test_pipeline_stages.py index fadea7e12..e7e117746 100644 --- a/tests/unit/v1/test_pipeline_stages.py +++ b/tests/unit/v1/test_pipeline_stages.py @@ -461,10 +461,20 @@ def _make_one(self, *args, **kwargs): ), ], ) - def test_ctor(self, input_args, expected_params): + def test_ctor_with_params(self, input_args, expected_params): instance = self._make_one(*input_args) assert instance.params == expected_params + def test_ctor_with_options(self): + options = {"index_field": Field.of("index")} + field = Field.of("field") + alias = Field.of("alias") + standard_unnest = stages.Unnest(field, alias, options=stages.UnnestOptions(**options)) + generic_unnest = stages.GenericStage("unnest", field, alias, options=options) + assert standard_unnest._pb_args() == generic_unnest._pb_args() + assert standard_unnest._pb_options() == generic_unnest._pb_options() + assert standard_unnest._to_pb() == generic_unnest._to_pb() + @pytest.mark.parametrize( "input_args,expected", [ From 68b9eff8e2f777dd2a6c5030ed38cea3c77269d8 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 27 Oct 2025 19:51:11 -0700 Subject: [PATCH 18/36] fixed unit tests --- tests/unit/v1/test_aggregation.py | 33 +++++++++++-------------- tests/unit/v1/test_async_aggregation.py | 15 +++++------ tests/unit/v1/test_async_pipeline.py | 4 +-- tests/unit/v1/test_base_query.py | 8 +++--- tests/unit/v1/test_pipeline.py | 4 +-- tests/unit/v1/test_pipeline_stages.py | 11 +++++---- 6 files changed, 34 insertions(+), 41 deletions(-) diff --git a/tests/unit/v1/test_aggregation.py b/tests/unit/v1/test_aggregation.py index 9a20fd386..66239f9ea 100644 --- a/tests/unit/v1/test_aggregation.py +++ b/tests/unit/v1/test_aggregation.py @@ -127,12 +127,12 @@ def test_avg_aggregation_no_alias_to_pb(): "in_alias,expected_alias", [("total", "total"), (None, "field_1")] ) def test_count_aggregation_to_pipeline_expr(in_alias, expected_alias): - from google.cloud.firestore_v1.pipeline_expressions import ExprWithAlias + from google.cloud.firestore_v1.pipeline_expressions import AliasedAggregate from google.cloud.firestore_v1.pipeline_expressions import Count count_aggregation = CountAggregation(alias=in_alias) got = count_aggregation._to_pipeline_expr(iter([1])) - assert isinstance(got, ExprWithAlias) + assert isinstance(got, AliasedAggregate) assert got.alias == expected_alias assert isinstance(got.expr, Count) assert len(got.expr.params) == 0 @@ -143,14 +143,13 @@ def test_count_aggregation_to_pipeline_expr(in_alias, expected_alias): [("total", "path", "total"), (None, "some_ref", "field_1")], ) def test_sum_aggregation_to_pipeline_expr(in_alias, expected_path, expected_alias): - from google.cloud.firestore_v1.pipeline_expressions import ExprWithAlias - from google.cloud.firestore_v1.pipeline_expressions import Sum + from google.cloud.firestore_v1.pipeline_expressions import AliasedAggregate count_aggregation = SumAggregation(expected_path, alias=in_alias) got = count_aggregation._to_pipeline_expr(iter([1])) - assert isinstance(got, ExprWithAlias) + assert isinstance(got, AliasedAggregate) assert got.alias == expected_alias - assert isinstance(got.expr, Sum) + assert got.expr.name == "sum" assert got.expr.params[0].path == expected_path @@ -159,14 +158,13 @@ def test_sum_aggregation_to_pipeline_expr(in_alias, expected_path, expected_alia [("total", "path", "total"), (None, "some_ref", "field_1")], ) def test_avg_aggregation_to_pipeline_expr(in_alias, expected_path, expected_alias): - from google.cloud.firestore_v1.pipeline_expressions import ExprWithAlias - from google.cloud.firestore_v1.pipeline_expressions import Avg + from google.cloud.firestore_v1.pipeline_expressions import AliasedAggregate count_aggregation = AvgAggregation(expected_path, alias=in_alias) got = count_aggregation._to_pipeline_expr(iter([1])) - assert isinstance(got, ExprWithAlias) + assert isinstance(got, AliasedAggregate) assert got.alias == expected_alias - assert isinstance(got.expr, Avg) + assert got.expr.name == "average" assert got.expr.params[0].path == expected_path @@ -1036,7 +1034,6 @@ def test_aggregation_from_query(): def test_aggreation_to_pipeline_sum(field, in_alias, out_alias): from google.cloud.firestore_v1.pipeline import Pipeline from google.cloud.firestore_v1._pipeline_stages import Collection, Aggregate - from google.cloud.firestore_v1.pipeline_expressions import Sum client = make_client() parent = client.collection("dee") @@ -1051,7 +1048,7 @@ def test_aggreation_to_pipeline_sum(field, in_alias, out_alias): aggregate_stage = pipeline.stages[1] assert isinstance(aggregate_stage, Aggregate) assert len(aggregate_stage.accumulators) == 1 - assert isinstance(aggregate_stage.accumulators[0].expr, Sum) + assert aggregate_stage.accumulators[0].expr.name == "sum" expected_field = field if isinstance(field, str) else field.to_api_repr() assert aggregate_stage.accumulators[0].expr.params[0].path == expected_field assert aggregate_stage.accumulators[0].alias == out_alias @@ -1068,7 +1065,6 @@ def test_aggreation_to_pipeline_sum(field, in_alias, out_alias): def test_aggreation_to_pipeline_avg(field, in_alias, out_alias): from google.cloud.firestore_v1.pipeline import Pipeline from google.cloud.firestore_v1._pipeline_stages import Collection, Aggregate - from google.cloud.firestore_v1.pipeline_expressions import Avg client = make_client() parent = client.collection("dee") @@ -1083,7 +1079,7 @@ def test_aggreation_to_pipeline_avg(field, in_alias, out_alias): aggregate_stage = pipeline.stages[1] assert isinstance(aggregate_stage, Aggregate) assert len(aggregate_stage.accumulators) == 1 - assert isinstance(aggregate_stage.accumulators[0].expr, Avg) + assert aggregate_stage.accumulators[0].expr.name == "average" expected_field = field if isinstance(field, str) else field.to_api_repr() assert aggregate_stage.accumulators[0].expr.params[0].path == expected_field assert aggregate_stage.accumulators[0].alias == out_alias @@ -1142,7 +1138,6 @@ def test_aggreation_to_pipeline_count_increment(): def test_aggreation_to_pipeline_complex(): from google.cloud.firestore_v1.pipeline import Pipeline from google.cloud.firestore_v1._pipeline_stages import Collection, Aggregate, Select - from google.cloud.firestore_v1.pipeline_expressions import Sum, Avg, Count client = make_client() query = client.collection("my_col").select(["field_a", "field_b.c"]) @@ -1159,11 +1154,11 @@ def test_aggreation_to_pipeline_complex(): assert isinstance(pipeline.stages[2], Aggregate) aggregate_stage = pipeline.stages[2] assert len(aggregate_stage.accumulators) == 4 - assert isinstance(aggregate_stage.accumulators[0].expr, Sum) + assert aggregate_stage.accumulators[0].expr.name == "sum" assert aggregate_stage.accumulators[0].alias == "alias" - assert isinstance(aggregate_stage.accumulators[1].expr, Count) + assert aggregate_stage.accumulators[1].expr.name == "count" assert aggregate_stage.accumulators[1].alias == "field_1" - assert isinstance(aggregate_stage.accumulators[2].expr, Avg) + assert aggregate_stage.accumulators[2].expr.name == "average" assert aggregate_stage.accumulators[2].alias == "field_2" - assert isinstance(aggregate_stage.accumulators[3].expr, Sum) + assert aggregate_stage.accumulators[3].expr.name == "sum" assert aggregate_stage.accumulators[3].alias == "field_3" diff --git a/tests/unit/v1/test_async_aggregation.py b/tests/unit/v1/test_async_aggregation.py index 701feab5b..f51db482d 100644 --- a/tests/unit/v1/test_async_aggregation.py +++ b/tests/unit/v1/test_async_aggregation.py @@ -710,7 +710,6 @@ async def test_aggregation_query_stream_w_explain_options_analyze_false(): def test_async_aggreation_to_pipeline_sum(field, in_alias, out_alias): from google.cloud.firestore_v1.async_pipeline import AsyncPipeline from google.cloud.firestore_v1._pipeline_stages import Collection, Aggregate - from google.cloud.firestore_v1.pipeline_expressions import Sum client = make_async_client() parent = client.collection("dee") @@ -725,7 +724,7 @@ def test_async_aggreation_to_pipeline_sum(field, in_alias, out_alias): aggregate_stage = pipeline.stages[1] assert isinstance(aggregate_stage, Aggregate) assert len(aggregate_stage.accumulators) == 1 - assert isinstance(aggregate_stage.accumulators[0].expr, Sum) + assert aggregate_stage.accumulators[0].expr.name == "sum" expected_field = field if isinstance(field, str) else field.to_api_repr() assert aggregate_stage.accumulators[0].expr.params[0].path == expected_field assert aggregate_stage.accumulators[0].alias == out_alias @@ -742,7 +741,6 @@ def test_async_aggreation_to_pipeline_sum(field, in_alias, out_alias): def test_async_aggreation_to_pipeline_avg(field, in_alias, out_alias): from google.cloud.firestore_v1.async_pipeline import AsyncPipeline from google.cloud.firestore_v1._pipeline_stages import Collection, Aggregate - from google.cloud.firestore_v1.pipeline_expressions import Avg client = make_async_client() parent = client.collection("dee") @@ -757,7 +755,7 @@ def test_async_aggreation_to_pipeline_avg(field, in_alias, out_alias): aggregate_stage = pipeline.stages[1] assert isinstance(aggregate_stage, Aggregate) assert len(aggregate_stage.accumulators) == 1 - assert isinstance(aggregate_stage.accumulators[0].expr, Avg) + assert aggregate_stage.accumulators[0].expr.name == "average" expected_field = field if isinstance(field, str) else field.to_api_repr() assert aggregate_stage.accumulators[0].expr.params[0].path == expected_field assert aggregate_stage.accumulators[0].alias == out_alias @@ -816,7 +814,6 @@ def test_aggreation_to_pipeline_count_increment(): def test_async_aggreation_to_pipeline_complex(): from google.cloud.firestore_v1.async_pipeline import AsyncPipeline from google.cloud.firestore_v1._pipeline_stages import Collection, Aggregate, Select - from google.cloud.firestore_v1.pipeline_expressions import Sum, Avg, Count client = make_async_client() query = client.collection("my_col").select(["field_a", "field_b.c"]) @@ -833,11 +830,11 @@ def test_async_aggreation_to_pipeline_complex(): assert isinstance(pipeline.stages[2], Aggregate) aggregate_stage = pipeline.stages[2] assert len(aggregate_stage.accumulators) == 4 - assert isinstance(aggregate_stage.accumulators[0].expr, Sum) + assert aggregate_stage.accumulators[0].expr.name == "sum" assert aggregate_stage.accumulators[0].alias == "alias" - assert isinstance(aggregate_stage.accumulators[1].expr, Count) + assert aggregate_stage.accumulators[1].expr.name == "count" assert aggregate_stage.accumulators[1].alias == "field_1" - assert isinstance(aggregate_stage.accumulators[2].expr, Avg) + assert aggregate_stage.accumulators[2].expr.name == "average" assert aggregate_stage.accumulators[2].alias == "field_2" - assert isinstance(aggregate_stage.accumulators[3].expr, Sum) + assert aggregate_stage.accumulators[3].expr.name == "sum" assert aggregate_stage.accumulators[3].alias == "field_3" diff --git a/tests/unit/v1/test_async_pipeline.py b/tests/unit/v1/test_async_pipeline.py index b3ed83337..a11a2951b 100644 --- a/tests/unit/v1/test_async_pipeline.py +++ b/tests/unit/v1/test_async_pipeline.py @@ -386,10 +386,10 @@ async def test_async_pipeline_stream_stream_equivalence_mocked(): ("select", ("name",), stages.Select), ("select", (Field.of("n"),), stages.Select), ("where", (Field.of("n").exists(),), stages.Where), - ("find_nearest", ("name", [0.1], 0), stages.FindNearest), + ("find_nearest", ("name", [0.1], "cosine"), stages.FindNearest), ( "find_nearest", - ("name", [0.1], 0, stages.FindNearestOptions(10)), + ("name", [0.1], "cosine", stages.FindNearestOptions(10)), stages.FindNearest, ), ("sort", (Field.of("n").descending(),), stages.Sort), diff --git a/tests/unit/v1/test_base_query.py b/tests/unit/v1/test_base_query.py index 9bb3e61f8..ae55a424e 100644 --- a/tests/unit/v1/test_base_query.py +++ b/tests/unit/v1/test_base_query.py @@ -2041,7 +2041,7 @@ def test__query_pipeline_composite_filter(): in_filter = FieldFilter("field_a", "==", "value_a") query = client.collection("my_col").where(filter=in_filter) with mock.patch.object( - expr.FilterCondition, "_from_query_filter_pb" + expr.BooleanExpr, "_from_query_filter_pb" ) as convert_mock: pipeline = query.pipeline() convert_mock.assert_called_once_with(in_filter._to_pb(), client) @@ -2080,9 +2080,9 @@ def test__query_pipeline_order_exists_multiple(): assert isinstance(where_stage.condition, expr.And) assert len(where_stage.condition.params) == 2 operands = [p for p in where_stage.condition.params] - assert isinstance(operands[0], expr.Exists) + assert operands[0].name == "exists" assert operands[0].params[0].path == "field_a" - assert isinstance(operands[1], expr.Exists) + assert operands[1].name == "exists" assert operands[1].params[0].path == "field_b" @@ -2098,7 +2098,7 @@ def test__query_pipeline_order_exists_single(): assert len(pipeline_single.stages) == 3 where_stage_single = pipeline_single.stages[1] assert isinstance(where_stage_single, stages.Where) - assert isinstance(where_stage_single.condition, expr.Exists) + assert where_stage_single.condition.name == "exists" assert where_stage_single.condition.params[0].path == "field_c" diff --git a/tests/unit/v1/test_pipeline.py b/tests/unit/v1/test_pipeline.py index f90279e00..161eef1cc 100644 --- a/tests/unit/v1/test_pipeline.py +++ b/tests/unit/v1/test_pipeline.py @@ -363,10 +363,10 @@ def test_pipeline_execute_stream_equivalence_mocked(): ("select", ("name",), stages.Select), ("select", (Field.of("n"),), stages.Select), ("where", (Field.of("n").exists(),), stages.Where), - ("find_nearest", ("name", [0.1], 0), stages.FindNearest), + ("find_nearest", ("name", [0.1], "cosine"), stages.FindNearest), ( "find_nearest", - ("name", [0.1], 0, stages.FindNearestOptions(10)), + ("name", [0.1], "cosine", stages.FindNearestOptions(10)), stages.FindNearest, ), ("sort", (Field.of("n").descending(),), stages.Sort), diff --git a/tests/unit/v1/test_pipeline_stages.py b/tests/unit/v1/test_pipeline_stages.py index e7e117746..d39a21925 100644 --- a/tests/unit/v1/test_pipeline_stages.py +++ b/tests/unit/v1/test_pipeline_stages.py @@ -274,12 +274,12 @@ def test_to_pb(self): instance = self._make_one("/projects/p/databases/d/documents/c/doc1", "/c/doc2") result = instance._to_pb() assert result.name == "documents" - assert len(result.args) == 1 + assert len(result.args) == 2 assert ( - result.args[0].array_value.values[0].string_value + result.args[0].reference_value == "/projects/p/databases/d/documents/c/doc1" ) - assert result.args[0].array_value.values[1].string_value == "/c/doc2" + assert result.args[1].reference_value == "/c/doc2" assert len(result.options) == 0 @@ -719,7 +719,8 @@ def _make_one_options(self, *args, **kwargs): def test_ctor_options(self): index_field_val = "my_index" instance = self._make_one_options(index_field=index_field_val) - assert instance.index_field == index_field_val + assert isinstance(instance.index_field, Field) + assert instance.index_field.path == index_field_val def test_repr(self): instance = self._make_one_options(index_field="my_idx") @@ -791,7 +792,7 @@ def test_to_pb_full(self): assert result.args[1].field_reference_value == alias_str assert len(result.options) == 1 - assert result.options["index_field"].string_value == "item_index" + assert result.options["index_field"].field_reference_value == "item_index" class TestWhere: From 9b2cc6bd32bc77ca824f974d4da70b75cd566370 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 27 Oct 2025 19:52:04 -0700 Subject: [PATCH 19/36] ran blacken --- google/cloud/firestore_v1/_pipeline_stages.py | 8 ++++++-- tests/unit/v1/test_base_query.py | 6 +----- tests/unit/v1/test_pipeline_stages.py | 7 ++++--- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/google/cloud/firestore_v1/_pipeline_stages.py b/google/cloud/firestore_v1/_pipeline_stages.py index f47ec3fc4..62503404e 100644 --- a/google/cloud/firestore_v1/_pipeline_stages.py +++ b/google/cloud/firestore_v1/_pipeline_stages.py @@ -113,7 +113,9 @@ class UnnestOptions: """ def __init__(self, index_field: Field | str): - self.index_field = index_field if isinstance(index_field, Field) else Field.of(index_field) + self.index_field = ( + index_field if isinstance(index_field, Field) else Field.of(index_field) + ) def __repr__(self): return f"{self.__class__.__name__}(index_field={self.index_field.path!r})" @@ -300,7 +302,9 @@ def _pb_options(self) -> dict[str, Value]: class GenericStage(Stage): """Represents a generic, named stage with parameters.""" - def __init__(self, name: str, *params: Expr | Value, options: dict[str, Expr | Value] = {}): + def __init__( + self, name: str, *params: Expr | Value, options: dict[str, Expr | Value] = {} + ): super().__init__(name) self.params: list[Value] = [ p._to_pb() if isinstance(p, Expr) else p for p in params diff --git a/tests/unit/v1/test_base_query.py b/tests/unit/v1/test_base_query.py index ae55a424e..7efa0dacf 100644 --- a/tests/unit/v1/test_base_query.py +++ b/tests/unit/v1/test_base_query.py @@ -2040,9 +2040,7 @@ def test__query_pipeline_composite_filter(): client = make_client() in_filter = FieldFilter("field_a", "==", "value_a") query = client.collection("my_col").where(filter=in_filter) - with mock.patch.object( - expr.BooleanExpr, "_from_query_filter_pb" - ) as convert_mock: + with mock.patch.object(expr.BooleanExpr, "_from_query_filter_pb") as convert_mock: pipeline = query.pipeline() convert_mock.assert_called_once_with(in_filter._to_pb(), client) assert len(pipeline.stages) == 2 @@ -2087,8 +2085,6 @@ def test__query_pipeline_order_exists_multiple(): def test__query_pipeline_order_exists_single(): - from google.cloud.firestore_v1 import pipeline_expressions as expr - client = make_client() query_single = client.collection("my_col").order_by("field_c") pipeline_single = query_single.pipeline() diff --git a/tests/unit/v1/test_pipeline_stages.py b/tests/unit/v1/test_pipeline_stages.py index d39a21925..1d2ff8760 100644 --- a/tests/unit/v1/test_pipeline_stages.py +++ b/tests/unit/v1/test_pipeline_stages.py @@ -276,8 +276,7 @@ def test_to_pb(self): assert result.name == "documents" assert len(result.args) == 2 assert ( - result.args[0].reference_value - == "/projects/p/databases/d/documents/c/doc1" + result.args[0].reference_value == "/projects/p/databases/d/documents/c/doc1" ) assert result.args[1].reference_value == "/c/doc2" assert len(result.options) == 0 @@ -469,7 +468,9 @@ def test_ctor_with_options(self): options = {"index_field": Field.of("index")} field = Field.of("field") alias = Field.of("alias") - standard_unnest = stages.Unnest(field, alias, options=stages.UnnestOptions(**options)) + standard_unnest = stages.Unnest( + field, alias, options=stages.UnnestOptions(**options) + ) generic_unnest = stages.GenericStage("unnest", field, alias, options=options) assert standard_unnest._pb_args() == generic_unnest._pb_args() assert standard_unnest._pb_options() == generic_unnest._pb_options() From 3ebdb13e6f5de9a592cfc7ec0d27ff8371a83cb8 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 27 Oct 2025 20:10:32 -0700 Subject: [PATCH 20/36] broke out aggregates --- tests/system/pipeline_e2e/data.yaml | 142 ++++++++++++++++++ .../system/pipeline_e2e/test_aggregates.yaml | 0 .../tests.yaml} | 142 ------------------ tests/system/test_pipeline_acceptance.py | 17 ++- 4 files changed, 155 insertions(+), 146 deletions(-) create mode 100644 tests/system/pipeline_e2e/data.yaml create mode 100644 tests/system/pipeline_e2e/test_aggregates.yaml rename tests/system/{pipeline_e2e.yaml => pipeline_e2e/tests.yaml} (97%) diff --git a/tests/system/pipeline_e2e/data.yaml b/tests/system/pipeline_e2e/data.yaml new file mode 100644 index 000000000..902f7782d --- /dev/null +++ b/tests/system/pipeline_e2e/data.yaml @@ -0,0 +1,142 @@ +data: + books: + book1: + title: "The Hitchhiker's Guide to the Galaxy" + author: "Douglas Adams" + genre: "Science Fiction" + published: 1979 + rating: 4.2 + tags: + - comedy + - space + - adventure + awards: + hugo: true + nebula: false + book2: + title: "Pride and Prejudice" + author: "Jane Austen" + genre: "Romance" + published: 1813 + rating: 4.5 + tags: + - classic + - social commentary + - love + awards: + none: true + book3: + title: "One Hundred Years of Solitude" + author: "Gabriel García Márquez" + genre: "Magical Realism" + published: 1967 + rating: 4.3 + tags: + - family + - history + - fantasy + awards: + nobel: true + nebula: false + book4: + title: "The Lord of the Rings" + author: "J.R.R. Tolkien" + genre: "Fantasy" + published: 1954 + rating: 4.7 + tags: + - adventure + - magic + - epic + awards: + hugo: false + nebula: false + book5: + title: "The Handmaid's Tale" + author: "Margaret Atwood" + genre: "Dystopian" + published: 1985 + rating: 4.1 + tags: + - feminism + - totalitarianism + - resistance + awards: + arthur c. clarke: true + booker prize: false + book6: + title: "Crime and Punishment" + author: "Fyodor Dostoevsky" + genre: "Psychological Thriller" + published: 1866 + rating: 4.3 + tags: + - philosophy + - crime + - redemption + awards: + none: true + book7: + title: "To Kill a Mockingbird" + author: "Harper Lee" + genre: "Southern Gothic" + published: 1960 + rating: 4.2 + tags: + - racism + - injustice + - coming-of-age + awards: + pulitzer: true + book8: + title: "1984" + author: "George Orwell" + genre: "Dystopian" + published: 1949 + rating: 4.2 + tags: + - surveillance + - totalitarianism + - propaganda + awards: + prometheus: true + book9: + title: "The Great Gatsby" + author: "F. Scott Fitzgerald" + genre: "Modernist" + published: 1925 + rating: 4.0 + tags: + - wealth + - american dream + - love + awards: + none: true + book10: + title: "Dune" + author: "Frank Herbert" + genre: "Science Fiction" + published: 1965 + rating: 4.6 + tags: + - politics + - desert + - ecology + awards: + hugo: true + nebula: true + timestamps: + ts1: + time: "1993-04-28T12:01:00.654321+00:00" + micros: 735998460654321 + millis: 735998460654 + seconds: 735998460 + vectors: + vec1: + embedding: [1.0, 2.0, 3.0] + vec2: + embedding: [4.0, 5.0, 6.0, 7.0] + vec3: + embedding: [5.0, 6.0, 7.0] + vec4: + embedding: [1.0, 2.0, 4.0] \ No newline at end of file diff --git a/tests/system/pipeline_e2e/test_aggregates.yaml b/tests/system/pipeline_e2e/test_aggregates.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/tests/system/pipeline_e2e.yaml b/tests/system/pipeline_e2e/tests.yaml similarity index 97% rename from tests/system/pipeline_e2e.yaml rename to tests/system/pipeline_e2e/tests.yaml index d7636eef1..b595dde85 100644 --- a/tests/system/pipeline_e2e.yaml +++ b/tests/system/pipeline_e2e/tests.yaml @@ -1,145 +1,3 @@ -data: - books: - book1: - title: "The Hitchhiker's Guide to the Galaxy" - author: "Douglas Adams" - genre: "Science Fiction" - published: 1979 - rating: 4.2 - tags: - - comedy - - space - - adventure - awards: - hugo: true - nebula: false - book2: - title: "Pride and Prejudice" - author: "Jane Austen" - genre: "Romance" - published: 1813 - rating: 4.5 - tags: - - classic - - social commentary - - love - awards: - none: true - book3: - title: "One Hundred Years of Solitude" - author: "Gabriel García Márquez" - genre: "Magical Realism" - published: 1967 - rating: 4.3 - tags: - - family - - history - - fantasy - awards: - nobel: true - nebula: false - book4: - title: "The Lord of the Rings" - author: "J.R.R. Tolkien" - genre: "Fantasy" - published: 1954 - rating: 4.7 - tags: - - adventure - - magic - - epic - awards: - hugo: false - nebula: false - book5: - title: "The Handmaid's Tale" - author: "Margaret Atwood" - genre: "Dystopian" - published: 1985 - rating: 4.1 - tags: - - feminism - - totalitarianism - - resistance - awards: - arthur c. clarke: true - booker prize: false - book6: - title: "Crime and Punishment" - author: "Fyodor Dostoevsky" - genre: "Psychological Thriller" - published: 1866 - rating: 4.3 - tags: - - philosophy - - crime - - redemption - awards: - none: true - book7: - title: "To Kill a Mockingbird" - author: "Harper Lee" - genre: "Southern Gothic" - published: 1960 - rating: 4.2 - tags: - - racism - - injustice - - coming-of-age - awards: - pulitzer: true - book8: - title: "1984" - author: "George Orwell" - genre: "Dystopian" - published: 1949 - rating: 4.2 - tags: - - surveillance - - totalitarianism - - propaganda - awards: - prometheus: true - book9: - title: "The Great Gatsby" - author: "F. Scott Fitzgerald" - genre: "Modernist" - published: 1925 - rating: 4.0 - tags: - - wealth - - american dream - - love - awards: - none: true - book10: - title: "Dune" - author: "Frank Herbert" - genre: "Science Fiction" - published: 1965 - rating: 4.6 - tags: - - politics - - desert - - ecology - awards: - hugo: true - nebula: true - timestamps: - ts1: - time: "1993-04-28T12:01:00.654321+00:00" - micros: 735998460654321 - millis: 735998460654 - seconds: 735998460 - vectors: - vec1: - embedding: [1.0, 2.0, 3.0] - vec2: - embedding: [4.0, 5.0, 6.0, 7.0] - vec3: - embedding: [5.0, 6.0, 7.0] - vec4: - embedding: [1.0, 2.0, 4.0] tests: - description: "testAggregates - count" pipeline: diff --git a/tests/system/test_pipeline_acceptance.py b/tests/system/test_pipeline_acceptance.py index 682fe5e23..996ebfcb4 100644 --- a/tests/system/test_pipeline_acceptance.py +++ b/tests/system/test_pipeline_acceptance.py @@ -40,13 +40,22 @@ test_dir_name = os.path.dirname(__file__) -def yaml_loader(field="tests", file_name="pipeline_e2e.yaml"): +def yaml_loader(field="tests", dir_name="pipeline_e2e"): """ Helper to load test cases or data from yaml file """ - with open(f"{test_dir_name}/{file_name}") as f: - test_cases = yaml.safe_load(f) - return test_cases[field] + combined_yaml = None + for file_name in os.listdir(f"{test_dir_name}/{dir_name}"): + with open(f"{test_dir_name}/{dir_name}/{file_name}") as f: + new_yaml = yaml.safe_load(f) + extracted = new_yaml.get(field, None) if new_yaml else None + if not combined_yaml: + combined_yaml = extracted + elif isinstance(combined_yaml, dict) and extracted: + combined_yaml.update(extracted) + elif isinstance(combined_yaml, list) and extracted: + combined_yaml.extend(extracted) + return combined_yaml @pytest.mark.parametrize( From a951789d963c5d42cf294eb98c199ce59afb94dd Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 27 Oct 2025 21:33:49 -0700 Subject: [PATCH 21/36] broke up test file --- .../{test_aggregates.yaml => aggregates.yaml} | 0 tests/system/pipeline_e2e/array.yaml | 388 ++ tests/system/pipeline_e2e/date_and_time.yaml | 103 + tests/system/pipeline_e2e/logical.yaml | 673 ++++ tests/system/pipeline_e2e/map.yaml | 269 ++ tests/system/pipeline_e2e/string.yaml | 654 ++++ tests/system/pipeline_e2e/tests.yaml | 3409 +++-------------- tests/system/pipeline_e2e/vector.yaml | 160 + 8 files changed, 2775 insertions(+), 2881 deletions(-) rename tests/system/pipeline_e2e/{test_aggregates.yaml => aggregates.yaml} (100%) create mode 100644 tests/system/pipeline_e2e/array.yaml create mode 100644 tests/system/pipeline_e2e/date_and_time.yaml create mode 100644 tests/system/pipeline_e2e/logical.yaml create mode 100644 tests/system/pipeline_e2e/map.yaml create mode 100644 tests/system/pipeline_e2e/string.yaml create mode 100644 tests/system/pipeline_e2e/vector.yaml diff --git a/tests/system/pipeline_e2e/test_aggregates.yaml b/tests/system/pipeline_e2e/aggregates.yaml similarity index 100% rename from tests/system/pipeline_e2e/test_aggregates.yaml rename to tests/system/pipeline_e2e/aggregates.yaml diff --git a/tests/system/pipeline_e2e/array.yaml b/tests/system/pipeline_e2e/array.yaml new file mode 100644 index 000000000..6e99f38ef --- /dev/null +++ b/tests/system/pipeline_e2e/array.yaml @@ -0,0 +1,388 @@ +tests: + - description: testArrayContains + pipeline: + - Collection: books + - Where: + - Expr.array_contains: + - Field: tags + - Constant: comedy + assert_results: + - title: The Hitchhiker's Guide to the Galaxy + author: Douglas Adams + awards: + hugo: true + nebula: false + genre: Science Fiction + published: 1979 + rating: 4.2 + tags: ["comedy", "space", "adventure"] + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: tags + - stringValue: comedy + name: array_contains + name: where + - description: testArrayContainsAny + pipeline: + - Collection: books + - Where: + - Expr.array_contains_any: + - Field: tags + - - Constant: comedy + - Constant: classic + - Select: + - title + - Sort: + - Ordering: + - Field: title + - ASCENDING + assert_results: + - title: Pride and Prejudice + - title: The Hitchhiker's Guide to the Galaxy + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: tags + - functionValue: + args: + - stringValue: comedy + - stringValue: classic + name: array + name: array_contains_any + name: where + - args: + - mapValue: + fields: + title: + fieldReferenceValue: title + name: select + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: title + name: sort + - description: testArrayContainsAll + pipeline: + - Collection: books + - Where: + - Expr.array_contains_all: + - Field: tags + - - Constant: adventure + - Constant: magic + - Select: + - title + assert_results: + - title: The Lord of the Rings + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: tags + - functionValue: + args: + - stringValue: adventure + - stringValue: magic + name: array + name: array_contains_all + name: where + - args: + - mapValue: + fields: + title: + fieldReferenceValue: title + name: select + - description: testArrayLength + pipeline: + - Collection: books + - Select: + - AliasedExpr: + - Expr.array_length: + - Field: tags + - "tagsCount" + - Where: + - Expr.equal: + - Field: tagsCount + - Constant: 3 + assert_results: # All documents have 3 tags + - tagsCount: 3 + - tagsCount: 3 + - tagsCount: 3 + - tagsCount: 3 + - tagsCount: 3 + - tagsCount: 3 + - tagsCount: 3 + - tagsCount: 3 + - tagsCount: 3 + - tagsCount: 3 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + tagsCount: + functionValue: + args: + - fieldReferenceValue: tags + name: array_length + name: select + - args: + - functionValue: + args: + - fieldReferenceValue: tagsCount + - integerValue: '3' + name: equal + name: where + - description: testArrayReverse + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: "The Hitchhiker's Guide to the Galaxy" + - Select: + - AliasedExpr: + - Expr.array_reverse: + - Field: tags + - "reversedTags" + assert_results: + - reversedTags: + - adventure + - space + - comedy + - description: testArrayConcat + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: "The Hitchhiker's Guide to the Galaxy" + - Select: + - AliasedExpr: + - Expr.array_concat: + - Field: tags + - Constant: ["new_tag", "another_tag"] + - "concatenatedTags" + assert_results: + - concatenatedTags: + - comedy + - space + - adventure + - new_tag + - another_tag + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: "The Hitchhiker's Guide to the Galaxy" + name: equal + name: where + - args: + - mapValue: + fields: + concatenatedTags: + functionValue: + args: + - fieldReferenceValue: tags + - functionValue: + args: + - stringValue: "new_tag" + - stringValue: "another_tag" + name: array + name: array_concat + name: select + - description: testArrayConcatMultiple + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: "Dune" + - Select: + - AliasedExpr: + - Expr.array_concat: + - Field: tags + - Constant: ["sci-fi"] + - Constant: ["classic", "epic"] + - "concatenatedTags" + assert_results: + - concatenatedTags: + - politics + - desert + - ecology + - sci-fi + - classic + - epic + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: "Dune" + name: equal + name: where + - args: + - mapValue: + fields: + concatenatedTags: + functionValue: + args: + - fieldReferenceValue: tags + - functionValue: + args: + - stringValue: "sci-fi" + name: array + - functionValue: + args: + - stringValue: "classic" + - stringValue: "epic" + name: array + name: array_concat + name: select + - description: testArrayContainsAnyWithField + pipeline: + - Collection: books + - AddFields: + - AliasedExpr: + - Expr.array_concat: + - Field: tags + - Array: ["Dystopian"] + - "new_tags" + - Where: + - Expr.array_contains_any: + - Field: new_tags + - - Constant: non_existent_tag + - Field: genre + - Select: + - title + - genre + - Sort: + - Ordering: + - Field: title + - ASCENDING + assert_results: + - title: "1984" + genre: "Dystopian" + - title: "The Handmaid's Tale" + genre: "Dystopian" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + new_tags: + functionValue: + args: + - fieldReferenceValue: tags + - functionValue: + args: + - stringValue: "Dystopian" + name: array + name: array_concat + name: add_fields + - args: + - functionValue: + args: + - fieldReferenceValue: new_tags + - functionValue: + args: + - stringValue: "non_existent_tag" + - fieldReferenceValue: genre + name: array + name: array_contains_any + name: where + - args: + - mapValue: + fields: + title: + fieldReferenceValue: title + genre: + fieldReferenceValue: genre + name: select + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: title + name: sort + - description: testArrayConcatLiterals + pipeline: + - Collection: books + - Limit: 1 + - Select: + - AliasedExpr: + - Expr.array_concat: + - Array: [1, 2, 3] + - Array: [4, 5] + - "concatenated" + assert_results: + - concatenated: [1, 2, 3, 4, 5] + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - integerValue: '1' + name: limit + - args: + - mapValue: + fields: + concatenated: + functionValue: + args: + - functionValue: + args: + - integerValue: '1' + - integerValue: '2' + - integerValue: '3' + name: array + - functionValue: + args: + - integerValue: '4' + - integerValue: '5' + name: array + name: array_concat + name: select diff --git a/tests/system/pipeline_e2e/date_and_time.yaml b/tests/system/pipeline_e2e/date_and_time.yaml new file mode 100644 index 000000000..bbb5f34fe --- /dev/null +++ b/tests/system/pipeline_e2e/date_and_time.yaml @@ -0,0 +1,103 @@ +tests: + - description: testCurrentTimestamp + pipeline: + - Collection: books + - Limit: 1 + - Select: + - AliasedExpr: + - And: + - Expr.greater_than_or_equal: + - CurrentTimestamp: [] + - Expr.unix_seconds_to_timestamp: + - Constant: 1735689600 # 2025-01-01 + - Expr.less_than: + - CurrentTimestamp: [] + - Expr.unix_seconds_to_timestamp: + - Constant: 4892438400 # 2125-01-01 + - "is_between_2025_and_2125" + assert_results: + - is_between_2025_and_2125: true + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - integerValue: '1' + name: limit + - args: + - mapValue: + fields: + is_between_2025_and_2125: + functionValue: + name: and + args: + - functionValue: + name: greater_than_or_equal + args: + - functionValue: + name: current_timestamp + - functionValue: + name: unix_seconds_to_timestamp + args: + - integerValue: '1735689600' + - functionValue: + name: less_than + args: + - functionValue: + name: current_timestamp + - functionValue: + name: unix_seconds_to_timestamp + args: + - integerValue: '4892438400' + name: select + - description: testTimestampFunctions + pipeline: + - Collection: timestamps + - Select: + - AliasedExpr: + - Expr.timestamp_to_unix_micros: + - Field: time + - "micros" + - AliasedExpr: + - Expr.timestamp_to_unix_millis: + - Field: time + - "millis" + - AliasedExpr: + - Expr.timestamp_to_unix_seconds: + - Field: time + - "seconds" + - AliasedExpr: + - Expr.unix_micros_to_timestamp: + - Field: micros + - "from_micros" + - AliasedExpr: + - Expr.unix_millis_to_timestamp: + - Field: millis + - "from_millis" + - AliasedExpr: + - Expr.unix_seconds_to_timestamp: + - Field: seconds + - "from_seconds" + - AliasedExpr: + - Expr.timestamp_add: + - Field: time + - Constant: "day" + - Constant: 1 + - "plus_day" + - AliasedExpr: + - Expr.timestamp_subtract: + - Field: time + - Constant: "hour" + - Constant: 1 + - "minus_hour" + assert_results: + - micros: 735998460654321 + millis: 735998460654 + seconds: 735998460 + from_micros: "1993-04-28T12:01:00.654321+00:00" + from_millis: "1993-04-28T12:01:00.654000+00:00" + from_seconds: "1993-04-28T12:01:00.000000+00:00" + plus_day: "1993-04-29T12:01:00.654321+00:00" + minus_hour: "1993-04-28T11:01:00.654321+00:00" diff --git a/tests/system/pipeline_e2e/logical.yaml b/tests/system/pipeline_e2e/logical.yaml new file mode 100644 index 000000000..203be290d --- /dev/null +++ b/tests/system/pipeline_e2e/logical.yaml @@ -0,0 +1,673 @@ +tests: + - description: whereByMultipleConditions + pipeline: + - Collection: books + - Where: + - And: + - Expr.greater_than: + - Field: rating + - Constant: 4.5 + - Expr.equal: + - Field: genre + - Constant: Science Fiction + assert_results: + - title: Dune + author: Frank Herbert + genre: Science Fiction + published: 1965 + rating: 4.6 + tags: + - politics + - desert + - ecology + awards: + hugo: true + nebula: true + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - functionValue: + args: + - fieldReferenceValue: rating + - doubleValue: 4.5 + name: greater_than + - functionValue: + args: + - fieldReferenceValue: genre + - stringValue: Science Fiction + name: equal + name: and + name: where + - description: whereByOrCondition + pipeline: + - Collection: books + - Where: + - Or: + - Expr.equal: + - Field: genre + - Constant: Romance + - Expr.equal: + - Field: genre + - Constant: Dystopian + - Select: + - title + - Sort: + - Ordering: + - Field: title + - ASCENDING + assert_results: + - title: "1984" + - title: Pride and Prejudice + - title: The Handmaid's Tale + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - functionValue: + args: + - fieldReferenceValue: genre + - stringValue: Romance + name: equal + - functionValue: + args: + - fieldReferenceValue: genre + - stringValue: Dystopian + name: equal + name: or + name: where + - args: + - mapValue: + fields: + title: + fieldReferenceValue: title + name: select + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: title + name: sort + - description: testComparisonOperators + pipeline: + - Collection: books + - Where: + - And: + - Expr.greater_than: + - Field: rating + - Constant: 4.2 + - Expr.less_than_or_equal: + - Field: rating + - Constant: 4.5 + - Expr.not_equal: + - Field: genre + - Constant: Science Fiction + - Select: + - rating + - title + - Sort: + - Ordering: + - title + - ASCENDING + assert_results: + - rating: 4.3 + title: Crime and Punishment + - rating: 4.3 + title: One Hundred Years of Solitude + - rating: 4.5 + title: Pride and Prejudice + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - functionValue: + args: + - fieldReferenceValue: rating + - doubleValue: 4.2 + name: greater_than + - functionValue: + args: + - fieldReferenceValue: rating + - doubleValue: 4.5 + name: less_than_or_equal + - functionValue: + args: + - fieldReferenceValue: genre + - stringValue: Science Fiction + name: not_equal + name: and + name: where + - args: + - mapValue: + fields: + rating: + fieldReferenceValue: rating + title: + fieldReferenceValue: title + name: select + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: title + name: sort + - description: testLogicalOperators + pipeline: + - Collection: books + - Where: + - Or: + - And: + - Expr.greater_than: + - Field: rating + - Constant: 4.5 + - Expr.equal: + - Field: genre + - Constant: Science Fiction + - Expr.less_than: + - Field: published + - Constant: 1900 + - Select: + - title + - Sort: + - Ordering: + - Field: title + - ASCENDING + assert_results: + - title: Crime and Punishment + - title: Dune + - title: Pride and Prejudice + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - functionValue: + args: + - functionValue: + args: + - fieldReferenceValue: rating + - doubleValue: 4.5 + name: greater_than + - functionValue: + args: + - fieldReferenceValue: genre + - stringValue: Science Fiction + name: equal + name: and + - functionValue: + args: + - fieldReferenceValue: published + - integerValue: '1900' + name: less_than + name: or + name: where + - args: + - mapValue: + fields: + title: + fieldReferenceValue: title + name: select + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: title + name: sort + - description: testChecks + pipeline: + - Collection: books + - Where: + - Not: + - Expr.is_nan: + - Field: rating + - Select: + - AliasedExpr: + - Not: + - Expr.is_nan: + - Field: rating + - "ratingIsNotNaN" + - Limit: 1 + assert_results: + - ratingIsNotNaN: true + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - functionValue: + args: + - fieldReferenceValue: rating + name: is_nan + name: not + name: where + - args: + - mapValue: + fields: + ratingIsNotNaN: + functionValue: + args: + - functionValue: + args: + - fieldReferenceValue: rating + name: is_nan + name: not + name: select + - args: + - integerValue: '1' + name: limit + - description: testIsNotNull + pipeline: + - Collection: books + - Where: + - Expr.is_not_null: + - Field: rating + assert_count: 10 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: rating + name: is_not_null + name: where + - description: testIsNotNaN + pipeline: + - Collection: books + - Where: + - Expr.is_not_nan: + - Field: rating + assert_count: 10 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: rating + name: is_not_nan + name: where + - description: testIsAbsent + pipeline: + - Collection: books + - Where: + - Expr.is_absent: + - Field: awards.pulitzer + assert_count: 9 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: awards.pulitzer + name: is_absent + name: where + - description: testIfAbsent + pipeline: + - Collection: books + - Select: + - AliasedExpr: + - Expr.if_absent: + - Field: awards.pulitzer + - Constant: false + - "pulitzer_award" + - title + - Where: + - Expr.equal: + - Field: pulitzer_award + - Constant: true + assert_results: + - pulitzer_award: true + title: To Kill a Mockingbird + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + pulitzer_award: + functionValue: + name: if_absent + args: + - fieldReferenceValue: awards.pulitzer + - booleanValue: false + title: + fieldReferenceValue: title + name: select + - args: + - functionValue: + args: + - fieldReferenceValue: pulitzer_award + - booleanValue: true + name: equal + name: where + - description: testIsError + pipeline: + - Collection: books + - Select: + - AliasedExpr: + - Expr.is_error: + - Expr.divide: + - Field: rating + - Constant: "string" + - "is_error_result" + - Limit: 1 + assert_results: + - is_error_result: true + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + is_error_result: + functionValue: + name: is_error + args: + - functionValue: + name: divide + args: + - fieldReferenceValue: rating + - stringValue: "string" + name: select + - args: + - integerValue: '1' + name: limit + - description: testIfError + pipeline: + - Collection: books + - Select: + - AliasedExpr: + - Expr.if_error: + - Expr.divide: + - Field: rating + - Field: genre + - Constant: "An error occurred" + - "if_error_result" + - Limit: 1 + assert_results: + - if_error_result: "An error occurred" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + if_error_result: + functionValue: + name: if_error + args: + - functionValue: + name: divide + args: + - fieldReferenceValue: rating + - fieldReferenceValue: genre + - stringValue: "An error occurred" + name: select + - args: + - integerValue: '1' + name: limit + - description: testLogicalMinMax + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: author + - Constant: Douglas Adams + - Select: + - AliasedExpr: + - Expr.logical_maximum: + - Field: rating + - Constant: 4.5 + - "max_rating" + - AliasedExpr: + - Expr.logical_minimum: + - Field: published + - Constant: 1900 + - "min_published" + assert_results: + - max_rating: 4.5 + min_published: 1900 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: author + - stringValue: Douglas Adams + name: equal + name: where + - args: + - mapValue: + fields: + min_published: + functionValue: + args: + - fieldReferenceValue: published + - integerValue: '1900' + name: minimum + max_rating: + functionValue: + args: + - fieldReferenceValue: rating + - doubleValue: 4.5 + name: maximum + name: select + - description: testGreaterThanOrEqual + pipeline: + - Collection: books + - Where: + - Expr.greater_than_or_equal: + - Field: rating + - Constant: 4.6 + - Select: + - title + - rating + - Sort: + - Ordering: + - Field: rating + - ASCENDING + assert_results: + - title: Dune + rating: 4.6 + - title: The Lord of the Rings + rating: 4.7 + - description: testInAndNotIn + pipeline: + - Collection: books + - Where: + - And: + - Expr.equal_any: + - Field: genre + - - Constant: Romance + - Constant: Dystopian + - Expr.not_equal_any: + - Field: author + - - Constant: "George Orwell" + assert_results: + - title: "Pride and Prejudice" + author: "Jane Austen" + genre: "Romance" + published: 1813 + rating: 4.5 + tags: + - classic + - social commentary + - love + awards: + none: true + - title: "The Handmaid's Tale" + author: "Margaret Atwood" + genre: "Dystopian" + published: 1985 + rating: 4.1 + tags: + - feminism + - totalitarianism + - resistance + awards: + "arthur c. clarke": true + "booker prize": false + - description: testExists + pipeline: + - Collection: books + - Where: + - And: + - Expr.exists: + - Field: awards.pulitzer + - Expr.equal: + - Field: awards.pulitzer + - Constant: true + - Select: + - title + assert_results: + - title: To Kill a Mockingbird + - description: testXor + pipeline: + - Collection: books + - Where: + - Xor: + - - Expr.equal: + - Field: genre + - Constant: Romance + - Expr.greater_than: + - Field: published + - Constant: 1980 + - Select: + - title + - genre + - published + - Sort: + - Ordering: + - Field: title + - ASCENDING + assert_results: + - title: "Pride and Prejudice" + genre: "Romance" + published: 1813 + - title: "The Handmaid's Tale" + genre: "Dystopian" + published: 1985 + - description: testConditional + pipeline: + - Collection: books + - Select: + - title + - AliasedExpr: + - Conditional: + - Expr.greater_than: + - Field: published + - Constant: 1950 + - Constant: "Modern" + - Constant: "Classic" + - "era" + - Sort: + - Ordering: + - Field: title + - ASCENDING + - Limit: 4 + assert_results: + - title: "1984" + era: "Classic" + - title: "Crime and Punishment" + era: "Classic" + - title: "Dune" + era: "Modern" + - title: "One Hundred Years of Solitude" + era: "Modern" + - description: testFieldToFieldComparison + pipeline: + - Collection: books + - Where: + - Expr.greater_than: + - Field: published + - Field: rating + - Select: + - title + assert_count: 10 # All books were published after year 4.7 + - description: testExistsNegative + pipeline: + - Collection: books + - Where: + - Expr.exists: + - Field: non_existent_field + assert_count: 0 + - description: testConditionalWithFields + pipeline: + - Collection: books + - Where: + - Expr.equal_any: + - Field: title + - - Constant: "Dune" + - Constant: "1984" + - Select: + - title + - AliasedExpr: + - Conditional: + - Expr.greater_than: + - Field: published + - Constant: 1950 + - Field: author + - Field: genre + - "conditional_field" + - Sort: + - Ordering: + - Field: title + - ASCENDING + assert_results: + - title: "1984" + conditional_field: "Dystopian" + - title: "Dune" + conditional_field: "Frank Herbert" diff --git a/tests/system/pipeline_e2e/map.yaml b/tests/system/pipeline_e2e/map.yaml new file mode 100644 index 000000000..638fe0798 --- /dev/null +++ b/tests/system/pipeline_e2e/map.yaml @@ -0,0 +1,269 @@ +tests: + - description: testMapGet + pipeline: + - Collection: books + - Sort: + - Ordering: + - Field: published + - DESCENDING + - Select: + - AliasedExpr: + - Expr.map_get: + - Field: awards + - hugo + - "hugoAward" + - Field: title + - Where: + - Expr.equal: + - Field: hugoAward + - Constant: true + assert_results: + - hugoAward: true + title: The Hitchhiker's Guide to the Galaxy + - hugoAward: true + title: Dune + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + direction: + stringValue: descending + expression: + fieldReferenceValue: published + name: sort + - args: + - mapValue: + fields: + hugoAward: + functionValue: + args: + - fieldReferenceValue: awards + - stringValue: hugo + name: map_get + title: + fieldReferenceValue: title + name: select + - args: + - functionValue: + args: + - fieldReferenceValue: hugoAward + - booleanValue: true + name: equal + name: where + - description: testMapGetWithField + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: "Dune" + - AddFields: + - AliasedExpr: + - Constant: "hugo" + - "award_name" + - Select: + - AliasedExpr: + - Expr.map_get: + - Field: awards + - Field: award_name + - "hugoAward" + - Field: title + assert_results: + - hugoAward: true + title: Dune + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: "Dune" + name: equal + name: where + - args: + - mapValue: + fields: + award_name: + stringValue: "hugo" + name: add_fields + - args: + - mapValue: + fields: + hugoAward: + functionValue: + name: map_get + args: + - fieldReferenceValue: awards + - fieldReferenceValue: award_name + title: + fieldReferenceValue: title + name: select + - description: testMapRemove + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: "Dune" + - Select: + - AliasedExpr: + - Expr.map_remove: + - Field: awards + - "nebula" + - "awards_removed" + assert_results: + - awards_removed: + hugo: true + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: "Dune" + name: equal + name: where + - args: + - mapValue: + fields: + awards_removed: + functionValue: + name: map_remove + args: + - fieldReferenceValue: awards + - stringValue: "nebula" + name: select + - description: testMapMerge + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: "Dune" + - Select: + - AliasedExpr: + - Expr.map_merge: + - Field: awards + - Map: + elements: {"new_award": true, "hugo": false} + - Map: + elements: {"another_award": "yes"} + - "awards_merged" + assert_results: + - awards_merged: + hugo: false + nebula: true + new_award: true + another_award: "yes" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: "Dune" + name: equal + name: where + - args: + - mapValue: + fields: + awards_merged: + functionValue: + name: map_merge + args: + - fieldReferenceValue: awards + - functionValue: + name: map + args: + - stringValue: "new_award" + - booleanValue: true + - stringValue: "hugo" + - booleanValue: false + - functionValue: + name: map + args: + - stringValue: "another_award" + - stringValue: "yes" + name: select + - description: testNestedFields + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: awards.hugo + - Constant: true + - Sort: + - Ordering: + - Field: title + - DESCENDING + - Select: + - title + - Field: awards.hugo + assert_results: + - title: The Hitchhiker's Guide to the Galaxy + awards.hugo: true + - title: Dune + awards.hugo: true + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: awards.hugo + - booleanValue: true + name: equal + name: where + - args: + - mapValue: + fields: + direction: + stringValue: descending + expression: + fieldReferenceValue: title + name: sort + - args: + - mapValue: + fields: + awards.hugo: + fieldReferenceValue: awards.hugo + title: + fieldReferenceValue: title + name: select + - description: testMapMergeLiterals + pipeline: + - Collection: books + - Limit: 1 + - Select: + - AliasedExpr: + - Expr.map_merge: + - Map: + elements: {"a": "orig", "b": "orig"} + - Map: + elements: {"b": "new", "c": "new"} + - "merged" + assert_results: + - merged: + a: "orig" + b: "new" + c: "new" diff --git a/tests/system/pipeline_e2e/string.yaml b/tests/system/pipeline_e2e/string.yaml new file mode 100644 index 000000000..783a0c919 --- /dev/null +++ b/tests/system/pipeline_e2e/string.yaml @@ -0,0 +1,654 @@ +tests: + - description: testStringConcat + pipeline: + - Collection: books + - Sort: + - Ordering: + - Field: author + - ASCENDING + - Select: + - AliasedExpr: + - Expr.string_concat: + - Field: author + - Constant: " - " + - Field: title + - "bookInfo" + - Limit: 1 + assert_results: + - bookInfo: Douglas Adams - The Hitchhiker's Guide to the Galaxy + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: author + name: sort + - args: + - mapValue: + fields: + bookInfo: + functionValue: + args: + - fieldReferenceValue: author + - stringValue: ' - ' + - fieldReferenceValue: title + name: string_concat + name: select + - args: + - integerValue: '1' + name: limit + - description: testStartsWith + pipeline: + - Collection: books + - Where: + - Expr.starts_with: + - Field: title + - Constant: The + - Select: + - title + - Sort: + - Ordering: + - Field: title + - ASCENDING + assert_results: + - title: The Great Gatsby + - title: The Handmaid's Tale + - title: The Hitchhiker's Guide to the Galaxy + - title: The Lord of the Rings + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: The + name: starts_with + name: where + - args: + - mapValue: + fields: + title: + fieldReferenceValue: title + name: select + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: title + name: sort + - description: testEndsWith + pipeline: + - Collection: books + - Where: + - Expr.ends_with: + - Field: title + - Constant: y + - Select: + - title + - Sort: + - Ordering: + - Field: title + - DESCENDING + assert_results: + - title: The Hitchhiker's Guide to the Galaxy + - title: The Great Gatsby + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: y + name: ends_with + name: where + - args: + - mapValue: + fields: + title: + fieldReferenceValue: title + name: select + - args: + - mapValue: + fields: + direction: + stringValue: descending + expression: + fieldReferenceValue: title + name: sort + - description: testConcat + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: "The Hitchhiker's Guide to the Galaxy" + - Select: + - AliasedExpr: + - Expr.concat: + - Field: author + - Constant: ": " + - Field: title + - "author_title" + - AliasedExpr: + - Expr.concat: + - Field: tags + - - Constant: "new_tag" + - "concatenatedTags" + assert_results: + - author_title: "Douglas Adams: The Hitchhiker's Guide to the Galaxy" + concatenatedTags: + - comedy + - space + - adventure + - new_tag + - description: testLength + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: "The Hitchhiker's Guide to the Galaxy" + - Select: + - AliasedExpr: + - Expr.length: + - Field: title + - "titleLength" + - AliasedExpr: + - Expr.length: + - Field: tags + - "tagsLength" + - AliasedExpr: + - Expr.length: + - Field: awards + - "awardsLength" + assert_results: + - titleLength: 36 + tagsLength: 3 + awardsLength: 2 + - description: testCharLength + pipeline: + - Collection: books + - Select: + - AliasedExpr: + - Expr.char_length: + - Field: title + - "titleLength" + - title + - Where: + - Expr.greater_than: + - Field: titleLength + - Constant: 20 + - Sort: + - Ordering: + - Field: title + - ASCENDING + assert_results: + - titleLength: 29 + title: One Hundred Years of Solitude + - titleLength: 36 + title: The Hitchhiker's Guide to the Galaxy + - titleLength: 21 + title: The Lord of the Rings + - titleLength: 21 + title: To Kill a Mockingbird + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + title: + fieldReferenceValue: title + titleLength: + functionValue: + args: + - fieldReferenceValue: title + name: char_length + name: select + - args: + - functionValue: + args: + - fieldReferenceValue: titleLength + - integerValue: '20' + name: greater_than + name: where + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: title + name: sort + - description: testStringFunctions - CharLength + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: author + - Constant: "Douglas Adams" + - Select: + - AliasedExpr: + - Expr.char_length: + - Field: title + - "title_length" + assert_results: + - title_length: 36 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: author + - stringValue: Douglas Adams + name: equal + name: where + - args: + - mapValue: + fields: + title_length: + functionValue: + args: + - fieldReferenceValue: title + name: char_length + name: select + - description: testStringFunctions - ByteLength + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: author + - Constant: Douglas Adams + - Select: + - AliasedExpr: + - Expr.byte_length: + - Expr.string_concat: + - Field: title + - Constant: _银河系漫游指南 + - "title_byte_length" + assert_results: + - title_byte_length: 58 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: author + - stringValue: Douglas Adams + name: equal + name: where + - args: + - mapValue: + fields: + title_byte_length: + functionValue: + args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: "_\u94F6\u6CB3\u7CFB\u6F2B\u6E38\u6307\u5357" + name: string_concat + name: byte_length + name: select + - description: testLike + pipeline: + - Collection: books + - Where: + - Expr.like: + - Field: title + - Constant: "%Guide%" + - Select: + - title + assert_results: + - title: The Hitchhiker's Guide to the Galaxy + - description: testRegexContains + # Find titles that contain either "the" or "of" (case-insensitive) + pipeline: + - Collection: books + - Where: + - Expr.regex_contains: + - Field: title + - Constant: "(?i)(the|of)" + assert_count: 5 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: "(?i)(the|of)" + name: regex_contains + name: where + - description: testRegexMatches + # Find titles that contain either "the" or "of" (case-insensitive) + pipeline: + - Collection: books + - Where: + - Expr.regex_match: + - Field: title + - Constant: ".*(?i)(the|of).*" + assert_count: 5 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: ".*(?i)(the|of).*" + name: regex_match + name: where + - description: testStringContains + pipeline: + - Collection: books + - Where: + - Expr.string_contains: + - Field: title + - Constant: "Hitchhiker's" + - Select: + - title + assert_results: + - title: "The Hitchhiker's Guide to the Galaxy" + - description: testStringFunctions - ToLower + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: author + - Constant: "Douglas Adams" + - Select: + - AliasedExpr: + - Expr.to_lower: + - Field: title + - "lower_title" + assert_results: + - lower_title: "the hitchhiker's guide to the galaxy" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: author + - stringValue: Douglas Adams + name: equal + name: where + - args: + - mapValue: + fields: + lower_title: + functionValue: + args: + - fieldReferenceValue: title + name: to_lower + name: select + - description: testStringFunctions - ToUpper + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: author + - Constant: "Douglas Adams" + - Select: + - AliasedExpr: + - Expr.to_upper: + - Field: title + - "upper_title" + assert_results: + - upper_title: "THE HITCHHIKER'S GUIDE TO THE GALAXY" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: author + - stringValue: Douglas Adams + name: equal + name: where + - args: + - mapValue: + fields: + upper_title: + functionValue: + args: + - fieldReferenceValue: title + name: to_upper + name: select + - description: testStringFunctions - Trim + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: author + - Constant: "Douglas Adams" + - Select: + - AliasedExpr: + - Expr.trim: + - Expr.string_concat: + - Constant: " " + - Field: title + - Constant: " " + - "trimmed_title" + assert_results: + - trimmed_title: "The Hitchhiker's Guide to the Galaxy" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: author + - stringValue: Douglas Adams + name: equal + name: where + - args: + - mapValue: + fields: + trimmed_title: + functionValue: + args: + - functionValue: + args: + - stringValue: " " + - fieldReferenceValue: title + - stringValue: " " + name: string_concat + name: trim + name: select + - description: testStringFunctions - StringReverse + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: author + - Constant: "Jane Austen" + - Select: + - AliasedExpr: + - Expr.string_reverse: + - Field: title + - "reversed_title" + assert_results: + - reversed_title: "ecidujerP dna edirP" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: author + - stringValue: "Jane Austen" + name: equal + name: where + - args: + - mapValue: + fields: + reversed_title: + functionValue: + args: + - fieldReferenceValue: title + name: string_reverse + name: select + - description: testStringFunctions - Substring + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: author + - Constant: "Douglas Adams" + - Select: + - AliasedExpr: + - Expr.substring: + - Field: title + - Constant: 4 + - Constant: 11 + - "substring_title" + assert_results: + - substring_title: "Hitchhiker'" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: author + - stringValue: "Douglas Adams" + name: equal + name: where + - args: + - mapValue: + fields: + substring_title: + functionValue: + args: + - fieldReferenceValue: title + - integerValue: '4' + - integerValue: '11' + name: substring + name: select + - description: testStringFunctions - Substring without length + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: author + - Constant: "Fyodor Dostoevsky" + - Select: + - AliasedExpr: + - Expr.substring: + - Field: title + - Constant: 10 + - "substring_title" + assert_results: + - substring_title: "Punishment" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: author + - stringValue: "Fyodor Dostoevsky" + name: equal + name: where + - args: + - mapValue: + fields: + substring_title: + functionValue: + args: + - fieldReferenceValue: title + - integerValue: '10' + name: substring + name: select + - description: testStringFunctions - Join + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: author + - Constant: "Douglas Adams" + - Select: + - AliasedExpr: + - Expr.join: + - Field: tags + - Constant: ", " + - "joined_tags" + assert_results: + - joined_tags: "comedy, space, adventure" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: author + - stringValue: "Douglas Adams" + name: equal + name: where + - args: + - mapValue: + fields: + joined_tags: + functionValue: + args: + - fieldReferenceValue: tags + - stringValue: ", " + name: join + name: select diff --git a/tests/system/pipeline_e2e/tests.yaml b/tests/system/pipeline_e2e/tests.yaml index b595dde85..5c959eb74 100644 --- a/tests/system/pipeline_e2e/tests.yaml +++ b/tests/system/pipeline_e2e/tests.yaml @@ -1,272 +1,4 @@ tests: - - description: "testAggregates - count" - pipeline: - - Collection: books - - Aggregate: - - AliasedExpr: - - Expr.count: - - Field: rating - - "count" - assert_results: - - count: 10 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - mapValue: - fields: - count: - functionValue: - name: count - args: - - fieldReferenceValue: rating - - mapValue: {} - name: aggregate - - description: "testAggregates - count_if" - pipeline: - - Collection: books - - Aggregate: - - AliasedExpr: - - Expr.count_if: - - Expr.greater_than: - - Field: rating - - Constant: 4.2 - - "count_if_rating_gt_4_2" - assert_results: - - count_if_rating_gt_4_2: 5 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - mapValue: - fields: - count_if_rating_gt_4_2: - functionValue: - name: count_if - args: - - functionValue: - name: greater_than - args: - - fieldReferenceValue: rating - - doubleValue: 4.2 - - mapValue: {} - name: aggregate - - description: "testAggregates - count_distinct" - pipeline: - - Collection: books - - Aggregate: - - AliasedExpr: - - Expr.count_distinct: - - Field: genre - - "distinct_genres" - assert_results: - - distinct_genres: 8 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - mapValue: - fields: - distinct_genres: - functionValue: - name: count_distinct - args: - - fieldReferenceValue: genre - - mapValue: {} - name: aggregate - - description: "testAggregates - avg, count, max" - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: genre - - Constant: Science Fiction - - Aggregate: - - AliasedExpr: - - Expr.count: - - Field: rating - - "count" - - AliasedExpr: - - Expr.average: - - Field: rating - - "avg_rating" - - AliasedExpr: - - Expr.maximum: - - Field: rating - - "max_rating" - assert_results: - - count: 2 - avg_rating: 4.4 - max_rating: 4.6 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: genre - - stringValue: Science Fiction - name: equal - name: where - - args: - - mapValue: - fields: - avg_rating: - functionValue: - args: - - fieldReferenceValue: rating - name: average - count: - functionValue: - name: count - args: - - fieldReferenceValue: rating - max_rating: - functionValue: - args: - - fieldReferenceValue: rating - name: maximum - - mapValue: {} - name: aggregate - - description: testGroupBysWithoutAccumulators - pipeline: - - Collection: books - - Where: - - Expr.less_than: - - Field: published - - Constant: 1900 - - Aggregate: - accumulators: [] - groups: [genre] - assert_error: ".* requires at least one accumulator" - - description: testGroupBysAndAggregate - pipeline: - - Collection: books - - Where: - - Expr.less_than: - - Field: published - - Constant: 1984 - - Aggregate: - accumulators: - - AliasedExpr: - - Expr.average: - - Field: rating - - "avg_rating" - groups: [genre] - - Where: - - Expr.greater_than: - - Field: avg_rating - - Constant: 4.3 - - Sort: - - Ordering: - - Field: avg_rating - - ASCENDING - assert_results: - - avg_rating: 4.4 - genre: Science Fiction - - avg_rating: 4.5 - genre: Romance - - avg_rating: 4.7 - genre: Fantasy - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: published - - integerValue: '1984' - name: less_than - name: where - - args: - - mapValue: - fields: - avg_rating: - functionValue: - args: - - fieldReferenceValue: rating - name: average - - mapValue: - fields: - genre: - fieldReferenceValue: genre - name: aggregate - - args: - - functionValue: - args: - - fieldReferenceValue: avg_rating - - doubleValue: 4.3 - name: greater_than - name: where - - args: - - mapValue: - fields: - direction: - stringValue: ascending - expression: - fieldReferenceValue: avg_rating - name: sort - - description: testMinMax - pipeline: - - Collection: books - - Aggregate: - - AliasedExpr: - - Expr.count: - - Field: rating - - "count" - - AliasedExpr: - - Expr.maximum: - - Field: rating - - "max_rating" - - AliasedExpr: - - Expr.minimum: - - Field: published - - "min_published" - assert_results: - - count: 10 - max_rating: 4.7 - min_published: 1813 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - mapValue: - fields: - count: - functionValue: - args: - - fieldReferenceValue: rating - name: count - max_rating: - functionValue: - args: - - fieldReferenceValue: rating - name: maximum - min_published: - functionValue: - args: - - fieldReferenceValue: published - name: minimum - - mapValue: {} - name: aggregate - description: selectSpecificFields pipeline: - Collection: books @@ -412,107 +144,7 @@ tests: expression: fieldReferenceValue: author_title name: sort - - description: whereByMultipleConditions - pipeline: - - Collection: books - - Where: - - And: - - Expr.greater_than: - - Field: rating - - Constant: 4.5 - - Expr.equal: - - Field: genre - - Constant: Science Fiction - assert_results: - - title: Dune - author: Frank Herbert - genre: Science Fiction - published: 1965 - rating: 4.6 - tags: - - politics - - desert - - ecology - awards: - hugo: true - nebula: true - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - functionValue: - args: - - fieldReferenceValue: rating - - doubleValue: 4.5 - name: greater_than - - functionValue: - args: - - fieldReferenceValue: genre - - stringValue: Science Fiction - name: equal - name: and - name: where - - description: whereByOrCondition - pipeline: - - Collection: books - - Where: - - Or: - - Expr.equal: - - Field: genre - - Constant: Romance - - Expr.equal: - - Field: genre - - Constant: Dystopian - - Select: - - title - - Sort: - - Ordering: - - Field: title - - ASCENDING - assert_results: - - title: "1984" - - title: Pride and Prejudice - - title: The Handmaid's Tale - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - functionValue: - args: - - fieldReferenceValue: genre - - stringValue: Romance - name: equal - - functionValue: - args: - - fieldReferenceValue: genre - - stringValue: Dystopian - name: equal - name: or - name: where - - args: - - mapValue: - fields: - title: - fieldReferenceValue: title - name: select - - args: - - mapValue: - fields: - direction: - stringValue: ascending - expression: - fieldReferenceValue: title - name: sort + - description: testPipelineWithOffsetAndLimit pipeline: - Collection: books @@ -560,96 +192,57 @@ tests: title: fieldReferenceValue: title name: select - - description: testArrayContains + - description: testArithmeticOperations pipeline: - Collection: books - Where: - - Expr.array_contains: - - Field: tags - - Constant: comedy - assert_results: - - title: The Hitchhiker's Guide to the Galaxy - author: Douglas Adams - awards: - hugo: true - nebula: false - genre: Science Fiction - published: 1979 - rating: 4.2 - tags: ["comedy", "space", "adventure"] - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: tags - - stringValue: comedy - name: array_contains - name: where - - description: testArrayContainsAny - pipeline: - - Collection: books - - Where: - - Expr.array_contains_any: - - Field: tags - - - Constant: comedy - - Constant: classic - - Select: - - title - - Sort: - - Ordering: - - Field: title - - ASCENDING - assert_results: - - title: Pride and Prejudice - - title: The Hitchhiker's Guide to the Galaxy - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: tags - - functionValue: - args: - - stringValue: comedy - - stringValue: classic - name: array - name: array_contains_any - name: where - - args: - - mapValue: - fields: - title: - fieldReferenceValue: title - name: select - - args: - - mapValue: - fields: - direction: - stringValue: ascending - expression: - fieldReferenceValue: title - name: sort - - description: testArrayContainsAll - pipeline: - - Collection: books - - Where: - - Expr.array_contains_all: - - Field: tags - - - Constant: adventure - - Constant: magic + - Expr.equal: + - Field: title + - Constant: To Kill a Mockingbird - Select: - - title + - AliasedExpr: + - Expr.add: + - Field: rating + - Constant: 1 + - "ratingPlusOne" + - AliasedExpr: + - Expr.subtract: + - Field: published + - Constant: 1900 + - "yearsSince1900" + - AliasedExpr: + - Expr.multiply: + - Field: rating + - Constant: 10 + - "ratingTimesTen" + - AliasedExpr: + - Expr.divide: + - Field: rating + - Constant: 2 + - "ratingDividedByTwo" + - AliasedExpr: + - Expr.multiply: + - Field: rating + - Constant: 20 + - "ratingTimes20" + - AliasedExpr: + - Expr.add: + - Field: rating + - Constant: 3 + - "ratingPlus3" + - AliasedExpr: + - Expr.mod: + - Field: rating + - Constant: 2 + - "ratingMod2" assert_results: - - title: The Lord of the Rings + - ratingPlusOne: 5.2 + yearsSince1900: 60 + ratingTimesTen: 42.0 + ratingDividedByTwo: 2.1 + ratingTimes20: 84 + ratingPlus3: 7.2 + ratingMod2: 0.20000000000000018 assert_proto: pipeline: stages: @@ -659,43 +252,61 @@ tests: - args: - functionValue: args: - - fieldReferenceValue: tags - - functionValue: - args: - - stringValue: adventure - - stringValue: magic - name: array - name: array_contains_all + - fieldReferenceValue: title + - stringValue: To Kill a Mockingbird + name: equal name: where - args: - mapValue: fields: - title: - fieldReferenceValue: title + ratingDividedByTwo: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '2' + name: divide + ratingPlusOne: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '1' + name: add + ratingTimesTen: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '10' + name: multiply + yearsSince1900: + functionValue: + args: + - fieldReferenceValue: published + - integerValue: '1900' + name: subtract + ratingTimes20: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '20' + name: multiply + ratingPlus3: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '3' + name: add + ratingMod2: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '2' + name: mod name: select - - description: testArrayLength + - description: testSampleLimit pipeline: - Collection: books - - Select: - - AliasedExpr: - - Expr.array_length: - - Field: tags - - "tagsCount" - - Where: - - Expr.equal: - - Field: tagsCount - - Constant: 3 - assert_results: # All documents have 3 tags - - tagsCount: 3 - - tagsCount: 3 - - tagsCount: 3 - - tagsCount: 3 - - tagsCount: 3 - - tagsCount: 3 - - tagsCount: 3 - - tagsCount: 3 - - tagsCount: 3 - - tagsCount: 3 + - Sample: 3 + assert_count: 3 # Results will vary due to randomness assert_proto: pipeline: stages: @@ -703,38 +314,16 @@ tests: - referenceValue: /books name: collection - args: - - mapValue: - fields: - tagsCount: - functionValue: - args: - - fieldReferenceValue: tags - name: array_length - name: select - - args: - - functionValue: - args: - - fieldReferenceValue: tagsCount - - integerValue: '3' - name: equal - name: where - - description: testStringConcat + - integerValue: '3' + - stringValue: documents + name: sample + - description: testSamplePercentage pipeline: - Collection: books - - Sort: - - Ordering: - - Field: author - - ASCENDING - - Select: - - AliasedExpr: - - Expr.string_concat: - - Field: author - - Constant: " - " - - Field: title - - "bookInfo" - - Limit: 1 - assert_results: - - bookInfo: Douglas Adams - The Hitchhiker's Guide to the Galaxy + - Sample: + - SampleOptions: + - 0.6 + - percent assert_proto: pipeline: stages: @@ -742,45 +331,33 @@ tests: - referenceValue: /books name: collection - args: - - mapValue: - fields: - direction: - stringValue: ascending - expression: - fieldReferenceValue: author - name: sort - - args: - - mapValue: - fields: - bookInfo: - functionValue: - args: - - fieldReferenceValue: author - - stringValue: ' - ' - - fieldReferenceValue: title - name: string_concat - name: select - - args: - - integerValue: '1' - name: limit - - description: testStartsWith + - doubleValue: 0.6 + - stringValue: percent + name: sample + - description: testUnion pipeline: - Collection: books - Where: - - Expr.starts_with: - - Field: title - - Constant: The + - Expr.equal: + - Field: genre + - Constant: Romance + - Union: + - Pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: genre + - Constant: Dystopian - Select: - title - Sort: - Ordering: - - Field: title - - ASCENDING + - Field: title + - ASCENDING assert_results: - - title: The Great Gatsby - - title: The Handmaid's Tale - - title: The Hitchhiker's Guide to the Galaxy - - title: The Lord of the Rings + - title: "1984" + - title: Pride and Prejudice + - title: "The Handmaid's Tale" assert_proto: pipeline: stages: @@ -790,10 +367,24 @@ tests: - args: - functionValue: args: - - fieldReferenceValue: title - - stringValue: The - name: starts_with + - fieldReferenceValue: genre + - stringValue: Romance + name: equal name: where + - args: + - pipelineValue: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: genre + - stringValue: Dystopian + name: equal + name: where + name: union - args: - mapValue: fields: @@ -808,2061 +399,84 @@ tests: expression: fieldReferenceValue: title name: sort - - description: testEndsWith + - description: testUnionFullCollection + pipeline: + - Collection: books + - Union: + - Pipeline: + - Collection: books + assert_count: 20 # Results will be duplicated + + - description: testGreaterThanOrEqual pipeline: - Collection: books - Where: - - Expr.ends_with: - - Field: title - - Constant: y + - Expr.greater_than_or_equal: + - Field: rating + - Constant: 4.6 - Select: - title + - rating - Sort: - Ordering: - - Field: title - - DESCENDING + - Field: rating + - ASCENDING assert_results: - - title: The Hitchhiker's Guide to the Galaxy - - title: The Great Gatsby - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: title - - stringValue: y - name: ends_with - name: where - - args: - - mapValue: - fields: - title: - fieldReferenceValue: title - name: select - - args: - - mapValue: - fields: - direction: - stringValue: descending - expression: - fieldReferenceValue: title - name: sort - - description: testConcat - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: title - - Constant: "The Hitchhiker's Guide to the Galaxy" - - Select: - - AliasedExpr: - - Expr.concat: - - Field: author - - Constant: ": " - - Field: title - - "author_title" - - AliasedExpr: - - Expr.concat: - - Field: tags - - - Constant: "new_tag" - - "concatenatedTags" - assert_results: - - author_title: "Douglas Adams: The Hitchhiker's Guide to the Galaxy" - concatenatedTags: - - comedy - - space - - adventure - - new_tag - - description: testLength - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: title - - Constant: "The Hitchhiker's Guide to the Galaxy" - - Select: - - AliasedExpr: - - Expr.length: - - Field: title - - "titleLength" - - AliasedExpr: - - Expr.length: - - Field: tags - - "tagsLength" - - AliasedExpr: - - Expr.length: - - Field: awards - - "awardsLength" - assert_results: - - titleLength: 36 - tagsLength: 3 - awardsLength: 2 - - description: testCharLength - pipeline: - - Collection: books - - Select: - - AliasedExpr: - - Expr.char_length: - - Field: title - - "titleLength" - - title - - Where: - - Expr.greater_than: - - Field: titleLength - - Constant: 20 - - Sort: - - Ordering: - - Field: title - - ASCENDING - assert_results: - - titleLength: 29 - title: One Hundred Years of Solitude - - titleLength: 36 - title: The Hitchhiker's Guide to the Galaxy - - titleLength: 21 - title: The Lord of the Rings - - titleLength: 21 - title: To Kill a Mockingbird - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - mapValue: - fields: - title: - fieldReferenceValue: title - titleLength: - functionValue: - args: - - fieldReferenceValue: title - name: char_length - name: select - - args: - - functionValue: - args: - - fieldReferenceValue: titleLength - - integerValue: '20' - name: greater_than - name: where - - args: - - mapValue: - fields: - direction: - stringValue: ascending - expression: - fieldReferenceValue: title - name: sort - - description: testStringFunctions - CharLength - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: author - - Constant: "Douglas Adams" - - Select: - - AliasedExpr: - - Expr.char_length: - - Field: title - - "title_length" - assert_results: - - title_length: 36 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: author - - stringValue: Douglas Adams - name: equal - name: where - - args: - - mapValue: - fields: - title_length: - functionValue: - args: - - fieldReferenceValue: title - name: char_length - name: select - - description: testStringFunctions - ByteLength - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: author - - Constant: Douglas Adams - - Select: - - AliasedExpr: - - Expr.byte_length: - - Expr.string_concat: - - Field: title - - Constant: _银河系漫游指南 - - "title_byte_length" - assert_results: - - title_byte_length: 58 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: author - - stringValue: Douglas Adams - name: equal - name: where - - args: - - mapValue: - fields: - title_byte_length: - functionValue: - args: - - functionValue: - args: - - fieldReferenceValue: title - - stringValue: "_\u94F6\u6CB3\u7CFB\u6F2B\u6E38\u6307\u5357" - name: string_concat - name: byte_length - name: select - - description: testLike - pipeline: - - Collection: books - - Where: - - Expr.like: - - Field: title - - Constant: "%Guide%" - - Select: - - title - assert_results: - - title: The Hitchhiker's Guide to the Galaxy - - description: testRegexContains - # Find titles that contain either "the" or "of" (case-insensitive) - pipeline: - - Collection: books - - Where: - - Expr.regex_contains: - - Field: title - - Constant: "(?i)(the|of)" - assert_count: 5 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: title - - stringValue: "(?i)(the|of)" - name: regex_contains - name: where - - description: testRegexMatches - # Find titles that contain either "the" or "of" (case-insensitive) - pipeline: - - Collection: books - - Where: - - Expr.regex_match: - - Field: title - - Constant: ".*(?i)(the|of).*" - assert_count: 5 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: title - - stringValue: ".*(?i)(the|of).*" - name: regex_match - name: where - - description: testArithmeticOperations - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: title - - Constant: To Kill a Mockingbird - - Select: - - AliasedExpr: - - Expr.add: - - Field: rating - - Constant: 1 - - "ratingPlusOne" - - AliasedExpr: - - Expr.subtract: - - Field: published - - Constant: 1900 - - "yearsSince1900" - - AliasedExpr: - - Expr.multiply: - - Field: rating - - Constant: 10 - - "ratingTimesTen" - - AliasedExpr: - - Expr.divide: - - Field: rating - - Constant: 2 - - "ratingDividedByTwo" - - AliasedExpr: - - Expr.multiply: - - Field: rating - - Constant: 20 - - "ratingTimes20" - - AliasedExpr: - - Expr.add: - - Field: rating - - Constant: 3 - - "ratingPlus3" - - AliasedExpr: - - Expr.mod: - - Field: rating - - Constant: 2 - - "ratingMod2" - assert_results: - - ratingPlusOne: 5.2 - yearsSince1900: 60 - ratingTimesTen: 42.0 - ratingDividedByTwo: 2.1 - ratingTimes20: 84 - ratingPlus3: 7.2 - ratingMod2: 0.20000000000000018 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: title - - stringValue: To Kill a Mockingbird - name: equal - name: where - - args: - - mapValue: - fields: - ratingDividedByTwo: - functionValue: - args: - - fieldReferenceValue: rating - - integerValue: '2' - name: divide - ratingPlusOne: - functionValue: - args: - - fieldReferenceValue: rating - - integerValue: '1' - name: add - ratingTimesTen: - functionValue: - args: - - fieldReferenceValue: rating - - integerValue: '10' - name: multiply - yearsSince1900: - functionValue: - args: - - fieldReferenceValue: published - - integerValue: '1900' - name: subtract - ratingTimes20: - functionValue: - args: - - fieldReferenceValue: rating - - integerValue: '20' - name: multiply - ratingPlus3: - functionValue: - args: - - fieldReferenceValue: rating - - integerValue: '3' - name: add - ratingMod2: - functionValue: - args: - - fieldReferenceValue: rating - - integerValue: '2' - name: mod - name: select - - description: testComparisonOperators - pipeline: - - Collection: books - - Where: - - And: - - Expr.greater_than: - - Field: rating - - Constant: 4.2 - - Expr.less_than_or_equal: - - Field: rating - - Constant: 4.5 - - Expr.not_equal: - - Field: genre - - Constant: Science Fiction - - Select: - - rating - - title - - Sort: - - Ordering: - - title - - ASCENDING - assert_results: - - rating: 4.3 - title: Crime and Punishment - - rating: 4.3 - title: One Hundred Years of Solitude - - rating: 4.5 - title: Pride and Prejudice - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - functionValue: - args: - - fieldReferenceValue: rating - - doubleValue: 4.2 - name: greater_than - - functionValue: - args: - - fieldReferenceValue: rating - - doubleValue: 4.5 - name: less_than_or_equal - - functionValue: - args: - - fieldReferenceValue: genre - - stringValue: Science Fiction - name: not_equal - name: and - name: where - - args: - - mapValue: - fields: - rating: - fieldReferenceValue: rating - title: - fieldReferenceValue: title - name: select - - args: - - mapValue: - fields: - direction: - stringValue: ascending - expression: - fieldReferenceValue: title - name: sort - - description: testLogicalOperators - pipeline: - - Collection: books - - Where: - - Or: - - And: - - Expr.greater_than: - - Field: rating - - Constant: 4.5 - - Expr.equal: - - Field: genre - - Constant: Science Fiction - - Expr.less_than: - - Field: published - - Constant: 1900 - - Select: - - title - - Sort: - - Ordering: - - Field: title - - ASCENDING - assert_results: - - title: Crime and Punishment - - title: Dune - - title: Pride and Prejudice - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - functionValue: - args: - - functionValue: - args: - - fieldReferenceValue: rating - - doubleValue: 4.5 - name: greater_than - - functionValue: - args: - - fieldReferenceValue: genre - - stringValue: Science Fiction - name: equal - name: and - - functionValue: - args: - - fieldReferenceValue: published - - integerValue: '1900' - name: less_than - name: or - name: where - - args: - - mapValue: - fields: - title: - fieldReferenceValue: title - name: select - - args: - - mapValue: - fields: - direction: - stringValue: ascending - expression: - fieldReferenceValue: title - name: sort - - description: testChecks - pipeline: - - Collection: books - - Where: - - Not: - - Expr.is_nan: - - Field: rating - - Select: - - AliasedExpr: - - Not: - - Expr.is_nan: - - Field: rating - - "ratingIsNotNaN" - - Limit: 1 - assert_results: - - ratingIsNotNaN: true - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - functionValue: - args: - - fieldReferenceValue: rating - name: is_nan - name: not - name: where - - args: - - mapValue: - fields: - ratingIsNotNaN: - functionValue: - args: - - functionValue: - args: - - fieldReferenceValue: rating - name: is_nan - name: not - name: select - - args: - - integerValue: '1' - name: limit - - description: testIsNotNull - pipeline: - - Collection: books - - Where: - - Expr.is_not_null: - - Field: rating - assert_count: 10 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: rating - name: is_not_null - name: where - - description: testIsNotNaN - pipeline: - - Collection: books - - Where: - - Expr.is_not_nan: - - Field: rating - assert_count: 10 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: rating - name: is_not_nan - name: where - - description: testIsAbsent - pipeline: - - Collection: books - - Where: - - Expr.is_absent: - - Field: awards.pulitzer - assert_count: 9 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: awards.pulitzer - name: is_absent - name: where - - description: testIfAbsent - pipeline: - - Collection: books - - Select: - - AliasedExpr: - - Expr.if_absent: - - Field: awards.pulitzer - - Constant: false - - "pulitzer_award" - - title - - Where: - - Expr.equal: - - Field: pulitzer_award - - Constant: true - assert_results: - - pulitzer_award: true - title: To Kill a Mockingbird - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - mapValue: - fields: - pulitzer_award: - functionValue: - name: if_absent - args: - - fieldReferenceValue: awards.pulitzer - - booleanValue: false - title: - fieldReferenceValue: title - name: select - - args: - - functionValue: - args: - - fieldReferenceValue: pulitzer_award - - booleanValue: true - name: equal - name: where - - description: testIsError - pipeline: - - Collection: books - - Select: - - AliasedExpr: - - Expr.is_error: - - Expr.divide: - - Field: rating - - Constant: "string" - - "is_error_result" - - Limit: 1 - assert_results: - - is_error_result: true - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - mapValue: - fields: - is_error_result: - functionValue: - name: is_error - args: - - functionValue: - name: divide - args: - - fieldReferenceValue: rating - - stringValue: "string" - name: select - - args: - - integerValue: '1' - name: limit - - description: testIfError - pipeline: - - Collection: books - - Select: - - AliasedExpr: - - Expr.if_error: - - Expr.divide: - - Field: rating - - Field: genre - - Constant: "An error occurred" - - "if_error_result" - - Limit: 1 - assert_results: - - if_error_result: "An error occurred" - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - mapValue: - fields: - if_error_result: - functionValue: - name: if_error - args: - - functionValue: - name: divide - args: - - fieldReferenceValue: rating - - fieldReferenceValue: genre - - stringValue: "An error occurred" - name: select - - args: - - integerValue: '1' - name: limit - - description: testLogicalMinMax - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: author - - Constant: Douglas Adams - - Select: - - AliasedExpr: - - Expr.logical_maximum: - - Field: rating - - Constant: 4.5 - - "max_rating" - - AliasedExpr: - - Expr.logical_minimum: - - Field: published - - Constant: 1900 - - "min_published" - assert_results: - - max_rating: 4.5 - min_published: 1900 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: author - - stringValue: Douglas Adams - name: equal - name: where - - args: - - mapValue: - fields: - min_published: - functionValue: - args: - - fieldReferenceValue: published - - integerValue: '1900' - name: minimum - max_rating: - functionValue: - args: - - fieldReferenceValue: rating - - doubleValue: 4.5 - name: maximum - name: select - - description: testMapGet - pipeline: - - Collection: books - - Sort: - - Ordering: - - Field: published - - DESCENDING - - Select: - - AliasedExpr: - - Expr.map_get: - - Field: awards - - hugo - - "hugoAward" - - Field: title - - Where: - - Expr.equal: - - Field: hugoAward - - Constant: true - assert_results: - - hugoAward: true - title: The Hitchhiker's Guide to the Galaxy - - hugoAward: true - title: Dune - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - mapValue: - fields: - direction: - stringValue: descending - expression: - fieldReferenceValue: published - name: sort - - args: - - mapValue: - fields: - hugoAward: - functionValue: - args: - - fieldReferenceValue: awards - - stringValue: hugo - name: map_get - title: - fieldReferenceValue: title - name: select - - args: - - functionValue: - args: - - fieldReferenceValue: hugoAward - - booleanValue: true - name: equal - name: where - - description: testMapGetWithField - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: title - - Constant: "Dune" - - AddFields: - - AliasedExpr: - - Constant: "hugo" - - "award_name" - - Select: - - AliasedExpr: - - Expr.map_get: - - Field: awards - - Field: award_name - - "hugoAward" - - Field: title - assert_results: - - hugoAward: true - title: Dune - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: title - - stringValue: "Dune" - name: equal - name: where - - args: - - mapValue: - fields: - award_name: - stringValue: "hugo" - name: add_fields - - args: - - mapValue: - fields: - hugoAward: - functionValue: - name: map_get - args: - - fieldReferenceValue: awards - - fieldReferenceValue: award_name - title: - fieldReferenceValue: title - name: select - - description: testMapRemove - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: title - - Constant: "Dune" - - Select: - - AliasedExpr: - - Expr.map_remove: - - Field: awards - - "nebula" - - "awards_removed" - assert_results: - - awards_removed: - hugo: true - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: title - - stringValue: "Dune" - name: equal - name: where - - args: - - mapValue: - fields: - awards_removed: - functionValue: - name: map_remove - args: - - fieldReferenceValue: awards - - stringValue: "nebula" - name: select - - description: testMapMerge - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: title - - Constant: "Dune" - - Select: - - AliasedExpr: - - Expr.map_merge: - - Field: awards - - Map: - elements: {"new_award": true, "hugo": false} - - Map: - elements: {"another_award": "yes"} - - "awards_merged" - assert_results: - - awards_merged: - hugo: false - nebula: true - new_award: true - another_award: "yes" - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: title - - stringValue: "Dune" - name: equal - name: where - - args: - - mapValue: - fields: - awards_merged: - functionValue: - name: map_merge - args: - - fieldReferenceValue: awards - - functionValue: - name: map - args: - - stringValue: "new_award" - - booleanValue: true - - stringValue: "hugo" - - booleanValue: false - - functionValue: - name: map - args: - - stringValue: "another_award" - - stringValue: "yes" - name: select - - description: testNestedFields - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: awards.hugo - - Constant: true - - Sort: - - Ordering: - - Field: title - - DESCENDING - - Select: - - title - - Field: awards.hugo - assert_results: - - title: The Hitchhiker's Guide to the Galaxy - awards.hugo: true - - title: Dune - awards.hugo: true - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: awards.hugo - - booleanValue: true - name: equal - name: where - - args: - - mapValue: - fields: - direction: - stringValue: descending - expression: - fieldReferenceValue: title - name: sort - - args: - - mapValue: - fields: - awards.hugo: - fieldReferenceValue: awards.hugo - title: - fieldReferenceValue: title - name: select - - description: testSampleLimit - pipeline: - - Collection: books - - Sample: 3 - assert_count: 3 # Results will vary due to randomness - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - integerValue: '3' - - stringValue: documents - name: sample - - description: testSamplePercentage - pipeline: - - Collection: books - - Sample: - - SampleOptions: - - 0.6 - - percent - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - doubleValue: 0.6 - - stringValue: percent - name: sample - - description: testUnion - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: genre - - Constant: Romance - - Union: - - Pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: genre - - Constant: Dystopian - - Select: - - title - - Sort: - - Ordering: - - Field: title - - ASCENDING - assert_results: - - title: "1984" - - title: Pride and Prejudice - - title: "The Handmaid's Tale" - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: genre - - stringValue: Romance - name: equal - name: where - - args: - - pipelineValue: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: genre - - stringValue: Dystopian - name: equal - name: where - name: union - - args: - - mapValue: - fields: - title: - fieldReferenceValue: title - name: select - - args: - - mapValue: - fields: - direction: - stringValue: ascending - expression: - fieldReferenceValue: title - name: sort - - description: testUnionFullCollection - pipeline: - - Collection: books - - Union: - - Pipeline: - - Collection: books - assert_count: 20 # Results will be duplicated - - description: testUnnest - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: title - - Constant: The Hitchhiker's Guide to the Galaxy - - Unnest: - - tags - - tags_alias - - Select: tags_alias - assert_results: - - tags_alias: comedy - - tags_alias: space - - tags_alias: adventure - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: title - - stringValue: The Hitchhiker's Guide to the Galaxy - name: equal - name: where - - args: - - fieldReferenceValue: tags - - fieldReferenceValue: tags_alias - name: unnest - - args: - - mapValue: - fields: - tags_alias: - fieldReferenceValue: tags_alias - name: select - - description: testUnnestWithOptions - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: title - - Constant: The Hitchhiker's Guide to the Galaxy - - Unnest: - field: tags - alias: tags_alias - options: - UnnestOptions: - - index - - Select: - - tags_alias - - index - assert_results: - - tags_alias: comedy - index: 0 - - tags_alias: space - index: 1 - - tags_alias: adventure - index: 2 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: title - - stringValue: The Hitchhiker's Guide to the Galaxy - name: equal - name: where - - args: - - fieldReferenceValue: tags - - fieldReferenceValue: tags_alias - name: unnest - options: - index_field: - fieldReferenceValue: index - - args: - - mapValue: - fields: - tags_alias: - fieldReferenceValue: tags_alias - index: - fieldReferenceValue: index - name: select - - description: testGreaterThanOrEqual - pipeline: - - Collection: books - - Where: - - Expr.greater_than_or_equal: - - Field: rating - - Constant: 4.6 - - Select: - - title - - rating - - Sort: - - Ordering: - - Field: rating - - ASCENDING - assert_results: - - title: Dune - rating: 4.6 - - title: The Lord of the Rings - rating: 4.7 - - description: testInAndNotIn - pipeline: - - Collection: books - - Where: - - And: - - Expr.equal_any: - - Field: genre - - - Constant: Romance - - Constant: Dystopian - - Expr.not_equal_any: - - Field: author - - - Constant: "George Orwell" - assert_results: - - title: "Pride and Prejudice" - author: "Jane Austen" - genre: "Romance" - published: 1813 - rating: 4.5 - tags: - - classic - - social commentary - - love - awards: - none: true - - title: "The Handmaid's Tale" - author: "Margaret Atwood" - genre: "Dystopian" - published: 1985 - rating: 4.1 - tags: - - feminism - - totalitarianism - - resistance - awards: - "arthur c. clarke": true - "booker prize": false - - description: testArrayReverse - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: title - - Constant: "The Hitchhiker's Guide to the Galaxy" - - Select: - - AliasedExpr: - - Expr.array_reverse: - - Field: tags - - "reversedTags" - assert_results: - - reversedTags: - - adventure - - space - - comedy - - description: testDocumentId - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: title - - Constant: "The Hitchhiker's Guide to the Galaxy" - - Select: - - AliasedExpr: - - Expr.document_id: - - Field: __name__ - - "doc_id" - assert_results: - - doc_id: "book1" - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: title - - stringValue: "The Hitchhiker's Guide to the Galaxy" - name: equal - name: where - - args: - - mapValue: - fields: - doc_id: - functionValue: - name: document_id - args: - - fieldReferenceValue: __name__ - name: select - - description: testCurrentTimestamp - pipeline: - - Collection: books - - Limit: 1 - - Select: - - AliasedExpr: - - And: - - Expr.greater_than_or_equal: - - CurrentTimestamp: [] - - Expr.unix_seconds_to_timestamp: - - Constant: 1735689600 # 2025-01-01 - - Expr.less_than: - - CurrentTimestamp: [] - - Expr.unix_seconds_to_timestamp: - - Constant: 4892438400 # 2125-01-01 - - "is_between_2025_and_2125" - assert_results: - - is_between_2025_and_2125: true - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - integerValue: '1' - name: limit - - args: - - mapValue: - fields: - is_between_2025_and_2125: - functionValue: - name: and - args: - - functionValue: - name: greater_than_or_equal - args: - - functionValue: - name: current_timestamp - - functionValue: - name: unix_seconds_to_timestamp - args: - - integerValue: '1735689600' - - functionValue: - name: less_than - args: - - functionValue: - name: current_timestamp - - functionValue: - name: unix_seconds_to_timestamp - args: - - integerValue: '4892438400' - name: select - - description: testArrayConcat - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: title - - Constant: "The Hitchhiker's Guide to the Galaxy" - - Select: - - AliasedExpr: - - Expr.array_concat: - - Field: tags - - Constant: ["new_tag", "another_tag"] - - "concatenatedTags" - assert_results: - - concatenatedTags: - - comedy - - space - - adventure - - new_tag - - another_tag - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: title - - stringValue: "The Hitchhiker's Guide to the Galaxy" - name: equal - name: where - - args: - - mapValue: - fields: - concatenatedTags: - functionValue: - args: - - fieldReferenceValue: tags - - functionValue: - args: - - stringValue: "new_tag" - - stringValue: "another_tag" - name: array - name: array_concat - name: select - - description: testArrayConcatMultiple - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: title - - Constant: "Dune" - - Select: - - AliasedExpr: - - Expr.array_concat: - - Field: tags - - Constant: ["sci-fi"] - - Constant: ["classic", "epic"] - - "concatenatedTags" - assert_results: - - concatenatedTags: - - politics - - desert - - ecology - - sci-fi - - classic - - epic - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: title - - stringValue: "Dune" - name: equal - name: where - - args: - - mapValue: - fields: - concatenatedTags: - functionValue: - args: - - fieldReferenceValue: tags - - functionValue: - args: - - stringValue: "sci-fi" - name: array - - functionValue: - args: - - stringValue: "classic" - - stringValue: "epic" - name: array - name: array_concat - name: select - - description: testMapMergeLiterals - pipeline: - - Collection: books - - Limit: 1 - - Select: - - AliasedExpr: - - Expr.map_merge: - - Map: - elements: {"a": "orig", "b": "orig"} - - Map: - elements: {"b": "new", "c": "new"} - - "merged" - assert_results: - - merged: - a: "orig" - b: "new" - c: "new" - - description: testArrayContainsAnyWithField - pipeline: - - Collection: books - - AddFields: - - AliasedExpr: - - Expr.array_concat: - - Field: tags - - Array: ["Dystopian"] - - "new_tags" - - Where: - - Expr.array_contains_any: - - Field: new_tags - - - Constant: non_existent_tag - - Field: genre - - Select: - - title - - genre - - Sort: - - Ordering: - - Field: title - - ASCENDING - assert_results: - - title: "1984" - genre: "Dystopian" - - title: "The Handmaid's Tale" - genre: "Dystopian" - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - mapValue: - fields: - new_tags: - functionValue: - args: - - fieldReferenceValue: tags - - functionValue: - args: - - stringValue: "Dystopian" - name: array - name: array_concat - name: add_fields - - args: - - functionValue: - args: - - fieldReferenceValue: new_tags - - functionValue: - args: - - stringValue: "non_existent_tag" - - fieldReferenceValue: genre - name: array - name: array_contains_any - name: where - - args: - - mapValue: - fields: - title: - fieldReferenceValue: title - genre: - fieldReferenceValue: genre - name: select - - args: - - mapValue: - fields: - direction: - stringValue: ascending - expression: - fieldReferenceValue: title - name: sort - - description: testArrayConcatLiterals - pipeline: - - Collection: books - - Limit: 1 - - Select: - - AliasedExpr: - - Expr.array_concat: - - Array: [1, 2, 3] - - Array: [4, 5] - - "concatenated" - assert_results: - - concatenated: [1, 2, 3, 4, 5] - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - integerValue: '1' - name: limit - - args: - - mapValue: - fields: - concatenated: - functionValue: - args: - - functionValue: - args: - - integerValue: '1' - - integerValue: '2' - - integerValue: '3' - name: array - - functionValue: - args: - - integerValue: '4' - - integerValue: '5' - name: array - name: array_concat - name: select - - description: testExists - pipeline: - - Collection: books - - Where: - - And: - - Expr.exists: - - Field: awards.pulitzer - - Expr.equal: - - Field: awards.pulitzer - - Constant: true - - Select: - - title - assert_results: - - title: To Kill a Mockingbird - - description: testSum - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: genre - - Constant: Science Fiction - - Aggregate: - - AliasedExpr: - - Expr.sum: - - Field: rating - - "total_rating" - assert_results: - - total_rating: 8.8 - - description: testStringContains - pipeline: - - Collection: books - - Where: - - Expr.string_contains: - - Field: title - - Constant: "Hitchhiker's" - - Select: - - title - assert_results: - - title: "The Hitchhiker's Guide to the Galaxy" - - description: testVectorLength - pipeline: - - Collection: vectors - - Select: - - AliasedExpr: - - Expr.vector_length: - - Field: embedding - - "embedding_length" - - Sort: - - Ordering: - - Field: embedding_length - - ASCENDING - assert_results: - - embedding_length: 3 - - embedding_length: 3 - - embedding_length: 3 - - embedding_length: 4 - - description: testTimestampFunctions - pipeline: - - Collection: timestamps - - Select: - - AliasedExpr: - - Expr.timestamp_to_unix_micros: - - Field: time - - "micros" - - AliasedExpr: - - Expr.timestamp_to_unix_millis: - - Field: time - - "millis" - - AliasedExpr: - - Expr.timestamp_to_unix_seconds: - - Field: time - - "seconds" - - AliasedExpr: - - Expr.unix_micros_to_timestamp: - - Field: micros - - "from_micros" - - AliasedExpr: - - Expr.unix_millis_to_timestamp: - - Field: millis - - "from_millis" - - AliasedExpr: - - Expr.unix_seconds_to_timestamp: - - Field: seconds - - "from_seconds" - - AliasedExpr: - - Expr.timestamp_add: - - Field: time - - Constant: "day" - - Constant: 1 - - "plus_day" - - AliasedExpr: - - Expr.timestamp_subtract: - - Field: time - - Constant: "hour" - - Constant: 1 - - "minus_hour" - assert_results: - - micros: 735998460654321 - millis: 735998460654 - seconds: 735998460 - from_micros: "1993-04-28T12:01:00.654321+00:00" - from_millis: "1993-04-28T12:01:00.654000+00:00" - from_seconds: "1993-04-28T12:01:00.000000+00:00" - plus_day: "1993-04-29T12:01:00.654321+00:00" - minus_hour: "1993-04-28T11:01:00.654321+00:00" - - description: testCollectionId - pipeline: - - Collection: books - - Limit: 1 - - Select: - - AliasedExpr: - - Expr.collection_id: - - Field: __name__ - - "collectionName" - assert_results: - - collectionName: "books" - - description: testXor - pipeline: - - Collection: books - - Where: - - Xor: - - - Expr.equal: - - Field: genre - - Constant: Romance - - Expr.greater_than: - - Field: published - - Constant: 1980 - - Select: - - title - - genre - - published - - Sort: - - Ordering: - - Field: title - - ASCENDING + - title: Dune + rating: 4.6 + - title: The Lord of the Rings + rating: 4.7 + - description: testInAndNotIn + pipeline: + - Collection: books + - Where: + - And: + - Expr.equal_any: + - Field: genre + - - Constant: Romance + - Constant: Dystopian + - Expr.not_equal_any: + - Field: author + - - Constant: "George Orwell" assert_results: - title: "Pride and Prejudice" + author: "Jane Austen" genre: "Romance" published: 1813 + rating: 4.5 + tags: + - classic + - social commentary + - love + awards: + none: true - title: "The Handmaid's Tale" + author: "Margaret Atwood" genre: "Dystopian" published: 1985 - - description: testConditional - pipeline: - - Collection: books - - Select: - - title - - AliasedExpr: - - Conditional: - - Expr.greater_than: - - Field: published - - Constant: 1950 - - Constant: "Modern" - - Constant: "Classic" - - "era" - - Sort: - - Ordering: - - Field: title - - ASCENDING - - Limit: 4 - assert_results: - - title: "1984" - era: "Classic" - - title: "Crime and Punishment" - era: "Classic" - - title: "Dune" - era: "Modern" - - title: "One Hundred Years of Solitude" - era: "Modern" - - description: testFieldToFieldArithmetic - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: title - - Constant: "Dune" - - Select: - - AliasedExpr: - - Expr.add: - - Field: published - - Field: rating - - "pub_plus_rating" - assert_results: - - pub_plus_rating: 1969.6 - - description: testFieldToFieldComparison - pipeline: - - Collection: books - - Where: - - Expr.greater_than: - - Field: published - - Field: rating - - Select: - - title - assert_count: 10 # All books were published after year 4.7 - - description: testExistsNegative - pipeline: - - Collection: books - - Where: - - Expr.exists: - - Field: non_existent_field - assert_count: 0 - - description: testConditionalWithFields - pipeline: - - Collection: books - - Where: - - Expr.equal_any: - - Field: title - - - Constant: "Dune" - - Constant: "1984" - - Select: - - title - - AliasedExpr: - - Conditional: - - Expr.greater_than: - - Field: published - - Constant: 1950 - - Field: author - - Field: genre - - "conditional_field" - - Sort: - - Ordering: - - Field: title - - ASCENDING - assert_results: - - title: "1984" - conditional_field: "Dystopian" - - title: "Dune" - conditional_field: "Frank Herbert" - - description: testFindNearestEuclidean - pipeline: - - Collection: vectors - - FindNearest: - field: embedding - vector: [1.0, 2.0, 3.0] - distance_measure: EUCLIDEAN - options: - FindNearestOptions: - limit: 2 - distance_field: - Field: distance - - Select: - - distance - assert_results: - - distance: 0.0 - - distance: 1.0 - assert_proto: - pipeline: - stages: - - name: collection - args: - - referenceValue: /vectors - - name: find_nearest - args: - - fieldReferenceValue: embedding - - mapValue: - fields: - __type__: - stringValue: __vector__ - value: - arrayValue: - values: - - doubleValue: 1.0 - - doubleValue: 2.0 - - doubleValue: 3.0 - - stringValue: euclidean - options: - limit: - integerValue: '2' - distance_field: - fieldReferenceValue: distance - - name: select - args: - - mapValue: - fields: - distance: - fieldReferenceValue: distance - - description: testMathExpressions - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: title - - Constant: To Kill a Mockingbird - - Select: - - AliasedExpr: - - Expr.abs: - - Field: rating - - "abs_rating" - - AliasedExpr: - - Expr.ceil: - - Field: rating - - "ceil_rating" - - AliasedExpr: - - Expr.exp: - - Field: rating - - "exp_rating" - - AliasedExpr: - - Expr.floor: - - Field: rating - - "floor_rating" - - AliasedExpr: - - Expr.ln: - - Field: rating - - "ln_rating" - - AliasedExpr: - - Expr.log10: - - Field: rating - - "log_rating_base10" - - AliasedExpr: - - Expr.log: - - Field: rating - - Constant: 2 - - "log_rating_base2" - - AliasedExpr: - - Expr.pow: - - Field: rating - - Constant: 2 - - "pow_rating" - - AliasedExpr: - - Expr.sqrt: - - Field: rating - - "sqrt_rating" - assert_results_approximate: - - abs_rating: 4.2 - ceil_rating: 5.0 - exp_rating: 66.686331 - floor_rating: 4.0 - ln_rating: 1.4350845 - log_rating_base10: 0.623249 - log_rating_base2: 2.0704 - pow_rating: 17.64 - sqrt_rating: 2.049390 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: title - - stringValue: To Kill a Mockingbird - name: equal - name: where - - args: - - mapValue: - fields: - abs_rating: - functionValue: - args: - - fieldReferenceValue: rating - name: abs - ceil_rating: - functionValue: - args: - - fieldReferenceValue: rating - name: ceil - exp_rating: - functionValue: - args: - - fieldReferenceValue: rating - name: exp - floor_rating: - functionValue: - args: - - fieldReferenceValue: rating - name: floor - ln_rating: - functionValue: - args: - - fieldReferenceValue: rating - name: ln - log_rating_base10: - functionValue: - args: - - fieldReferenceValue: rating - name: log10 - log_rating_base2: - functionValue: - args: - - fieldReferenceValue: rating - - integerValue: '2' - name: log - pow_rating: - functionValue: - args: - - fieldReferenceValue: rating - - integerValue: '2' - name: pow - sqrt_rating: - functionValue: - args: - - fieldReferenceValue: rating - name: sqrt - name: select - - description: testRoundExpressions + rating: 4.1 + tags: + - feminism + - totalitarianism + - resistance + awards: + "arthur c. clarke": true + "booker prize": false + + - description: testDocumentId pipeline: - Collection: books - Where: - - Expr.equal_any: + - Expr.equal: - Field: title - - - Constant: "To Kill a Mockingbird" # rating 4.2 - - Constant: "Pride and Prejudice" # rating 4.5 - - Constant: "The Lord of the Rings" # rating 4.7 + - Constant: "The Hitchhiker's Guide to the Galaxy" - Select: - - title - AliasedExpr: - - Expr.round: - - Field: rating - - "round_rating" - - Sort: - - Ordering: - - Field: title - - ASCENDING + - Expr.document_id: + - Field: __name__ + - "doc_id" assert_results: - - title: "Pride and Prejudice" - round_rating: 5.0 - - title: "The Lord of the Rings" - round_rating: 5.0 - - title: "To Kill a Mockingbird" - round_rating: 4.0 + - doc_id: "book1" assert_proto: pipeline: stages: @@ -2873,295 +487,212 @@ tests: - functionValue: args: - fieldReferenceValue: title - - functionValue: - args: - - stringValue: "To Kill a Mockingbird" - - stringValue: "Pride and Prejudice" - - stringValue: "The Lord of the Rings" - name: array - name: equal_any + - stringValue: "The Hitchhiker's Guide to the Galaxy" + name: equal name: where - args: - mapValue: fields: - title: - fieldReferenceValue: title - round_rating: + doc_id: functionValue: + name: document_id args: - - fieldReferenceValue: rating - name: round + - fieldReferenceValue: __name__ name: select - - args: - - mapValue: - fields: - direction: - stringValue: ascending - expression: - fieldReferenceValue: title - name: sort - - description: testFindNearestDotProduct - pipeline: - - Collection: vectors - - FindNearest: - field: embedding - vector: [1.0, 2.0, 3.0] - distance_measure: DOT_PRODUCT - options: - FindNearestOptions: - limit: 3 - distance_field: - Field: distance - - Select: - - distance - assert_results: - - distance: 38.0 - - distance: 17.0 - - distance: 14.0 - assert_proto: - pipeline: - stages: - - name: collection - args: - - referenceValue: /vectors - - name: find_nearest - args: - - fieldReferenceValue: embedding - - mapValue: - fields: - __type__: - stringValue: __vector__ - value: - arrayValue: - values: - - doubleValue: 1.0 - - doubleValue: 2.0 - - doubleValue: 3.0 - - stringValue: dot_product - options: - limit: - integerValue: '3' - distance_field: - fieldReferenceValue: distance - - name: select - args: - - mapValue: - fields: - distance: - fieldReferenceValue: distance - - description: testDotProductWithConstant + - description: testSum pipeline: - - Collection: vectors + - Collection: books - Where: - Expr.equal: - - Field: embedding - - Vector: [1.0, 2.0, 3.0] - - Select: + - Field: genre + - Constant: Science Fiction + - Aggregate: - AliasedExpr: - - Expr.dot_product: - - Field: embedding - - Vector: [1.0, 1.0, 1.0] - - "dot_product_result" + - Expr.sum: + - Field: rating + - "total_rating" assert_results: - - dot_product_result: 6.0 - - description: testEuclideanDistanceWithConstant + - total_rating: 8.8 + + - description: testCollectionId pipeline: - - Collection: vectors - - Where: - - Expr.equal: - - Field: embedding - - Vector: [1.0, 2.0, 3.0] + - Collection: books + - Limit: 1 - Select: - AliasedExpr: - - Expr.euclidean_distance: - - Field: embedding - - Vector: [1.0, 2.0, 3.0] - - "euclidean_distance_result" + - Expr.collection_id: + - Field: __name__ + - "collectionName" assert_results: - - euclidean_distance_result: 0.0 - - description: testCosineDistanceWithConstant + - collectionName: "books" + - description: testXor pipeline: - - Collection: vectors + - Collection: books - Where: - - Expr.equal: - - Field: embedding - - Vector: [1.0, 2.0, 3.0] + - Xor: + - - Expr.equal: + - Field: genre + - Constant: Romance + - Expr.greater_than: + - Field: published + - Constant: 1980 - Select: - - AliasedExpr: - - Expr.cosine_distance: - - Field: embedding - - Vector: [1.0, 2.0, 3.0] - - "cosine_distance_result" + - title + - genre + - published + - Sort: + - Ordering: + - Field: title + - ASCENDING assert_results: - - cosine_distance_result: 0.0 - - description: testStringFunctions - ToLower + - title: "Pride and Prejudice" + genre: "Romance" + published: 1813 + - title: "The Handmaid's Tale" + genre: "Dystopian" + published: 1985 + - description: testConditional pipeline: - Collection: books - - Where: - - Expr.equal: - - Field: author - - Constant: "Douglas Adams" - Select: + - title - AliasedExpr: - - Expr.to_lower: - - Field: title - - "lower_title" + - Conditional: + - Expr.greater_than: + - Field: published + - Constant: 1950 + - Constant: "Modern" + - Constant: "Classic" + - "era" + - Sort: + - Ordering: + - Field: title + - ASCENDING + - Limit: 4 assert_results: - - lower_title: "the hitchhiker's guide to the galaxy" - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: author - - stringValue: Douglas Adams - name: equal - name: where - - args: - - mapValue: - fields: - lower_title: - functionValue: - args: - - fieldReferenceValue: title - name: to_lower - name: select - - description: testStringFunctions - ToUpper + - title: "1984" + era: "Classic" + - title: "Crime and Punishment" + era: "Classic" + - title: "Dune" + era: "Modern" + - title: "One Hundred Years of Solitude" + era: "Modern" + - description: testFieldToFieldArithmetic pipeline: - Collection: books - Where: - Expr.equal: - - Field: author - - Constant: "Douglas Adams" + - Field: title + - Constant: "Dune" - Select: - AliasedExpr: - - Expr.to_upper: - - Field: title - - "upper_title" + - Expr.add: + - Field: published + - Field: rating + - "pub_plus_rating" assert_results: - - upper_title: "THE HITCHHIKER'S GUIDE TO THE GALAXY" - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: author - - stringValue: Douglas Adams - name: equal - name: where - - args: - - mapValue: - fields: - upper_title: - functionValue: - args: - - fieldReferenceValue: title - name: to_upper - name: select - - description: testStringFunctions - Trim + - pub_plus_rating: 1969.6 + - description: testFieldToFieldComparison pipeline: - Collection: books - Where: - - Expr.equal: - - Field: author - - Constant: "Douglas Adams" + - Expr.greater_than: + - Field: published + - Field: rating - Select: - - AliasedExpr: - - Expr.trim: - - Expr.string_concat: - - Constant: " " - - Field: title - - Constant: " " - - "trimmed_title" - assert_results: - - trimmed_title: "The Hitchhiker's Guide to the Galaxy" - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: author - - stringValue: Douglas Adams - name: equal - name: where - - args: - - mapValue: - fields: - trimmed_title: - functionValue: - args: - - functionValue: - args: - - stringValue: " " - - fieldReferenceValue: title - - stringValue: " " - name: string_concat - name: trim - name: select - - description: testStringFunctions - StringReverse + - title + assert_count: 10 # All books were published after year 4.7 + - description: testExistsNegative pipeline: - Collection: books - Where: - - Expr.equal: - - Field: author - - Constant: "Jane Austen" + - Expr.exists: + - Field: non_existent_field + assert_count: 0 + - description: testConditionalWithFields + pipeline: + - Collection: books + - Where: + - Expr.equal_any: + - Field: title + - - Constant: "Dune" + - Constant: "1984" - Select: + - title - AliasedExpr: - - Expr.string_reverse: - - Field: title - - "reversed_title" + - Conditional: + - Expr.greater_than: + - Field: published + - Constant: 1950 + - Field: author + - Field: genre + - "conditional_field" + - Sort: + - Ordering: + - Field: title + - ASCENDING assert_results: - - reversed_title: "ecidujerP dna edirP" - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: author - - stringValue: "Jane Austen" - name: equal - name: where - - args: - - mapValue: - fields: - reversed_title: - functionValue: - args: - - fieldReferenceValue: title - name: string_reverse - name: select - - description: testStringFunctions - Substring + - title: "1984" + conditional_field: "Dystopian" + - title: "Dune" + conditional_field: "Frank Herbert" + - description: testMathExpressions pipeline: - Collection: books - Where: - - Expr.equal: - - Field: author - - Constant: "Douglas Adams" + - Expr.equal: + - Field: title + - Constant: To Kill a Mockingbird - Select: - AliasedExpr: - - Expr.substring: - - Field: title - - Constant: 4 - - Constant: 11 - - "substring_title" - assert_results: - - substring_title: "Hitchhiker'" + - Expr.abs: + - Field: rating + - "abs_rating" + - AliasedExpr: + - Expr.ceil: + - Field: rating + - "ceil_rating" + - AliasedExpr: + - Expr.exp: + - Field: rating + - "exp_rating" + - AliasedExpr: + - Expr.floor: + - Field: rating + - "floor_rating" + - AliasedExpr: + - Expr.ln: + - Field: rating + - "ln_rating" + - AliasedExpr: + - Expr.log10: + - Field: rating + - "log_rating_base10" + - AliasedExpr: + - Expr.log: + - Field: rating + - Constant: 2 + - "log_rating_base2" + - AliasedExpr: + - Expr.pow: + - Field: rating + - Constant: 2 + - "pow_rating" + - AliasedExpr: + - Expr.sqrt: + - Field: rating + - "sqrt_rating" + assert_results_approximate: + - abs_rating: 4.2 + ceil_rating: 5.0 + exp_rating: 66.686331 + floor_rating: 4.0 + ln_rating: 1.4350845 + log_rating_base10: 0.623249 + log_rating_base2: 2.0704 + pow_rating: 17.64 + sqrt_rating: 2.049390 assert_proto: pipeline: stages: @@ -3171,36 +702,87 @@ tests: - args: - functionValue: args: - - fieldReferenceValue: author - - stringValue: "Douglas Adams" + - fieldReferenceValue: title + - stringValue: To Kill a Mockingbird name: equal name: where - args: - mapValue: fields: - substring_title: + abs_rating: functionValue: args: - - fieldReferenceValue: title - - integerValue: '4' - - integerValue: '11' - name: substring + - fieldReferenceValue: rating + name: abs + ceil_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: ceil + exp_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: exp + floor_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: floor + ln_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: ln + log_rating_base10: + functionValue: + args: + - fieldReferenceValue: rating + name: log10 + log_rating_base2: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '2' + name: log + pow_rating: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '2' + name: pow + sqrt_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: sqrt name: select - - description: testStringFunctions - Substring without length + - description: testRoundExpressions pipeline: - Collection: books - Where: - - Expr.equal: - - Field: author - - Constant: "Fyodor Dostoevsky" + - Expr.equal_any: + - Field: title + - - Constant: "To Kill a Mockingbird" # rating 4.2 + - Constant: "Pride and Prejudice" # rating 4.5 + - Constant: "The Lord of the Rings" # rating 4.7 - Select: + - title - AliasedExpr: - - Expr.substring: - - Field: title - - Constant: 10 - - "substring_title" + - Expr.round: + - Field: rating + - "round_rating" + - Sort: + - Ordering: + - Field: title + - ASCENDING assert_results: - - substring_title: "Punishment" + - title: "Pride and Prejudice" + round_rating: 5.0 + - title: "The Lord of the Rings" + round_rating: 5.0 + - title: "To Kill a Mockingbird" + round_rating: 4.0 assert_proto: pipeline: stages: @@ -3210,58 +792,34 @@ tests: - args: - functionValue: args: - - fieldReferenceValue: author - - stringValue: "Fyodor Dostoevsky" - name: equal + - fieldReferenceValue: title + - functionValue: + args: + - stringValue: "To Kill a Mockingbird" + - stringValue: "Pride and Prejudice" + - stringValue: "The Lord of the Rings" + name: array + name: equal_any name: where - args: - mapValue: fields: - substring_title: + title: + fieldReferenceValue: title + round_rating: functionValue: args: - - fieldReferenceValue: title - - integerValue: '10' - name: substring + - fieldReferenceValue: rating + name: round name: select - - description: testStringFunctions - Join - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: author - - Constant: "Douglas Adams" - - Select: - - AliasedExpr: - - Expr.join: - - Field: tags - - Constant: ", " - - "joined_tags" - assert_results: - - joined_tags: "comedy, space, adventure" - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: author - - stringValue: "Douglas Adams" - name: equal - name: where - args: - mapValue: fields: - joined_tags: - functionValue: - args: - - fieldReferenceValue: tags - - stringValue: ", " - name: join - name: select + direction: + stringValue: ascending + expression: + fieldReferenceValue: title + name: sort - description: testCollectionGroup pipeline: - CollectionGroup: books @@ -3523,4 +1081,93 @@ tests: fields: author: fieldReferenceValue: author + name: select + - description: testUnnest + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: The Hitchhiker's Guide to the Galaxy + - Unnest: + - tags + - tags_alias + - Select: tags_alias + assert_results: + - tags_alias: comedy + - tags_alias: space + - tags_alias: adventure + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: The Hitchhiker's Guide to the Galaxy + name: equal + name: where + - args: + - fieldReferenceValue: tags + - fieldReferenceValue: tags_alias + name: unnest + - args: + - mapValue: + fields: + tags_alias: + fieldReferenceValue: tags_alias + name: select + - description: testUnnestWithOptions + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: The Hitchhiker's Guide to the Galaxy + - Unnest: + field: tags + alias: tags_alias + options: + UnnestOptions: + - index + - Select: + - tags_alias + - index + assert_results: + - tags_alias: comedy + index: 0 + - tags_alias: space + index: 1 + - tags_alias: adventure + index: 2 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: The Hitchhiker's Guide to the Galaxy + name: equal + name: where + - args: + - fieldReferenceValue: tags + - fieldReferenceValue: tags_alias + name: unnest + options: + index_field: + fieldReferenceValue: index + - args: + - mapValue: + fields: + tags_alias: + fieldReferenceValue: tags_alias + index: + fieldReferenceValue: index name: select \ No newline at end of file diff --git a/tests/system/pipeline_e2e/vector.yaml b/tests/system/pipeline_e2e/vector.yaml new file mode 100644 index 000000000..15fc9bcaa --- /dev/null +++ b/tests/system/pipeline_e2e/vector.yaml @@ -0,0 +1,160 @@ +tests: + - description: testVectorLength + pipeline: + - Collection: vectors + - Select: + - AliasedExpr: + - Expr.vector_length: + - Field: embedding + - "embedding_length" + - Sort: + - Ordering: + - Field: embedding_length + - ASCENDING + assert_results: + - embedding_length: 3 + - embedding_length: 3 + - embedding_length: 3 + - embedding_length: 4 + - description: testFindNearestEuclidean + pipeline: + - Collection: vectors + - FindNearest: + field: embedding + vector: [1.0, 2.0, 3.0] + distance_measure: EUCLIDEAN + options: + FindNearestOptions: + limit: 2 + distance_field: + Field: distance + - Select: + - distance + assert_results: + - distance: 0.0 + - distance: 1.0 + assert_proto: + pipeline: + stages: + - name: collection + args: + - referenceValue: /vectors + - name: find_nearest + args: + - fieldReferenceValue: embedding + - mapValue: + fields: + __type__: + stringValue: __vector__ + value: + arrayValue: + values: + - doubleValue: 1.0 + - doubleValue: 2.0 + - doubleValue: 3.0 + - stringValue: euclidean + options: + limit: + integerValue: '2' + distance_field: + fieldReferenceValue: distance + - name: select + args: + - mapValue: + fields: + distance: + fieldReferenceValue: distance + - description: testFindNearestDotProduct + pipeline: + - Collection: vectors + - FindNearest: + field: embedding + vector: [1.0, 2.0, 3.0] + distance_measure: DOT_PRODUCT + options: + FindNearestOptions: + limit: 3 + distance_field: + Field: distance + - Select: + - distance + assert_results: + - distance: 38.0 + - distance: 17.0 + - distance: 14.0 + assert_proto: + pipeline: + stages: + - name: collection + args: + - referenceValue: /vectors + - name: find_nearest + args: + - fieldReferenceValue: embedding + - mapValue: + fields: + __type__: + stringValue: __vector__ + value: + arrayValue: + values: + - doubleValue: 1.0 + - doubleValue: 2.0 + - doubleValue: 3.0 + - stringValue: dot_product + options: + limit: + integerValue: '3' + distance_field: + fieldReferenceValue: distance + - name: select + args: + - mapValue: + fields: + distance: + fieldReferenceValue: distance + - description: testDotProductWithConstant + pipeline: + - Collection: vectors + - Where: + - Expr.equal: + - Field: embedding + - Vector: [1.0, 2.0, 3.0] + - Select: + - AliasedExpr: + - Expr.dot_product: + - Field: embedding + - Vector: [1.0, 1.0, 1.0] + - "dot_product_result" + assert_results: + - dot_product_result: 6.0 + - description: testEuclideanDistanceWithConstant + pipeline: + - Collection: vectors + - Where: + - Expr.equal: + - Field: embedding + - Vector: [1.0, 2.0, 3.0] + - Select: + - AliasedExpr: + - Expr.euclidean_distance: + - Field: embedding + - Vector: [1.0, 2.0, 3.0] + - "euclidean_distance_result" + assert_results: + - euclidean_distance_result: 0.0 + - description: testCosineDistanceWithConstant + pipeline: + - Collection: vectors + - Where: + - Expr.equal: + - Field: embedding + - Vector: [1.0, 2.0, 3.0] + - Select: + - AliasedExpr: + - Expr.cosine_distance: + - Field: embedding + - Vector: [1.0, 2.0, 3.0] + - "cosine_distance_result" + assert_results: + - cosine_distance_result: 0.0 From 1d48d4d970ea5c7fbf51c417a669eac56dbbe918 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 27 Oct 2025 22:01:53 -0700 Subject: [PATCH 22/36] removed duplicates --- tests/system/pipeline_e2e/aggregates.yaml | 269 ++++++++++++++++++ .../pipeline_e2e/{tests.yaml => general.yaml} | 191 ------------- 2 files changed, 269 insertions(+), 191 deletions(-) rename tests/system/pipeline_e2e/{tests.yaml => general.yaml} (85%) diff --git a/tests/system/pipeline_e2e/aggregates.yaml b/tests/system/pipeline_e2e/aggregates.yaml index e69de29bb..18902aff4 100644 --- a/tests/system/pipeline_e2e/aggregates.yaml +++ b/tests/system/pipeline_e2e/aggregates.yaml @@ -0,0 +1,269 @@ +tests: + - description: "testAggregates - count" + pipeline: + - Collection: books + - Aggregate: + - AliasedExpr: + - Expr.count: + - Field: rating + - "count" + assert_results: + - count: 10 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + count: + functionValue: + name: count + args: + - fieldReferenceValue: rating + - mapValue: {} + name: aggregate + - description: "testAggregates - count_if" + pipeline: + - Collection: books + - Aggregate: + - AliasedExpr: + - Expr.count_if: + - Expr.greater_than: + - Field: rating + - Constant: 4.2 + - "count_if_rating_gt_4_2" + assert_results: + - count_if_rating_gt_4_2: 5 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + count_if_rating_gt_4_2: + functionValue: + name: count_if + args: + - functionValue: + name: greater_than + args: + - fieldReferenceValue: rating + - doubleValue: 4.2 + - mapValue: {} + name: aggregate + - description: "testAggregates - count_distinct" + pipeline: + - Collection: books + - Aggregate: + - AliasedExpr: + - Expr.count_distinct: + - Field: genre + - "distinct_genres" + assert_results: + - distinct_genres: 8 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + distinct_genres: + functionValue: + name: count_distinct + args: + - fieldReferenceValue: genre + - mapValue: {} + name: aggregate + - description: "testAggregates - avg, count, max" + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: genre + - Constant: Science Fiction + - Aggregate: + - AliasedExpr: + - Expr.count: + - Field: rating + - "count" + - AliasedExpr: + - Expr.average: + - Field: rating + - "avg_rating" + - AliasedExpr: + - Expr.maximum: + - Field: rating + - "max_rating" + assert_results: + - count: 2 + avg_rating: 4.4 + max_rating: 4.6 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: genre + - stringValue: Science Fiction + name: equal + name: where + - args: + - mapValue: + fields: + avg_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: average + count: + functionValue: + name: count + args: + - fieldReferenceValue: rating + max_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: maximum + - mapValue: {} + name: aggregate + - description: testGroupBysWithoutAccumulators + pipeline: + - Collection: books + - Where: + - Expr.less_than: + - Field: published + - Constant: 1900 + - Aggregate: + accumulators: [] + groups: [genre] + assert_error: ".* requires at least one accumulator" + - description: testGroupBysAndAggregate + pipeline: + - Collection: books + - Where: + - Expr.less_than: + - Field: published + - Constant: 1984 + - Aggregate: + accumulators: + - AliasedExpr: + - Expr.average: + - Field: rating + - "avg_rating" + groups: [genre] + - Where: + - Expr.greater_than: + - Field: avg_rating + - Constant: 4.3 + - Sort: + - Ordering: + - Field: avg_rating + - ASCENDING + assert_results: + - avg_rating: 4.4 + genre: Science Fiction + - avg_rating: 4.5 + genre: Romance + - avg_rating: 4.7 + genre: Fantasy + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: published + - integerValue: '1984' + name: less_than + name: where + - args: + - mapValue: + fields: + avg_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: average + - mapValue: + fields: + genre: + fieldReferenceValue: genre + name: aggregate + - args: + - functionValue: + args: + - fieldReferenceValue: avg_rating + - doubleValue: 4.3 + name: greater_than + name: where + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: avg_rating + name: sort + - description: testMinMax + pipeline: + - Collection: books + - Aggregate: + - AliasedExpr: + - Expr.count: + - Field: rating + - "count" + - AliasedExpr: + - Expr.maximum: + - Field: rating + - "max_rating" + - AliasedExpr: + - Expr.minimum: + - Field: published + - "min_published" + assert_results: + - count: 10 + max_rating: 4.7 + min_published: 1813 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + count: + functionValue: + args: + - fieldReferenceValue: rating + name: count + max_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: maximum + min_published: + functionValue: + args: + - fieldReferenceValue: published + name: minimum + - mapValue: {} + name: aggregate \ No newline at end of file diff --git a/tests/system/pipeline_e2e/tests.yaml b/tests/system/pipeline_e2e/general.yaml similarity index 85% rename from tests/system/pipeline_e2e/tests.yaml rename to tests/system/pipeline_e2e/general.yaml index 5c959eb74..d01ec6662 100644 --- a/tests/system/pipeline_e2e/tests.yaml +++ b/tests/system/pipeline_e2e/general.yaml @@ -406,63 +406,6 @@ tests: - Pipeline: - Collection: books assert_count: 20 # Results will be duplicated - - - description: testGreaterThanOrEqual - pipeline: - - Collection: books - - Where: - - Expr.greater_than_or_equal: - - Field: rating - - Constant: 4.6 - - Select: - - title - - rating - - Sort: - - Ordering: - - Field: rating - - ASCENDING - assert_results: - - title: Dune - rating: 4.6 - - title: The Lord of the Rings - rating: 4.7 - - description: testInAndNotIn - pipeline: - - Collection: books - - Where: - - And: - - Expr.equal_any: - - Field: genre - - - Constant: Romance - - Constant: Dystopian - - Expr.not_equal_any: - - Field: author - - - Constant: "George Orwell" - assert_results: - - title: "Pride and Prejudice" - author: "Jane Austen" - genre: "Romance" - published: 1813 - rating: 4.5 - tags: - - classic - - social commentary - - love - awards: - none: true - - title: "The Handmaid's Tale" - author: "Margaret Atwood" - genre: "Dystopian" - published: 1985 - rating: 4.1 - tags: - - feminism - - totalitarianism - - resistance - awards: - "arthur c. clarke": true - "booker prize": false - - description: testDocumentId pipeline: - Collection: books @@ -525,59 +468,6 @@ tests: - "collectionName" assert_results: - collectionName: "books" - - description: testXor - pipeline: - - Collection: books - - Where: - - Xor: - - - Expr.equal: - - Field: genre - - Constant: Romance - - Expr.greater_than: - - Field: published - - Constant: 1980 - - Select: - - title - - genre - - published - - Sort: - - Ordering: - - Field: title - - ASCENDING - assert_results: - - title: "Pride and Prejudice" - genre: "Romance" - published: 1813 - - title: "The Handmaid's Tale" - genre: "Dystopian" - published: 1985 - - description: testConditional - pipeline: - - Collection: books - - Select: - - title - - AliasedExpr: - - Conditional: - - Expr.greater_than: - - Field: published - - Constant: 1950 - - Constant: "Modern" - - Constant: "Classic" - - "era" - - Sort: - - Ordering: - - Field: title - - ASCENDING - - Limit: 4 - assert_results: - - title: "1984" - era: "Classic" - - title: "Crime and Punishment" - era: "Classic" - - title: "Dune" - era: "Modern" - - title: "One Hundred Years of Solitude" - era: "Modern" - description: testFieldToFieldArithmetic pipeline: - Collection: books @@ -593,50 +483,6 @@ tests: - "pub_plus_rating" assert_results: - pub_plus_rating: 1969.6 - - description: testFieldToFieldComparison - pipeline: - - Collection: books - - Where: - - Expr.greater_than: - - Field: published - - Field: rating - - Select: - - title - assert_count: 10 # All books were published after year 4.7 - - description: testExistsNegative - pipeline: - - Collection: books - - Where: - - Expr.exists: - - Field: non_existent_field - assert_count: 0 - - description: testConditionalWithFields - pipeline: - - Collection: books - - Where: - - Expr.equal_any: - - Field: title - - - Constant: "Dune" - - Constant: "1984" - - Select: - - title - - AliasedExpr: - - Conditional: - - Expr.greater_than: - - Field: published - - Constant: 1950 - - Field: author - - Field: genre - - "conditional_field" - - Sort: - - Ordering: - - Field: title - - ASCENDING - assert_results: - - title: "1984" - conditional_field: "Dystopian" - - title: "Dune" - conditional_field: "Frank Herbert" - description: testMathExpressions pipeline: - Collection: books @@ -945,43 +791,6 @@ tests: fieldReferenceValue: title name: sort - - description: testSampleWithLimit - pipeline: - - Collection: books - - Sample: - limit_or_options: 3 - assert_count: 3 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - integerValue: '3' - - stringValue: documents - name: sample - - - description: testSampleWithPercentage - pipeline: - - Collection: books - - Sample: - limit_or_options: - SampleOptions: - value: 1 - mode: "percent" - assert_count: 10 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - doubleValue: 1.0 - - stringValue: percent - name: sample - - description: testDatabase pipeline: - Database From 390ea729ba5651af1989041f1da0dd6e679ca2a5 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 27 Oct 2025 22:08:12 -0700 Subject: [PATCH 23/36] added math file --- tests/system/pipeline_e2e/general.yaml | 198 ---------------- tests/system/pipeline_e2e/math.yaml | 309 +++++++++++++++++++++++++ 2 files changed, 309 insertions(+), 198 deletions(-) create mode 100644 tests/system/pipeline_e2e/math.yaml diff --git a/tests/system/pipeline_e2e/general.yaml b/tests/system/pipeline_e2e/general.yaml index d01ec6662..c135853d1 100644 --- a/tests/system/pipeline_e2e/general.yaml +++ b/tests/system/pipeline_e2e/general.yaml @@ -468,204 +468,6 @@ tests: - "collectionName" assert_results: - collectionName: "books" - - description: testFieldToFieldArithmetic - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: title - - Constant: "Dune" - - Select: - - AliasedExpr: - - Expr.add: - - Field: published - - Field: rating - - "pub_plus_rating" - assert_results: - - pub_plus_rating: 1969.6 - - description: testMathExpressions - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: title - - Constant: To Kill a Mockingbird - - Select: - - AliasedExpr: - - Expr.abs: - - Field: rating - - "abs_rating" - - AliasedExpr: - - Expr.ceil: - - Field: rating - - "ceil_rating" - - AliasedExpr: - - Expr.exp: - - Field: rating - - "exp_rating" - - AliasedExpr: - - Expr.floor: - - Field: rating - - "floor_rating" - - AliasedExpr: - - Expr.ln: - - Field: rating - - "ln_rating" - - AliasedExpr: - - Expr.log10: - - Field: rating - - "log_rating_base10" - - AliasedExpr: - - Expr.log: - - Field: rating - - Constant: 2 - - "log_rating_base2" - - AliasedExpr: - - Expr.pow: - - Field: rating - - Constant: 2 - - "pow_rating" - - AliasedExpr: - - Expr.sqrt: - - Field: rating - - "sqrt_rating" - assert_results_approximate: - - abs_rating: 4.2 - ceil_rating: 5.0 - exp_rating: 66.686331 - floor_rating: 4.0 - ln_rating: 1.4350845 - log_rating_base10: 0.623249 - log_rating_base2: 2.0704 - pow_rating: 17.64 - sqrt_rating: 2.049390 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: title - - stringValue: To Kill a Mockingbird - name: equal - name: where - - args: - - mapValue: - fields: - abs_rating: - functionValue: - args: - - fieldReferenceValue: rating - name: abs - ceil_rating: - functionValue: - args: - - fieldReferenceValue: rating - name: ceil - exp_rating: - functionValue: - args: - - fieldReferenceValue: rating - name: exp - floor_rating: - functionValue: - args: - - fieldReferenceValue: rating - name: floor - ln_rating: - functionValue: - args: - - fieldReferenceValue: rating - name: ln - log_rating_base10: - functionValue: - args: - - fieldReferenceValue: rating - name: log10 - log_rating_base2: - functionValue: - args: - - fieldReferenceValue: rating - - integerValue: '2' - name: log - pow_rating: - functionValue: - args: - - fieldReferenceValue: rating - - integerValue: '2' - name: pow - sqrt_rating: - functionValue: - args: - - fieldReferenceValue: rating - name: sqrt - name: select - - description: testRoundExpressions - pipeline: - - Collection: books - - Where: - - Expr.equal_any: - - Field: title - - - Constant: "To Kill a Mockingbird" # rating 4.2 - - Constant: "Pride and Prejudice" # rating 4.5 - - Constant: "The Lord of the Rings" # rating 4.7 - - Select: - - title - - AliasedExpr: - - Expr.round: - - Field: rating - - "round_rating" - - Sort: - - Ordering: - - Field: title - - ASCENDING - assert_results: - - title: "Pride and Prejudice" - round_rating: 5.0 - - title: "The Lord of the Rings" - round_rating: 5.0 - - title: "To Kill a Mockingbird" - round_rating: 4.0 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: title - - functionValue: - args: - - stringValue: "To Kill a Mockingbird" - - stringValue: "Pride and Prejudice" - - stringValue: "The Lord of the Rings" - name: array - name: equal_any - name: where - - args: - - mapValue: - fields: - title: - fieldReferenceValue: title - round_rating: - functionValue: - args: - - fieldReferenceValue: rating - name: round - name: select - - args: - - mapValue: - fields: - direction: - stringValue: ascending - expression: - fieldReferenceValue: title - name: sort - description: testCollectionGroup pipeline: - CollectionGroup: books diff --git a/tests/system/pipeline_e2e/math.yaml b/tests/system/pipeline_e2e/math.yaml new file mode 100644 index 000000000..a5a47d4c0 --- /dev/null +++ b/tests/system/pipeline_e2e/math.yaml @@ -0,0 +1,309 @@ +tests: + - description: testFieldToFieldArithmetic + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: "Dune" + - Select: + - AliasedExpr: + - Expr.add: + - Field: published + - Field: rating + - "pub_plus_rating" + assert_results: + - pub_plus_rating: 1969.6 + - description: testMathExpressions + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: To Kill a Mockingbird + - Select: + - AliasedExpr: + - Expr.abs: + - Field: rating + - "abs_rating" + - AliasedExpr: + - Expr.ceil: + - Field: rating + - "ceil_rating" + - AliasedExpr: + - Expr.exp: + - Field: rating + - "exp_rating" + - AliasedExpr: + - Expr.floor: + - Field: rating + - "floor_rating" + - AliasedExpr: + - Expr.ln: + - Field: rating + - "ln_rating" + - AliasedExpr: + - Expr.log10: + - Field: rating + - "log_rating_base10" + - AliasedExpr: + - Expr.log: + - Field: rating + - Constant: 2 + - "log_rating_base2" + - AliasedExpr: + - Expr.pow: + - Field: rating + - Constant: 2 + - "pow_rating" + - AliasedExpr: + - Expr.sqrt: + - Field: rating + - "sqrt_rating" + assert_results_approximate: + - abs_rating: 4.2 + ceil_rating: 5.0 + exp_rating: 66.686331 + floor_rating: 4.0 + ln_rating: 1.4350845 + log_rating_base10: 0.623249 + log_rating_base2: 2.0704 + pow_rating: 17.64 + sqrt_rating: 2.049390 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: To Kill a Mockingbird + name: equal + name: where + - args: + - mapValue: + fields: + abs_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: abs + ceil_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: ceil + exp_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: exp + floor_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: floor + ln_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: ln + log_rating_base10: + functionValue: + args: + - fieldReferenceValue: rating + name: log10 + log_rating_base2: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '2' + name: log + pow_rating: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '2' + name: pow + sqrt_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: sqrt + name: select + - description: testRoundExpressions + pipeline: + - Collection: books + - Where: + - Expr.equal_any: + - Field: title + - - Constant: "To Kill a Mockingbird" # rating 4.2 + - Constant: "Pride and Prejudice" # rating 4.5 + - Constant: "The Lord of the Rings" # rating 4.7 + - Select: + - title + - AliasedExpr: + - Expr.round: + - Field: rating + - "round_rating" + - Sort: + - Ordering: + - Field: title + - ASCENDING + assert_results: + - title: "Pride and Prejudice" + round_rating: 5.0 + - title: "The Lord of the Rings" + round_rating: 5.0 + - title: "To Kill a Mockingbird" + round_rating: 4.0 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - functionValue: + args: + - stringValue: "To Kill a Mockingbird" + - stringValue: "Pride and Prejudice" + - stringValue: "The Lord of the Rings" + name: array + name: equal_any + name: where + - args: + - mapValue: + fields: + title: + fieldReferenceValue: title + round_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: round + name: select + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: title + name: sort + - description: testArithmeticOperations + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: To Kill a Mockingbird + - Select: + - AliasedExpr: + - Expr.add: + - Field: rating + - Constant: 1 + - "ratingPlusOne" + - AliasedExpr: + - Expr.subtract: + - Field: published + - Constant: 1900 + - "yearsSince1900" + - AliasedExpr: + - Expr.multiply: + - Field: rating + - Constant: 10 + - "ratingTimesTen" + - AliasedExpr: + - Expr.divide: + - Field: rating + - Constant: 2 + - "ratingDividedByTwo" + - AliasedExpr: + - Expr.multiply: + - Field: rating + - Constant: 20 + - "ratingTimes20" + - AliasedExpr: + - Expr.add: + - Field: rating + - Constant: 3 + - "ratingPlus3" + - AliasedExpr: + - Expr.mod: + - Field: rating + - Constant: 2 + - "ratingMod2" + assert_results: + - ratingPlusOne: 5.2 + yearsSince1900: 60 + ratingTimesTen: 42.0 + ratingDividedByTwo: 2.1 + ratingTimes20: 84 + ratingPlus3: 7.2 + ratingMod2: 0.20000000000000018 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: To Kill a Mockingbird + name: equal + name: where + - args: + - mapValue: + fields: + ratingDividedByTwo: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '2' + name: divide + ratingPlusOne: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '1' + name: add + ratingTimesTen: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '10' + name: multiply + yearsSince1900: + functionValue: + args: + - fieldReferenceValue: published + - integerValue: '1900' + name: subtract + ratingTimes20: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '20' + name: multiply + ratingPlus3: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '3' + name: add + ratingMod2: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '2' + name: mod + name: select \ No newline at end of file From 7c2f41f4709e000339a67f4db1321ddc2c0b4e4e Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 27 Oct 2025 22:27:32 -0700 Subject: [PATCH 24/36] renamed tests --- tests/system/pipeline_e2e/string.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/system/pipeline_e2e/string.yaml b/tests/system/pipeline_e2e/string.yaml index 783a0c919..b1e3a0b64 100644 --- a/tests/system/pipeline_e2e/string.yaml +++ b/tests/system/pipeline_e2e/string.yaml @@ -240,7 +240,7 @@ tests: expression: fieldReferenceValue: title name: sort - - description: testStringFunctions - CharLength + - description: CharLength pipeline: - Collection: books - Where: @@ -276,7 +276,7 @@ tests: - fieldReferenceValue: title name: char_length name: select - - description: testStringFunctions - ByteLength + - description: ByteLength pipeline: - Collection: books - Where: @@ -384,7 +384,7 @@ tests: - title assert_results: - title: "The Hitchhiker's Guide to the Galaxy" - - description: testStringFunctions - ToLower + - description: ToLower pipeline: - Collection: books - Where: @@ -420,7 +420,7 @@ tests: - fieldReferenceValue: title name: to_lower name: select - - description: testStringFunctions - ToUpper + - description: ToUpper pipeline: - Collection: books - Where: @@ -456,7 +456,7 @@ tests: - fieldReferenceValue: title name: to_upper name: select - - description: testStringFunctions - Trim + - description: Trim pipeline: - Collection: books - Where: @@ -500,7 +500,7 @@ tests: name: string_concat name: trim name: select - - description: testStringFunctions - StringReverse + - description: StringReverse pipeline: - Collection: books - Where: @@ -536,7 +536,7 @@ tests: - fieldReferenceValue: title name: string_reverse name: select - - description: testStringFunctions - Substring + - description: Substring pipeline: - Collection: books - Where: @@ -576,7 +576,7 @@ tests: - integerValue: '11' name: substring name: select - - description: testStringFunctions - Substring without length + - description: Substring without length pipeline: - Collection: books - Where: @@ -614,7 +614,7 @@ tests: - integerValue: '10' name: substring name: select - - description: testStringFunctions - Join + - description: Join pipeline: - Collection: books - Where: From 41ff06d347915929d8cbd8e319f8c39a51ee5913 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 27 Oct 2025 22:27:47 -0700 Subject: [PATCH 25/36] include test file in test name --- tests/system/test_pipeline_acceptance.py | 26 ++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/tests/system/test_pipeline_acceptance.py b/tests/system/test_pipeline_acceptance.py index 996ebfcb4..2c635d585 100644 --- a/tests/system/test_pipeline_acceptance.py +++ b/tests/system/test_pipeline_acceptance.py @@ -39,8 +39,9 @@ test_dir_name = os.path.dirname(__file__) +id_format = lambda x: f"{x.get('file_name', '')}: {x.get('description', '')}" -def yaml_loader(field="tests", dir_name="pipeline_e2e"): +def yaml_loader(field="tests", dir_name="pipeline_e2e", attach_file_name=True): """ Helper to load test cases or data from yaml file """ @@ -48,7 +49,16 @@ def yaml_loader(field="tests", dir_name="pipeline_e2e"): for file_name in os.listdir(f"{test_dir_name}/{dir_name}"): with open(f"{test_dir_name}/{dir_name}/{file_name}") as f: new_yaml = yaml.safe_load(f) - extracted = new_yaml.get(field, None) if new_yaml else None + assert new_yaml is not None, f"found empty yaml in {file_name}" + extracted = new_yaml.get(field, None) + # attach file_name field + if attach_file_name: + if isinstance(extracted, list): + for item in extracted: + item["file_name"] = file_name + elif isinstance(extracted, dict): + extracted["file_name"] = file_name + # aggregate files if not combined_yaml: combined_yaml = extracted elif isinstance(combined_yaml, dict) and extracted: @@ -61,7 +71,7 @@ def yaml_loader(field="tests", dir_name="pipeline_e2e"): @pytest.mark.parametrize( "test_dict", [t for t in yaml_loader() if "assert_proto" in t], - ids=lambda x: f"{x.get('description', '')}", + ids=id_format, ) def test_pipeline_parse_proto(test_dict, client): """ @@ -78,7 +88,7 @@ def test_pipeline_parse_proto(test_dict, client): @pytest.mark.parametrize( "test_dict", [t for t in yaml_loader() if "assert_error" in t], - ids=lambda x: f"{x.get('description', '')}", + ids=id_format, ) def test_pipeline_expected_errors(test_dict, client): """ @@ -103,7 +113,7 @@ def test_pipeline_expected_errors(test_dict, client): or "assert_count" in t or "assert_results_approximate" in t ], - ids=lambda x: f"{x.get('description', '')}", + ids=id_format, ) def test_pipeline_results(test_dict, client): """ @@ -134,7 +144,7 @@ def test_pipeline_results(test_dict, client): @pytest.mark.parametrize( "test_dict", [t for t in yaml_loader() if "assert_error" in t], - ids=lambda x: f"{x.get('description', '')}", + ids=id_format, ) @pytest.mark.asyncio async def test_pipeline_expected_errors_async(test_dict, async_client): @@ -160,7 +170,7 @@ async def test_pipeline_expected_errors_async(test_dict, async_client): or "assert_count" in t or "assert_results_approximate" in t ], - ids=lambda x: f"{x.get('description', '')}", + ids=id_format, ) @pytest.mark.asyncio async def test_pipeline_results_async(test_dict, async_client): @@ -341,7 +351,7 @@ def client(): Build a client to use for requests """ client = Client(project=FIRESTORE_PROJECT, database=FIRESTORE_ENTERPRISE_DB) - data = yaml_loader("data") + data = yaml_loader("data", attach_file_name=False) to_delete = [] try: # setup data From d1dc233d194dfec4be638feba4c8d5c207de8894 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Mon, 27 Oct 2025 22:30:03 -0700 Subject: [PATCH 26/36] fixed lint --- tests/system/test_pipeline_acceptance.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/system/test_pipeline_acceptance.py b/tests/system/test_pipeline_acceptance.py index 2c635d585..3083b07a5 100644 --- a/tests/system/test_pipeline_acceptance.py +++ b/tests/system/test_pipeline_acceptance.py @@ -39,7 +39,10 @@ test_dir_name = os.path.dirname(__file__) -id_format = lambda x: f"{x.get('file_name', '')}: {x.get('description', '')}" +id_format = ( + lambda x: f"{x.get('file_name', '')}: {x.get('description', '')}" +) # noqa: E731 + def yaml_loader(field="tests", dir_name="pipeline_e2e", attach_file_name=True): """ From 7425832b32836274ac57387136e95c1c7fa04189 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 28 Oct 2025 15:37:12 -0700 Subject: [PATCH 27/36] fixed static function access --- google/cloud/firestore_v1/pipeline_expressions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/google/cloud/firestore_v1/pipeline_expressions.py b/google/cloud/firestore_v1/pipeline_expressions.py index 93ceca265..a2b59adb6 100644 --- a/google/cloud/firestore_v1/pipeline_expressions.py +++ b/google/cloud/firestore_v1/pipeline_expressions.py @@ -152,7 +152,7 @@ def static_func(self, first_arg, *other_args, **kwargs): def __get__(self, instance, owner): if instance is None: - return self.static_func.__get__(instance, owner) + return self.static_func else: return self.instance_func.__get__(instance, owner) From 086c3af3d162cd51d71c46619d1dfa8910668ca9 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 28 Oct 2025 15:37:22 -0700 Subject: [PATCH 28/36] ignore upgrade warning --- pytest.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index eac8ea123..308d1b494 100644 --- a/pytest.ini +++ b/pytest.ini @@ -22,4 +22,5 @@ filterwarnings = ignore:.*The \`credentials_file\` argument is deprecated.*:DeprecationWarning # Remove after updating test dependencies that use asyncio.iscoroutinefunction ignore:.*\'asyncio.iscoroutinefunction\' is deprecated.*:DeprecationWarning - ignore:.*\'asyncio.get_event_loop_policy\' is deprecated.*:DeprecationWarning \ No newline at end of file + ignore:.*\'asyncio.get_event_loop_policy\' is deprecated.*:DeprecationWarning + ignore:.*Please upgrade to the latest Python version.*:FutureWarning From ce2fab2e3c50c0bc4bbf0e5282e4c95fe9365cc5 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 28 Oct 2025 15:47:40 -0700 Subject: [PATCH 29/36] renamed Expr to Expression --- google/cloud/firestore_v1/_pipeline_stages.py | 20 +- google/cloud/firestore_v1/base_aggregation.py | 4 +- google/cloud/firestore_v1/base_pipeline.py | 28 +- .../firestore_v1/pipeline_expressions.py | 448 +++++++++--------- tests/unit/v1/test_pipeline_expressions.py | 309 ++++++------ 5 files changed, 391 insertions(+), 418 deletions(-) diff --git a/google/cloud/firestore_v1/_pipeline_stages.py b/google/cloud/firestore_v1/_pipeline_stages.py index 1cad362b1..2c202bbec 100644 --- a/google/cloud/firestore_v1/_pipeline_stages.py +++ b/google/cloud/firestore_v1/_pipeline_stages.py @@ -24,10 +24,10 @@ from google.cloud.firestore_v1.base_vector_query import DistanceMeasure from google.cloud.firestore_v1.pipeline_expressions import ( AggregateFunction, - Expr, - AliasedExpr, + Expression, + AliasedExpression, Field, - BooleanExpr, + BooleanExpression, Selectable, Ordering, ) @@ -164,8 +164,8 @@ class Aggregate(Stage): def __init__( self, - *args: AliasedExpr[AggregateFunction], - accumulators: Sequence[AliasedExpr[AggregateFunction]] = (), + *args: AliasedExpression[AggregateFunction], + accumulators: Sequence[AliasedExpression[AggregateFunction]] = (), groups: Sequence[str | Selectable] = (), ): super().__init__() @@ -271,13 +271,13 @@ class FindNearest(Stage): def __init__( self, - field: str | Expr, + field: str | Expression, vector: Sequence[float] | Vector, distance_measure: "DistanceMeasure" | str, options: Optional["FindNearestOptions"] = None, ): super().__init__("find_nearest") - self.field: Expr = Field(field) if isinstance(field, str) else field + self.field: Expression = Field(field) if isinstance(field, str) else field self.vector: Vector = vector if isinstance(vector, Vector) else Vector(vector) self.distance_measure = ( distance_measure @@ -305,10 +305,10 @@ def _pb_options(self) -> dict[str, Value]: class GenericStage(Stage): """Represents a generic, named stage with parameters.""" - def __init__(self, name: str, *params: Expr | Value): + def __init__(self, name: str, *params: Expression | Value): super().__init__(name) self.params: list[Value] = [ - p._to_pb() if isinstance(p, Expr) else p for p in params + p._to_pb() if isinstance(p, Expression) else p for p in params ] def _pb_args(self): @@ -443,7 +443,7 @@ def _pb_options(self): class Where(Stage): """Filters documents based on a specified condition.""" - def __init__(self, condition: BooleanExpr): + def __init__(self, condition: BooleanExpression): super().__init__() self.condition = condition diff --git a/google/cloud/firestore_v1/base_aggregation.py b/google/cloud/firestore_v1/base_aggregation.py index 3dd7a453e..d8d7cc6b4 100644 --- a/google/cloud/firestore_v1/base_aggregation.py +++ b/google/cloud/firestore_v1/base_aggregation.py @@ -36,7 +36,7 @@ ) from google.cloud.firestore_v1.pipeline_expressions import AggregateFunction from google.cloud.firestore_v1.pipeline_expressions import Count -from google.cloud.firestore_v1.pipeline_expressions import AliasedExpr +from google.cloud.firestore_v1.pipeline_expressions import AliasedExpression from google.cloud.firestore_v1.pipeline_expressions import Field # Types needed only for Type Hints @@ -86,7 +86,7 @@ def _to_protobuf(self): @abc.abstractmethod def _to_pipeline_expr( self, autoindexer: Iterable[int] - ) -> AliasedExpr[AggregateFunction]: + ) -> AliasedExpression[AggregateFunction]: """ Convert this instance to a pipeline expression for use with pipeline.aggregate() diff --git a/google/cloud/firestore_v1/base_pipeline.py b/google/cloud/firestore_v1/base_pipeline.py index 67bffd41e..608a3b74b 100644 --- a/google/cloud/firestore_v1/base_pipeline.py +++ b/google/cloud/firestore_v1/base_pipeline.py @@ -24,10 +24,10 @@ from google.cloud.firestore_v1.pipeline_result import PipelineResult from google.cloud.firestore_v1.pipeline_expressions import ( AggregateFunction, - AliasedExpr, - Expr, + AliasedExpression, + Expression, Field, - BooleanExpr, + BooleanExpression, Selectable, ) from google.cloud.firestore_v1 import _helpers @@ -147,7 +147,7 @@ def add_fields(self, *fields: Selectable) -> "_BasePipeline": The added fields are defined using `Selectable` expressions, which can be: - `Field`: References an existing document field. - `Function`: Performs a calculation using functions like `add`, - `multiply` with assigned aliases using `Expr.as_()`. + `multiply` with assigned aliases using `Expression.as_()`. Example: >>> from google.cloud.firestore_v1.pipeline_expressions import Field, add @@ -194,7 +194,7 @@ def select(self, *selections: str | Selectable) -> "_BasePipeline": The selected fields are defined using `Selectable` expressions or field names: - `Field`: References an existing document field. - `Function`: Represents the result of a function with an assigned alias - name using `Expr.as_()`. + name using `Expression.as_()`. - `str`: The name of an existing field. If no selections are provided, the output of this stage is empty. Use @@ -220,14 +220,14 @@ def select(self, *selections: str | Selectable) -> "_BasePipeline": """ return self._append(stages.Select(*selections)) - def where(self, condition: BooleanExpr) -> "_BasePipeline": + def where(self, condition: BooleanExpression) -> "_BasePipeline": """ Filters the documents from previous stages to only include those matching - the specified `BooleanExpr`. + the specified `BooleanExpression`. This stage allows you to apply conditions to the data, similar to a "WHERE" clause in SQL. You can filter documents based on their field values, using - implementations of `BooleanExpr`, typically including but not limited to: + implementations of `BooleanExpression`, typically including but not limited to: - field comparators: `eq`, `lt` (less than), `gt` (greater than), etc. - logical operators: `And`, `Or`, `Not`, etc. - advanced functions: `regex_matches`, `array_contains`, etc. @@ -252,7 +252,7 @@ def where(self, condition: BooleanExpr) -> "_BasePipeline": Args: - condition: The `BooleanExpr` to apply. + condition: The `BooleanExpression` to apply. Returns: A new Pipeline object with this stage appended to the stage list @@ -261,7 +261,7 @@ def where(self, condition: BooleanExpr) -> "_BasePipeline": def find_nearest( self, - field: str | Expr, + field: str | Expression, vector: Sequence[float] | "Vector", distance_measure: "DistanceMeasure", options: stages.FindNearestOptions | None = None, @@ -298,7 +298,7 @@ def find_nearest( ... ) Args: - field: The name of the field (str) or an expression (`Expr`) that + field: The name of the field (str) or an expression (`Expression`) that evaluates to the vector data. This field should store vector values. vector: The target vector (sequence of floats or `Vector` object) to compare against. @@ -458,7 +458,7 @@ def unnest( """ return self._append(stages.Unnest(field, alias, options)) - def generic_stage(self, name: str, *params: Expr) -> "_BasePipeline": + def generic_stage(self, name: str, *params: Expression) -> "_BasePipeline": """ Adds a generic, named stage to the pipeline with specified parameters. @@ -474,7 +474,7 @@ def generic_stage(self, name: str, *params: Expr) -> "_BasePipeline": Args: name: The name of the generic stage. - *params: A sequence of `Expr` objects representing the parameters for the stage. + *params: A sequence of `Expression` objects representing the parameters for the stage. Returns: A new Pipeline object with this stage appended to the stage list @@ -531,7 +531,7 @@ def limit(self, limit: int) -> "_BasePipeline": def aggregate( self, - *accumulators: AliasedExpr[AggregateFunction], + *accumulators: AliasedExpression[AggregateFunction], groups: Sequence[str | Selectable] = (), ) -> "_BasePipeline": """ diff --git a/google/cloud/firestore_v1/pipeline_expressions.py b/google/cloud/firestore_v1/pipeline_expressions.py index 38c5fdd4b..628640f4e 100644 --- a/google/cloud/firestore_v1/pipeline_expressions.py +++ b/google/cloud/firestore_v1/pipeline_expressions.py @@ -56,12 +56,12 @@ def __init__(self, expr, order_dir: Direction | str = Direction.ASCENDING): Initializes an Ordering instance Args: - expr (Expr | str): The expression or field path string to sort by. + expr (Expression | str): The expression or field path string to sort by. If a string is provided, it's treated as a field path. order_dir (Direction | str): The direction to sort in. Defaults to ascending """ - self.expr = expr if isinstance(expr, Expr) else Field.of(expr) + self.expr = expr if isinstance(expr, Expression) else Field.of(expr) self.order_dir = ( Ordering.Direction[order_dir.upper()] if isinstance(order_dir, str) @@ -86,11 +86,11 @@ def _to_pb(self) -> Value: ) -class Expr(ABC): +class Expression(ABC): """Represents an expression that can be evaluated to a value within the execution of a pipeline. - Expressions are the building blocks for creating complex queries and + Expressionessions are the building blocks for creating complex queries and transformations in Firestore pipelines. They can represent: - **Field references:** Access values from document fields. @@ -98,7 +98,7 @@ class Expr(ABC): - **Function calls:** Apply functions to one or more expressions. - **Aggregations:** Calculate aggregate values (e.g., sum, average) over a set of documents. - The `Expr` class provides a fluent API for building expressions. You can chain + The `Expression` class provides a fluent API for building expressions. You can chain together method calls to create complex expressions. """ @@ -110,11 +110,11 @@ def _to_pb(self) -> Value: raise NotImplementedError @staticmethod - def _cast_to_expr_or_convert_to_constant(o: Any, include_vector=False) -> "Expr": - """Convert arbitrary object to an Expr.""" + def _cast_to_expr_or_convert_to_constant(o: Any, include_vector=False) -> "Expression": + """Convert arbitrary object to an Expression.""" if isinstance(o, Constant) and isinstance(o.value, list): o = o.value - if isinstance(o, Expr): + if isinstance(o, Expression): return o if isinstance(o, dict): return Map(o) @@ -141,12 +141,12 @@ def __init__(self, instance_func): self.instance_func = instance_func def static_func(self, first_arg, *other_args, **kwargs): - if not isinstance(first_arg, (Expr, str)): + if not isinstance(first_arg, (Expression, str)): raise TypeError( - f"`expressions must be called on an Expr or a string representing a field name. got {type(first_arg)}." + f"`expressions must be called on an Expression or a string representing a field name. got {type(first_arg)}." ) first_expr = ( - Field.of(first_arg) if not isinstance(first_arg, Expr) else first_arg + Field.of(first_arg) if not isinstance(first_arg, Expression) else first_arg ) return self.instance_func(first_expr, *other_args, **kwargs) @@ -157,7 +157,7 @@ def __get__(self, instance, owner): return self.instance_func.__get__(instance, owner) @expose_as_static - def add(self, other: Expr | float) -> "Expr": + def add(self, other: Expression | float) -> "Expression": """Creates an expression that adds this expression to another expression or constant. Example: @@ -170,12 +170,12 @@ def add(self, other: Expr | float) -> "Expr": other: The expression or constant value to add to this expression. Returns: - A new `Expr` representing the addition operation. + A new `Expression` representing the addition operation. """ return Function("add", [self, self._cast_to_expr_or_convert_to_constant(other)]) @expose_as_static - def subtract(self, other: Expr | float) -> "Expr": + def subtract(self, other: Expression | float) -> "Expression": """Creates an expression that subtracts another expression or constant from this expression. Example: @@ -188,14 +188,14 @@ def subtract(self, other: Expr | float) -> "Expr": other: The expression or constant value to subtract from this expression. Returns: - A new `Expr` representing the subtraction operation. + A new `Expression` representing the subtraction operation. """ return Function( "subtract", [self, self._cast_to_expr_or_convert_to_constant(other)] ) @expose_as_static - def multiply(self, other: Expr | float) -> "Expr": + def multiply(self, other: Expression | float) -> "Expression": """Creates an expression that multiplies this expression by another expression or constant. Example: @@ -208,14 +208,14 @@ def multiply(self, other: Expr | float) -> "Expr": other: The expression or constant value to multiply by. Returns: - A new `Expr` representing the multiplication operation. + A new `Expression` representing the multiplication operation. """ return Function( "multiply", [self, self._cast_to_expr_or_convert_to_constant(other)] ) @expose_as_static - def divide(self, other: Expr | float) -> "Expr": + def divide(self, other: Expression | float) -> "Expression": """Creates an expression that divides this expression by another expression or constant. Example: @@ -228,14 +228,14 @@ def divide(self, other: Expr | float) -> "Expr": other: The expression or constant value to divide by. Returns: - A new `Expr` representing the division operation. + A new `Expression` representing the division operation. """ return Function( "divide", [self, self._cast_to_expr_or_convert_to_constant(other)] ) @expose_as_static - def mod(self, other: Expr | float) -> "Expr": + def mod(self, other: Expression | float) -> "Expression": """Creates an expression that calculates the modulo (remainder) to another expression or constant. Example: @@ -248,12 +248,12 @@ def mod(self, other: Expr | float) -> "Expr": other: The divisor expression or constant. Returns: - A new `Expr` representing the modulo operation. + A new `Expression` representing the modulo operation. """ return Function("mod", [self, self._cast_to_expr_or_convert_to_constant(other)]) @expose_as_static - def abs(self) -> "Expr": + def abs(self) -> "Expression": """Creates an expression that calculates the absolute value of this expression. Example: @@ -261,12 +261,12 @@ def abs(self) -> "Expr": >>> Field.of("change").abs() Returns: - A new `Expr` representing the absolute value. + A new `Expression` representing the absolute value. """ return Function("abs", [self]) @expose_as_static - def ceil(self) -> "Expr": + def ceil(self) -> "Expression": """Creates an expression that calculates the ceiling of this expression. Example: @@ -274,12 +274,12 @@ def ceil(self) -> "Expr": >>> Field.of("value").ceil() Returns: - A new `Expr` representing the ceiling value. + A new `Expression` representing the ceiling value. """ return Function("ceil", [self]) @expose_as_static - def exp(self) -> "Expr": + def exp(self) -> "Expression": """Creates an expression that computes e to the power of this expression. Example: @@ -287,12 +287,12 @@ def exp(self) -> "Expr": >>> Field.of("value").exp() Returns: - A new `Expr` representing the exponential value. + A new `Expression` representing the exponential value. """ return Function("exp", [self]) @expose_as_static - def floor(self) -> "Expr": + def floor(self) -> "Expression": """Creates an expression that calculates the floor of this expression. Example: @@ -300,12 +300,12 @@ def floor(self) -> "Expr": >>> Field.of("value").floor() Returns: - A new `Expr` representing the floor value. + A new `Expression` representing the floor value. """ return Function("floor", [self]) @expose_as_static - def ln(self) -> "Expr": + def ln(self) -> "Expression": """Creates an expression that calculates the natural logarithm of this expression. Example: @@ -313,12 +313,12 @@ def ln(self) -> "Expr": >>> Field.of("value").ln() Returns: - A new `Expr` representing the natural logarithm. + A new `Expression` representing the natural logarithm. """ return Function("ln", [self]) @expose_as_static - def log(self, base: Expr | float) -> "Expr": + def log(self, base: Expression | float) -> "Expression": """Creates an expression that calculates the logarithm of this expression with a given base. Example: @@ -331,24 +331,24 @@ def log(self, base: Expr | float) -> "Expr": base: The base of the logarithm. Returns: - A new `Expr` representing the logarithm. + A new `Expression` representing the logarithm. """ return Function("log", [self, self._cast_to_expr_or_convert_to_constant(base)]) @expose_as_static - def log10(self) -> "Expr": + def log10(self) -> "Expression": """Creates an expression that calculates the base 10 logarithm of this expression. Example: >>> Field.of("value").log10() Returns: - A new `Expr` representing the logarithm. + A new `Expression` representing the logarithm. """ return Function("log10", [self]) @expose_as_static - def pow(self, exponent: Expr | float) -> "Expr": + def pow(self, exponent: Expression | float) -> "Expression": """Creates an expression that calculates this expression raised to the power of the exponent. Example: @@ -361,14 +361,14 @@ def pow(self, exponent: Expr | float) -> "Expr": exponent: The exponent. Returns: - A new `Expr` representing the power operation. + A new `Expression` representing the power operation. """ return Function( "pow", [self, self._cast_to_expr_or_convert_to_constant(exponent)] ) @expose_as_static - def round(self) -> "Expr": + def round(self) -> "Expression": """Creates an expression that rounds this expression to the nearest integer. Example: @@ -376,12 +376,12 @@ def round(self) -> "Expr": >>> Field.of("value").round() Returns: - A new `Expr` representing the rounded value. + A new `Expression` representing the rounded value. """ return Function("round", [self]) @expose_as_static - def sqrt(self) -> "Expr": + def sqrt(self) -> "Expression": """Creates an expression that calculates the square root of this expression. Example: @@ -389,12 +389,12 @@ def sqrt(self) -> "Expr": >>> Field.of("area").sqrt() Returns: - A new `Expr` representing the square root. + A new `Expression` representing the square root. """ return Function("sqrt", [self]) @expose_as_static - def logical_maximum(self, other: Expr | CONSTANT_TYPE) -> "Expr": + def logical_maximum(self, other: Expression | CONSTANT_TYPE) -> "Expression": """Creates an expression that returns the larger value between this expression and another expression or constant, based on Firestore's value type ordering. @@ -411,7 +411,7 @@ def logical_maximum(self, other: Expr | CONSTANT_TYPE) -> "Expr": other: The other expression or constant value to compare with. Returns: - A new `Expr` representing the logical maximum operation. + A new `Expression` representing the logical maximum operation. """ return Function( "maximum", @@ -420,7 +420,7 @@ def logical_maximum(self, other: Expr | CONSTANT_TYPE) -> "Expr": ) @expose_as_static - def logical_minimum(self, other: Expr | CONSTANT_TYPE) -> "Expr": + def logical_minimum(self, other: Expression | CONSTANT_TYPE) -> "Expression": """Creates an expression that returns the smaller value between this expression and another expression or constant, based on Firestore's value type ordering. @@ -437,7 +437,7 @@ def logical_minimum(self, other: Expr | CONSTANT_TYPE) -> "Expr": other: The other expression or constant value to compare with. Returns: - A new `Expr` representing the logical minimum operation. + A new `Expression` representing the logical minimum operation. """ return Function( "minimum", @@ -446,7 +446,7 @@ def logical_minimum(self, other: Expr | CONSTANT_TYPE) -> "Expr": ) @expose_as_static - def equal(self, other: Expr | CONSTANT_TYPE) -> "BooleanExpr": + def equal(self, other: Expression | CONSTANT_TYPE) -> "BooleanExpression": """Creates an expression that checks if this expression is equal to another expression or constant value. @@ -460,14 +460,14 @@ def equal(self, other: Expr | CONSTANT_TYPE) -> "BooleanExpr": other: The expression or constant value to compare for equality. Returns: - A new `Expr` representing the equality comparison. + A new `Expression` representing the equality comparison. """ - return BooleanExpr( + return BooleanExpression( "equal", [self, self._cast_to_expr_or_convert_to_constant(other)] ) @expose_as_static - def not_equal(self, other: Expr | CONSTANT_TYPE) -> "BooleanExpr": + def not_equal(self, other: Expression | CONSTANT_TYPE) -> "BooleanExpression": """Creates an expression that checks if this expression is not equal to another expression or constant value. @@ -481,14 +481,14 @@ def not_equal(self, other: Expr | CONSTANT_TYPE) -> "BooleanExpr": other: The expression or constant value to compare for inequality. Returns: - A new `Expr` representing the inequality comparison. + A new `Expression` representing the inequality comparison. """ - return BooleanExpr( + return BooleanExpression( "not_equal", [self, self._cast_to_expr_or_convert_to_constant(other)] ) @expose_as_static - def greater_than(self, other: Expr | CONSTANT_TYPE) -> "BooleanExpr": + def greater_than(self, other: Expression | CONSTANT_TYPE) -> "BooleanExpression": """Creates an expression that checks if this expression is greater than another expression or constant value. @@ -502,14 +502,14 @@ def greater_than(self, other: Expr | CONSTANT_TYPE) -> "BooleanExpr": other: The expression or constant value to compare for greater than. Returns: - A new `Expr` representing the greater than comparison. + A new `Expression` representing the greater than comparison. """ - return BooleanExpr( + return BooleanExpression( "greater_than", [self, self._cast_to_expr_or_convert_to_constant(other)] ) @expose_as_static - def greater_than_or_equal(self, other: Expr | CONSTANT_TYPE) -> "BooleanExpr": + def greater_than_or_equal(self, other: Expression | CONSTANT_TYPE) -> "BooleanExpression": """Creates an expression that checks if this expression is greater than or equal to another expression or constant value. @@ -523,15 +523,15 @@ def greater_than_or_equal(self, other: Expr | CONSTANT_TYPE) -> "BooleanExpr": other: The expression or constant value to compare for greater than or equal to. Returns: - A new `Expr` representing the greater than or equal to comparison. + A new `Expression` representing the greater than or equal to comparison. """ - return BooleanExpr( + return BooleanExpression( "greater_than_or_equal", [self, self._cast_to_expr_or_convert_to_constant(other)], ) @expose_as_static - def less_than(self, other: Expr | CONSTANT_TYPE) -> "BooleanExpr": + def less_than(self, other: Expression | CONSTANT_TYPE) -> "BooleanExpression": """Creates an expression that checks if this expression is less than another expression or constant value. @@ -545,14 +545,14 @@ def less_than(self, other: Expr | CONSTANT_TYPE) -> "BooleanExpr": other: The expression or constant value to compare for less than. Returns: - A new `Expr` representing the less than comparison. + A new `Expression` representing the less than comparison. """ - return BooleanExpr( + return BooleanExpression( "less_than", [self, self._cast_to_expr_or_convert_to_constant(other)] ) @expose_as_static - def less_than_or_equal(self, other: Expr | CONSTANT_TYPE) -> "BooleanExpr": + def less_than_or_equal(self, other: Expression | CONSTANT_TYPE) -> "BooleanExpression": """Creates an expression that checks if this expression is less than or equal to another expression or constant value. @@ -566,17 +566,17 @@ def less_than_or_equal(self, other: Expr | CONSTANT_TYPE) -> "BooleanExpr": other: The expression or constant value to compare for less than or equal to. Returns: - A new `Expr` representing the less than or equal to comparison. + A new `Expression` representing the less than or equal to comparison. """ - return BooleanExpr( + return BooleanExpression( "less_than_or_equal", [self, self._cast_to_expr_or_convert_to_constant(other)], ) @expose_as_static def equal_any( - self, array: Array | Sequence[Expr | CONSTANT_TYPE] | Expr - ) -> "BooleanExpr": + self, array: Array | Sequence[Expression | CONSTANT_TYPE] | Expression + ) -> "BooleanExpression": """Creates an expression that checks if this expression is equal to any of the provided values or expressions. @@ -588,9 +588,9 @@ def equal_any( array: The values or expressions to check against. Returns: - A new `Expr` representing the 'IN' comparison. + A new `Expression` representing the 'IN' comparison. """ - return BooleanExpr( + return BooleanExpression( "equal_any", [ self, @@ -600,8 +600,8 @@ def equal_any( @expose_as_static def not_equal_any( - self, array: Array | list[Expr | CONSTANT_TYPE] | Expr - ) -> "BooleanExpr": + self, array: Array | list[Expression | CONSTANT_TYPE] | Expression + ) -> "BooleanExpression": """Creates an expression that checks if this expression is not equal to any of the provided values or expressions. @@ -613,9 +613,9 @@ def not_equal_any( array: The values or expressions to check against. Returns: - A new `Expr` representing the 'NOT IN' comparison. + A new `Expression` representing the 'NOT IN' comparison. """ - return BooleanExpr( + return BooleanExpression( "not_equal_any", [ self, @@ -624,7 +624,7 @@ def not_equal_any( ) @expose_as_static - def array_contains(self, element: Expr | CONSTANT_TYPE) -> "BooleanExpr": + def array_contains(self, element: Expression | CONSTANT_TYPE) -> "BooleanExpression": """Creates an expression that checks if an array contains a specific element or value. Example: @@ -637,17 +637,17 @@ def array_contains(self, element: Expr | CONSTANT_TYPE) -> "BooleanExpr": element: The element (expression or constant) to search for in the array. Returns: - A new `Expr` representing the 'array_contains' comparison. + A new `Expression` representing the 'array_contains' comparison. """ - return BooleanExpr( + return BooleanExpression( "array_contains", [self, self._cast_to_expr_or_convert_to_constant(element)] ) @expose_as_static def array_contains_all( self, - elements: Array | list[Expr | CONSTANT_TYPE] | Expr, - ) -> "BooleanExpr": + elements: Array | list[Expression | CONSTANT_TYPE] | Expression, + ) -> "BooleanExpression": """Creates an expression that checks if an array contains all the specified elements. Example: @@ -660,9 +660,9 @@ def array_contains_all( elements: The list of elements (expressions or constants) to check for in the array. Returns: - A new `Expr` representing the 'array_contains_all' comparison. + A new `Expression` representing the 'array_contains_all' comparison. """ - return BooleanExpr( + return BooleanExpression( "array_contains_all", [ self, @@ -673,8 +673,8 @@ def array_contains_all( @expose_as_static def array_contains_any( self, - elements: Array | list[Expr | CONSTANT_TYPE] | Expr, - ) -> "BooleanExpr": + elements: Array | list[Expression | CONSTANT_TYPE] | Expression, + ) -> "BooleanExpression": """Creates an expression that checks if an array contains any of the specified elements. Example: @@ -688,9 +688,9 @@ def array_contains_any( elements: The list of elements (expressions or constants) to check for in the array. Returns: - A new `Expr` representing the 'array_contains_any' comparison. + A new `Expression` representing the 'array_contains_any' comparison. """ - return BooleanExpr( + return BooleanExpression( "array_contains_any", [ self, @@ -699,7 +699,7 @@ def array_contains_any( ) @expose_as_static - def array_length(self) -> "Expr": + def array_length(self) -> "Expression": """Creates an expression that calculates the length of an array. Example: @@ -707,12 +707,12 @@ def array_length(self) -> "Expr": >>> Field.of("cart").array_length() Returns: - A new `Expr` representing the length of the array. + A new `Expression` representing the length of the array. """ return Function("array_length", [self]) @expose_as_static - def array_reverse(self) -> "Expr": + def array_reverse(self) -> "Expression": """Creates an expression that returns the reversed content of an array. Example: @@ -720,14 +720,14 @@ def array_reverse(self) -> "Expr": >>> Field.of("preferences").array_reverse() Returns: - A new `Expr` representing the reversed array. + A new `Expression` representing the reversed array. """ return Function("array_reverse", [self]) @expose_as_static def array_concat( - self, *other_arrays: Array | list[Expr | CONSTANT_TYPE] | Expr - ) -> "Expr": + self, *other_arrays: Array | list[Expression | CONSTANT_TYPE] | Expression + ) -> "Expression": """Creates an expression that concatenates an array expression with another array. Example: @@ -738,7 +738,7 @@ def array_concat( array: The list of constants or expressions to concat with. Returns: - A new `Expr` representing the concatenated array. + A new `Expression` representing the concatenated array. """ return Function( "array_concat", @@ -747,14 +747,14 @@ def array_concat( ) @expose_as_static - def concat(self, *others: Expr | CONSTANT_TYPE) -> "Expr": + def concat(self, *others: Expression | CONSTANT_TYPE) -> "Expression": """Creates an expression that concatenates expressions together Args: *others: The expressions to concatenate. Returns: - A new `Expr` representing the concatenated value. + A new `Expression` representing the concatenated value. """ return Function( "concat", @@ -762,7 +762,7 @@ def concat(self, *others: Expr | CONSTANT_TYPE) -> "Expr": ) @expose_as_static - def length(self) -> "Expr": + def length(self) -> "Expression": """ Creates an expression that calculates the length of the expression if it is a string, array, map, or blob. @@ -771,12 +771,12 @@ def length(self) -> "Expr": >>> Field.of("name").length() Returns: - A new `Expr` representing the length of the expression. + A new `Expression` representing the length of the expression. """ return Function("length", [self]) @expose_as_static - def is_absent(self) -> "BooleanExpr": + def is_absent(self) -> "BooleanExpression": """Creates an expression that returns true if a value is absent. Otherwise, returns false even if the value is null. @@ -785,12 +785,12 @@ def is_absent(self) -> "BooleanExpr": >>> Field.of("email").is_absent() Returns: - A new `BooleanExpression` representing the isAbsent operation. + A new `BooleanExpressionession` representing the isAbsent operation. """ - return BooleanExpr("is_absent", [self]) + return BooleanExpression("is_absent", [self]) @expose_as_static - def if_absent(self, default_value: Expr | CONSTANT_TYPE) -> "Expr": + def if_absent(self, default_value: Expression | CONSTANT_TYPE) -> "Expression": """Creates an expression that returns a default value if an expression evaluates to an absent value. Example: @@ -801,7 +801,7 @@ def if_absent(self, default_value: Expr | CONSTANT_TYPE) -> "Expr": default_value: The expression or constant value to return if this expression is absent. Returns: - A new `Expr` representing the ifAbsent operation. + A new `Expression` representing the ifAbsent operation. """ return Function( "if_absent", @@ -809,7 +809,7 @@ def if_absent(self, default_value: Expr | CONSTANT_TYPE) -> "Expr": ) @expose_as_static - def is_nan(self) -> "BooleanExpr": + def is_nan(self) -> "BooleanExpression": """Creates an expression that checks if this expression evaluates to 'NaN' (Not a Number). Example: @@ -817,12 +817,12 @@ def is_nan(self) -> "BooleanExpr": >>> Field.of("value").divide(0).is_nan() Returns: - A new `Expr` representing the 'isNaN' check. + A new `Expression` representing the 'isNaN' check. """ - return BooleanExpr("is_nan", [self]) + return BooleanExpression("is_nan", [self]) @expose_as_static - def is_not_nan(self) -> "BooleanExpr": + def is_not_nan(self) -> "BooleanExpression": """Creates an expression that checks if this expression evaluates to a non-'NaN' (Not a Number) value. Example: @@ -830,33 +830,33 @@ def is_not_nan(self) -> "BooleanExpr": >>> Field.of("value").divide(1).is_not_nan() Returns: - A new `Expr` representing the 'is not NaN' check. + A new `Expression` representing the 'is not NaN' check. """ - return BooleanExpr("is_not_nan", [self]) + return BooleanExpression("is_not_nan", [self]) @expose_as_static - def is_null(self) -> "BooleanExpr": + def is_null(self) -> "BooleanExpression": """Creates an expression that checks if the value of a field is 'Null'. Example: >>> Field.of("value").is_null() Returns: - A new `Expr` representing the 'isNull' check. + A new `Expression` representing the 'isNull' check. """ - return BooleanExpr("is_null", [self]) + return BooleanExpression("is_null", [self]) @expose_as_static - def is_not_null(self) -> "BooleanExpr": + def is_not_null(self) -> "BooleanExpression": """Creates an expression that checks if the value of a field is not 'Null'. Example: >>> Field.of("value").is_not_null() Returns: - A new `Expr` representing the 'isNotNull' check. + A new `Expression` representing the 'isNotNull' check. """ - return BooleanExpr("is_not_null", [self]) + return BooleanExpression("is_not_null", [self]) @expose_as_static def is_error(self): @@ -867,12 +867,12 @@ def is_error(self): >>> Field.of("value").divide("string").is_error() Returns: - A new `Expr` representing the isError operation. + A new `Expression` representing the isError operation. """ return Function("is_error", [self]) @expose_as_static - def if_error(self, then_value: Expr | CONSTANT_TYPE) -> "Expr": + def if_error(self, then_value: Expression | CONSTANT_TYPE) -> "Expression": """Creates an expression that returns ``then_value`` if this expression evaluates to an error. Otherwise, returns the value of this expression. @@ -884,14 +884,14 @@ def if_error(self, then_value: Expr | CONSTANT_TYPE) -> "Expr": then_value: The value to return if this expression evaluates to an error. Returns: - A new `Expr` representing the ifError operation. + A new `Expression` representing the ifError operation. """ return Function( "if_error", [self, self._cast_to_expr_or_convert_to_constant(then_value)] ) @expose_as_static - def exists(self) -> "BooleanExpr": + def exists(self) -> "BooleanExpression": """Creates an expression that checks if a field exists in the document. Example: @@ -899,12 +899,12 @@ def exists(self) -> "BooleanExpr": >>> Field.of("phoneNumber").exists() Returns: - A new `Expr` representing the 'exists' check. + A new `Expression` representing the 'exists' check. """ - return BooleanExpr("exists", [self]) + return BooleanExpression("exists", [self]) @expose_as_static - def sum(self) -> "Expr": + def sum(self) -> "Expression": """Creates an aggregation that calculates the sum of a numeric field across multiple stage inputs. Example: @@ -917,7 +917,7 @@ def sum(self) -> "Expr": return AggregateFunction("sum", [self]) @expose_as_static - def average(self) -> "Expr": + def average(self) -> "Expression": """Creates an aggregation that calculates the average (mean) of a numeric field across multiple stage inputs. @@ -931,7 +931,7 @@ def average(self) -> "Expr": return AggregateFunction("average", [self]) @expose_as_static - def count(self) -> "Expr": + def count(self) -> "Expression": """Creates an aggregation that counts the number of stage inputs with valid evaluations of the expression or field. @@ -945,7 +945,7 @@ def count(self) -> "Expr": return AggregateFunction("count", [self]) @expose_as_static - def count_if(self) -> "Expr": + def count_if(self) -> "Expression": """Creates an aggregation that counts the number of values of the provided field or expression that evaluate to True. @@ -960,7 +960,7 @@ def count_if(self) -> "Expr": return AggregateFunction("count_if", [self]) @expose_as_static - def count_distinct(self) -> "Expr": + def count_distinct(self) -> "Expression": """Creates an aggregation that counts the number of distinct values of the provided field or expression. @@ -974,7 +974,7 @@ def count_distinct(self) -> "Expr": return AggregateFunction("count_distinct", [self]) @expose_as_static - def minimum(self) -> "Expr": + def minimum(self) -> "Expression": """Creates an aggregation that finds the minimum value of a field across multiple stage inputs. Example: @@ -987,7 +987,7 @@ def minimum(self) -> "Expr": return AggregateFunction("minimum", [self]) @expose_as_static - def maximum(self) -> "Expr": + def maximum(self) -> "Expression": """Creates an aggregation that finds the maximum value of a field across multiple stage inputs. Example: @@ -1000,7 +1000,7 @@ def maximum(self) -> "Expr": return AggregateFunction("maximum", [self]) @expose_as_static - def char_length(self) -> "Expr": + def char_length(self) -> "Expression": """Creates an expression that calculates the character length of a string. Example: @@ -1008,12 +1008,12 @@ def char_length(self) -> "Expr": >>> Field.of("name").char_length() Returns: - A new `Expr` representing the length of the string. + A new `Expression` representing the length of the string. """ return Function("char_length", [self]) @expose_as_static - def byte_length(self) -> "Expr": + def byte_length(self) -> "Expression": """Creates an expression that calculates the byte length of a string in its UTF-8 form. Example: @@ -1021,12 +1021,12 @@ def byte_length(self) -> "Expr": >>> Field.of("name").byte_length() Returns: - A new `Expr` representing the byte length of the string. + A new `Expression` representing the byte length of the string. """ return Function("byte_length", [self]) @expose_as_static - def like(self, pattern: Expr | str) -> "BooleanExpr": + def like(self, pattern: Expression | str) -> "BooleanExpression": """Creates an expression that performs a case-sensitive string comparison. Example: @@ -1039,14 +1039,14 @@ def like(self, pattern: Expr | str) -> "BooleanExpr": pattern: The pattern (string or expression) to search for. You can use "%" as a wildcard character. Returns: - A new `Expr` representing the 'like' comparison. + A new `Expression` representing the 'like' comparison. """ - return BooleanExpr( + return BooleanExpression( "like", [self, self._cast_to_expr_or_convert_to_constant(pattern)] ) @expose_as_static - def regex_contains(self, regex: Expr | str) -> "BooleanExpr": + def regex_contains(self, regex: Expression | str) -> "BooleanExpression": """Creates an expression that checks if a string contains a specified regular expression as a substring. @@ -1060,14 +1060,14 @@ def regex_contains(self, regex: Expr | str) -> "BooleanExpr": regex: The regular expression (string or expression) to use for the search. Returns: - A new `Expr` representing the 'contains' comparison. + A new `Expression` representing the 'contains' comparison. """ - return BooleanExpr( + return BooleanExpression( "regex_contains", [self, self._cast_to_expr_or_convert_to_constant(regex)] ) @expose_as_static - def regex_match(self, regex: Expr | str) -> "BooleanExpr": + def regex_match(self, regex: Expression | str) -> "BooleanExpression": """Creates an expression that checks if a string matches a specified regular expression. Example: @@ -1080,14 +1080,14 @@ def regex_match(self, regex: Expr | str) -> "BooleanExpr": regex: The regular expression (string or expression) to use for the match. Returns: - A new `Expr` representing the regular expression match. + A new `Expression` representing the regular expression match. """ - return BooleanExpr( + return BooleanExpression( "regex_match", [self, self._cast_to_expr_or_convert_to_constant(regex)] ) @expose_as_static - def string_contains(self, substring: Expr | str) -> "BooleanExpr": + def string_contains(self, substring: Expression | str) -> "BooleanExpression": """Creates an expression that checks if this string expression contains a specified substring. Example: @@ -1100,15 +1100,15 @@ def string_contains(self, substring: Expr | str) -> "BooleanExpr": substring: The substring (string or expression) to use for the search. Returns: - A new `Expr` representing the 'contains' comparison. + A new `Expression` representing the 'contains' comparison. """ - return BooleanExpr( + return BooleanExpression( "string_contains", [self, self._cast_to_expr_or_convert_to_constant(substring)], ) @expose_as_static - def starts_with(self, prefix: Expr | str) -> "BooleanExpr": + def starts_with(self, prefix: Expression | str) -> "BooleanExpression": """Creates an expression that checks if a string starts with a given prefix. Example: @@ -1121,14 +1121,14 @@ def starts_with(self, prefix: Expr | str) -> "BooleanExpr": prefix: The prefix (string or expression) to check for. Returns: - A new `Expr` representing the 'starts with' comparison. + A new `Expression` representing the 'starts with' comparison. """ - return BooleanExpr( + return BooleanExpression( "starts_with", [self, self._cast_to_expr_or_convert_to_constant(prefix)] ) @expose_as_static - def ends_with(self, postfix: Expr | str) -> "BooleanExpr": + def ends_with(self, postfix: Expression | str) -> "BooleanExpression": """Creates an expression that checks if a string ends with a given postfix. Example: @@ -1141,14 +1141,14 @@ def ends_with(self, postfix: Expr | str) -> "BooleanExpr": postfix: The postfix (string or expression) to check for. Returns: - A new `Expr` representing the 'ends with' comparison. + A new `Expression` representing the 'ends with' comparison. """ - return BooleanExpr( + return BooleanExpression( "ends_with", [self, self._cast_to_expr_or_convert_to_constant(postfix)] ) @expose_as_static - def string_concat(self, *elements: Expr | CONSTANT_TYPE) -> "Expr": + def string_concat(self, *elements: Expression | CONSTANT_TYPE) -> "Expression": """Creates an expression that concatenates string expressions, fields or constants together. Example: @@ -1159,7 +1159,7 @@ def string_concat(self, *elements: Expr | CONSTANT_TYPE) -> "Expr": *elements: The expressions or constants (typically strings) to concatenate. Returns: - A new `Expr` representing the concatenated string. + A new `Expression` representing the concatenated string. """ return Function( "string_concat", @@ -1167,7 +1167,7 @@ def string_concat(self, *elements: Expr | CONSTANT_TYPE) -> "Expr": ) @expose_as_static - def to_lower(self) -> "Expr": + def to_lower(self) -> "Expression": """Creates an expression that converts a string to lowercase. Example: @@ -1175,12 +1175,12 @@ def to_lower(self) -> "Expr": >>> Field.of("name").to_lower() Returns: - A new `Expr` representing the lowercase string. + A new `Expression` representing the lowercase string. """ return Function("to_lower", [self]) @expose_as_static - def to_upper(self) -> "Expr": + def to_upper(self) -> "Expression": """Creates an expression that converts a string to uppercase. Example: @@ -1188,12 +1188,12 @@ def to_upper(self) -> "Expr": >>> Field.of("title").to_upper() Returns: - A new `Expr` representing the uppercase string. + A new `Expression` representing the uppercase string. """ return Function("to_upper", [self]) @expose_as_static - def trim(self) -> "Expr": + def trim(self) -> "Expression": """Creates an expression that removes leading and trailing whitespace from a string. Example: @@ -1201,12 +1201,12 @@ def trim(self) -> "Expr": >>> Field.of("userInput").trim() Returns: - A new `Expr` representing the trimmed string. + A new `Expression` representing the trimmed string. """ return Function("trim", [self]) @expose_as_static - def string_reverse(self) -> "Expr": + def string_reverse(self) -> "Expression": """Creates an expression that reverses a string. Example: @@ -1214,14 +1214,14 @@ def string_reverse(self) -> "Expr": >>> Field.of("userInput").reverse() Returns: - A new `Expr` representing the reversed string. + A new `Expression` representing the reversed string. """ return Function("string_reverse", [self]) @expose_as_static def substring( - self, position: Expr | int, length: Expr | int | None = None - ) -> "Expr": + self, position: Expression | int, length: Expression | int | None = None + ) -> "Expression": """Creates an expression that returns a substring of the results of this expression. @@ -1235,7 +1235,7 @@ def substring( will end at the end of the input. Returns: - A new `Expr` representing the extracted substring. + A new `Expression` representing the extracted substring. """ args = [self, self._cast_to_expr_or_convert_to_constant(position)] if length is not None: @@ -1243,7 +1243,7 @@ def substring( return Function("substring", args) @expose_as_static - def join(self, delimeter: Expr | str) -> "Expr": + def join(self, delimeter: Expression | str) -> "Expression": """Creates an expression that joins the elements of an array into a string @@ -1254,14 +1254,14 @@ def join(self, delimeter: Expr | str) -> "Expr": delimiter: The delimiter to add between the elements of the array. Returns: - A new `Expr` representing the joined string. + A new `Expression` representing the joined string. """ return Function( "join", [self, self._cast_to_expr_or_convert_to_constant(delimeter)] ) @expose_as_static - def map_get(self, key: str | Constant[str]) -> "Expr": + def map_get(self, key: str | Constant[str]) -> "Expression": """Accesses a value from the map produced by evaluating this expression. Example: @@ -1272,14 +1272,14 @@ def map_get(self, key: str | Constant[str]) -> "Expr": key: The key to access in the map. Returns: - A new `Expr` representing the value associated with the given key in the map. + A new `Expression` representing the value associated with the given key in the map. """ return Function( "map_get", [self, self._cast_to_expr_or_convert_to_constant(key)] ) @expose_as_static - def map_remove(self, key: str | Constant[str]) -> "Expr": + def map_remove(self, key: str | Constant[str]) -> "Expression": """Remove a key from a the map produced by evaluating this expression. Example: @@ -1290,7 +1290,7 @@ def map_remove(self, key: str | Constant[str]) -> "Expr": key: The key to ewmove in the map. Returns: - A new `Expr` representing the map_remove operation. + A new `Expression` representing the map_remove operation. """ return Function( "map_remove", [self, self._cast_to_expr_or_convert_to_constant(key)] @@ -1298,8 +1298,8 @@ def map_remove(self, key: str | Constant[str]) -> "Expr": @expose_as_static def map_merge( - self, *other_maps: Map | dict[str | Constant[str], Expr | CONSTANT_TYPE] | Expr - ) -> "Expr": + self, *other_maps: Map | dict[str | Constant[str], Expression | CONSTANT_TYPE] | Expression + ) -> "Expression": """Creates an expression that merges one or more dicts into a single map. Example: @@ -1310,7 +1310,7 @@ def map_merge( *other_maps: Sequence of maps to merge into the resulting map. Returns: - A new `Expr` representing the value associated with the given key in the map. + A new `Expression` representing the value associated with the given key in the map. """ return Function( "map_merge", @@ -1318,7 +1318,7 @@ def map_merge( ) @expose_as_static - def cosine_distance(self, other: Expr | list[float] | Vector) -> "Expr": + def cosine_distance(self, other: Expression | list[float] | Vector) -> "Expression": """Calculates the cosine distance between two vectors. Example: @@ -1328,10 +1328,10 @@ def cosine_distance(self, other: Expr | list[float] | Vector) -> "Expr": >>> Field.of("location").cosine_distance([37.7749, -122.4194]) Args: - other: The other vector (represented as an Expr, list of floats, or Vector) to compare against. + other: The other vector (represented as an Expression, list of floats, or Vector) to compare against. Returns: - A new `Expr` representing the cosine distance between the two vectors. + A new `Expression` representing the cosine distance between the two vectors. """ return Function( "cosine_distance", @@ -1342,7 +1342,7 @@ def cosine_distance(self, other: Expr | list[float] | Vector) -> "Expr": ) @expose_as_static - def euclidean_distance(self, other: Expr | list[float] | Vector) -> "Expr": + def euclidean_distance(self, other: Expression | list[float] | Vector) -> "Expression": """Calculates the Euclidean distance between two vectors. Example: @@ -1352,10 +1352,10 @@ def euclidean_distance(self, other: Expr | list[float] | Vector) -> "Expr": >>> Field.of("pointA").euclidean_distance(Field.of("pointB")) Args: - other: The other vector (represented as an Expr, list of floats, or Vector) to compare against. + other: The other vector (represented as an Expression, list of floats, or Vector) to compare against. Returns: - A new `Expr` representing the Euclidean distance between the two vectors. + A new `Expression` representing the Euclidean distance between the two vectors. """ return Function( "euclidean_distance", @@ -1366,7 +1366,7 @@ def euclidean_distance(self, other: Expr | list[float] | Vector) -> "Expr": ) @expose_as_static - def dot_product(self, other: Expr | list[float] | Vector) -> "Expr": + def dot_product(self, other: Expression | list[float] | Vector) -> "Expression": """Calculates the dot product between two vectors. Example: @@ -1376,10 +1376,10 @@ def dot_product(self, other: Expr | list[float] | Vector) -> "Expr": >>> Field.of("docVector1").dot_product(Field.of("docVector2")) Args: - other: The other vector (represented as an Expr, list of floats, or Vector) to calculate dot product with. + other: The other vector (represented as an Expression, list of floats, or Vector) to calculate dot product with. Returns: - A new `Expr` representing the dot product between the two vectors. + A new `Expression` representing the dot product between the two vectors. """ return Function( "dot_product", @@ -1390,7 +1390,7 @@ def dot_product(self, other: Expr | list[float] | Vector) -> "Expr": ) @expose_as_static - def vector_length(self) -> "Expr": + def vector_length(self) -> "Expression": """Creates an expression that calculates the length (dimension) of a Firestore Vector. Example: @@ -1398,12 +1398,12 @@ def vector_length(self) -> "Expr": >>> Field.of("embedding").vector_length() Returns: - A new `Expr` representing the length of the vector. + A new `Expression` representing the length of the vector. """ return Function("vector_length", [self]) @expose_as_static - def timestamp_to_unix_micros(self) -> "Expr": + def timestamp_to_unix_micros(self) -> "Expression": """Creates an expression that converts a timestamp to the number of microseconds since the epoch (1970-01-01 00:00:00 UTC). @@ -1414,12 +1414,12 @@ def timestamp_to_unix_micros(self) -> "Expr": >>> Field.of("timestamp").timestamp_to_unix_micros() Returns: - A new `Expr` representing the number of microseconds since the epoch. + A new `Expression` representing the number of microseconds since the epoch. """ return Function("timestamp_to_unix_micros", [self]) @expose_as_static - def unix_micros_to_timestamp(self) -> "Expr": + def unix_micros_to_timestamp(self) -> "Expression": """Creates an expression that converts a number of microseconds since the epoch (1970-01-01 00:00:00 UTC) to a timestamp. @@ -1428,12 +1428,12 @@ def unix_micros_to_timestamp(self) -> "Expr": >>> Field.of("microseconds").unix_micros_to_timestamp() Returns: - A new `Expr` representing the timestamp. + A new `Expression` representing the timestamp. """ return Function("unix_micros_to_timestamp", [self]) @expose_as_static - def timestamp_to_unix_millis(self) -> "Expr": + def timestamp_to_unix_millis(self) -> "Expression": """Creates an expression that converts a timestamp to the number of milliseconds since the epoch (1970-01-01 00:00:00 UTC). @@ -1444,12 +1444,12 @@ def timestamp_to_unix_millis(self) -> "Expr": >>> Field.of("timestamp").timestamp_to_unix_millis() Returns: - A new `Expr` representing the number of milliseconds since the epoch. + A new `Expression` representing the number of milliseconds since the epoch. """ return Function("timestamp_to_unix_millis", [self]) @expose_as_static - def unix_millis_to_timestamp(self) -> "Expr": + def unix_millis_to_timestamp(self) -> "Expression": """Creates an expression that converts a number of milliseconds since the epoch (1970-01-01 00:00:00 UTC) to a timestamp. @@ -1458,12 +1458,12 @@ def unix_millis_to_timestamp(self) -> "Expr": >>> Field.of("milliseconds").unix_millis_to_timestamp() Returns: - A new `Expr` representing the timestamp. + A new `Expression` representing the timestamp. """ return Function("unix_millis_to_timestamp", [self]) @expose_as_static - def timestamp_to_unix_seconds(self) -> "Expr": + def timestamp_to_unix_seconds(self) -> "Expression": """Creates an expression that converts a timestamp to the number of seconds since the epoch (1970-01-01 00:00:00 UTC). @@ -1474,12 +1474,12 @@ def timestamp_to_unix_seconds(self) -> "Expr": >>> Field.of("timestamp").timestamp_to_unix_seconds() Returns: - A new `Expr` representing the number of seconds since the epoch. + A new `Expression` representing the number of seconds since the epoch. """ return Function("timestamp_to_unix_seconds", [self]) @expose_as_static - def unix_seconds_to_timestamp(self) -> "Expr": + def unix_seconds_to_timestamp(self) -> "Expression": """Creates an expression that converts a number of seconds since the epoch (1970-01-01 00:00:00 UTC) to a timestamp. @@ -1488,12 +1488,12 @@ def unix_seconds_to_timestamp(self) -> "Expr": >>> Field.of("seconds").unix_seconds_to_timestamp() Returns: - A new `Expr` representing the timestamp. + A new `Expression` representing the timestamp. """ return Function("unix_seconds_to_timestamp", [self]) @expose_as_static - def timestamp_add(self, unit: Expr | str, amount: Expr | float) -> "Expr": + def timestamp_add(self, unit: Expression | str, amount: Expression | float) -> "Expression": """Creates an expression that adds a specified amount of time to this timestamp expression. Example: @@ -1508,7 +1508,7 @@ def timestamp_add(self, unit: Expr | str, amount: Expr | float) -> "Expr": amount: The expression or float representing the amount of time to add. Returns: - A new `Expr` representing the resulting timestamp. + A new `Expression` representing the resulting timestamp. """ return Function( "timestamp_add", @@ -1520,7 +1520,7 @@ def timestamp_add(self, unit: Expr | str, amount: Expr | float) -> "Expr": ) @expose_as_static - def timestamp_subtract(self, unit: Expr | str, amount: Expr | float) -> "Expr": + def timestamp_subtract(self, unit: Expression | str, amount: Expression | float) -> "Expression": """Creates an expression that subtracts a specified amount of time from this timestamp expression. Example: @@ -1535,7 +1535,7 @@ def timestamp_subtract(self, unit: Expr | str, amount: Expr | float) -> "Expr": amount: The expression or float representing the amount of time to subtract. Returns: - A new `Expr` representing the resulting timestamp. + A new `Expression` representing the resulting timestamp. """ return Function( "timestamp_subtract", @@ -1555,7 +1555,7 @@ def collection_id(self): >>> Field.of("__name__").collection_id() Returns: - A new `Expr` representing the collection ID. + A new `Expression` representing the collection ID. """ return Function("collection_id", [self]) @@ -1568,7 +1568,7 @@ def document_id(self): >>> Field.of("__name__").document_id() Returns: - A new `Expr` representing the document ID. + A new `Expression` representing the document ID. """ return Function("document_id", [self]) @@ -1596,7 +1596,7 @@ def descending(self) -> Ordering: """ return Ordering(self, Ordering.Direction.DESCENDING) - def as_(self, alias: str) -> "AliasedExpr": + def as_(self, alias: str) -> "AliasedExpression": """Assigns an alias to this expression. Aliases are useful for renaming fields in the output of a stage or for giving meaningful @@ -1612,13 +1612,13 @@ def as_(self, alias: str) -> "AliasedExpr": alias: The alias to assign to this expression. Returns: - A new `Selectable` (typically an `AliasedExpr`) that wraps this + A new `Selectable` (typically an `AliasedExpression`) that wraps this expression and associates it with the provided alias. """ - return AliasedExpr(self, alias) + return AliasedExpression(self, alias) -class Constant(Expr, Generic[CONSTANT_TYPE]): +class Constant(Expression, Generic[CONSTANT_TYPE]): """Represents a constant literal value in an expression.""" def __init__(self, value: CONSTANT_TYPE): @@ -1645,13 +1645,13 @@ def _to_pb(self) -> Value: return encode_value(self.value) -class Function(Expr): +class Function(Expression): """A base class for expressions that represent function calls.""" def __init__( self, name: str, - params: Sequence[Expr], + params: Sequence[Expression], *, use_infix_repr: bool = True, infix_name_override: str | None = None, @@ -1696,7 +1696,7 @@ class AggregateFunction(Function): """A base class for aggregation functions that operate across multiple inputs.""" -class Selectable(Expr): +class Selectable(Expression): """Base class for expressions that can be selected or aliased in projection stages.""" def __eq__(self, other): @@ -1732,10 +1732,10 @@ def _to_value(field_list: Sequence[Selectable]) -> Value: ) -T = TypeVar("T", bound=Expr) +T = TypeVar("T", bound=Expression) -class AliasedExpr(Selectable, Generic[T]): +class AliasedExpression(Selectable, Generic[T]): """Wraps an expression with an alias.""" def __init__(self, expr: T, alias: str): @@ -1789,14 +1789,14 @@ def _to_pb(self): return Value(field_reference_value=self.path) -class BooleanExpr(Function): +class BooleanExpression(Function): """Filters the given data in some way.""" @staticmethod def _from_query_filter_pb(filter_pb, client): if isinstance(filter_pb, Query_pb.CompositeFilter): sub_filters = [ - BooleanExpr._from_query_filter_pb(f, client) for f in filter_pb.filters + BooleanExpression._from_query_filter_pb(f, client) for f in filter_pb.filters ] if filter_pb.op == Query_pb.CompositeFilter.Operator.OR: return Or(*sub_filters) @@ -1850,7 +1850,7 @@ def _from_query_filter_pb(filter_pb, client): or filter_pb.field_filter or filter_pb.unary_filter ) - return BooleanExpr._from_query_filter_pb(f, client) + return BooleanExpression._from_query_filter_pb(f, client) else: raise TypeError(f"Unexpected filter type: {type(filter_pb)}") @@ -1860,13 +1860,13 @@ class Array(Function): Creates an expression that creates a Firestore array value from an input list. Example: - >>> Expr.array(["bar", Field.of("baz")]) + >>> Expression.array(["bar", Field.of("baz")]) Args: elements: THe input list to evaluate in the expression """ - def __init__(self, elements: list[Expr | CONSTANT_TYPE]): + def __init__(self, elements: list[Expression | CONSTANT_TYPE]): if not isinstance(elements, list): raise TypeError("Array must be constructed with a list") converted_elements = [ @@ -1883,13 +1883,13 @@ class Map(Function): Creates an expression that creates a Firestore map value from an input dict. Example: - >>> Expr.map({"foo": "bar", "baz": Field.of("baz")}) + >>> Expression.map({"foo": "bar", "baz": Field.of("baz")}) Args: elements: THe input dict to evaluate in the expression """ - def __init__(self, elements: dict[str | Constant[str], Expr | CONSTANT_TYPE]): + def __init__(self, elements: dict[str | Constant[str], Expression | CONSTANT_TYPE]): element_list = [] for k, v in elements.items(): element_list.append(self._cast_to_expr_or_convert_to_constant(k)) @@ -1904,7 +1904,7 @@ def __repr__(self): return f"Map({d})" -class And(BooleanExpr): +class And(BooleanExpression): """ Represents an expression that performs a logical 'AND' operation on multiple filter conditions. @@ -1917,11 +1917,11 @@ class And(BooleanExpr): *conditions: The filter conditions to 'AND' together. """ - def __init__(self, *conditions: "BooleanExpr"): + def __init__(self, *conditions: "BooleanExpression"): super().__init__("and", conditions, use_infix_repr=False) -class Not(BooleanExpr): +class Not(BooleanExpression): """ Represents an expression that negates a filter condition. @@ -1933,11 +1933,11 @@ class Not(BooleanExpr): condition: The filter condition to negate. """ - def __init__(self, condition: BooleanExpr): + def __init__(self, condition: BooleanExpression): super().__init__("not", [condition], use_infix_repr=False) -class Or(BooleanExpr): +class Or(BooleanExpression): """ Represents expression that performs a logical 'OR' operation on multiple filter conditions. @@ -1950,11 +1950,11 @@ class Or(BooleanExpr): *conditions: The filter conditions to 'OR' together. """ - def __init__(self, *conditions: "BooleanExpr"): + def __init__(self, *conditions: "BooleanExpression"): super().__init__("or", conditions, use_infix_repr=False) -class Xor(BooleanExpr): +class Xor(BooleanExpression): """ Represents an expression that performs a logical 'XOR' (exclusive OR) operation on multiple filter conditions. @@ -1967,11 +1967,11 @@ class Xor(BooleanExpr): *conditions: The filter conditions to 'XOR' together. """ - def __init__(self, conditions: Sequence["BooleanExpr"]): + def __init__(self, conditions: Sequence["BooleanExpression"]): super().__init__("xor", conditions, use_infix_repr=False) -class Conditional(BooleanExpr): +class Conditional(BooleanExpression): """ Represents a conditional expression that evaluates to a 'then' expression if a condition is true and an 'else' expression if the condition is false. @@ -1986,7 +1986,7 @@ class Conditional(BooleanExpr): else_expr: The expression to return if the condition is false """ - def __init__(self, condition: BooleanExpr, then_expr: Expr, else_expr: Expr): + def __init__(self, condition: BooleanExpression, then_expr: Expression, else_expr: Expression): super().__init__( "conditional", [condition, then_expr, else_expr], use_infix_repr=False ) @@ -2007,7 +2007,7 @@ class Count(AggregateFunction): expression: The expression or field to count. If None, counts all stage inputs. """ - def __init__(self, expression: Expr | None = None): + def __init__(self, expression: Expression | None = None): expression_list = [expression] if expression else [] super().__init__("count", expression_list, use_infix_repr=bool(expression_list)) @@ -2016,7 +2016,7 @@ class CurrentTimestamp(Function): """Creates an expression that returns the current timestamp Returns: - A new `Expr` representing the current timestamp. + A new `Expression` representing the current timestamp. """ def __init__(self): diff --git a/tests/unit/v1/test_pipeline_expressions.py b/tests/unit/v1/test_pipeline_expressions.py index aec721e7d..00d636ca1 100644 --- a/tests/unit/v1/test_pipeline_expressions.py +++ b/tests/unit/v1/test_pipeline_expressions.py @@ -23,8 +23,8 @@ from google.cloud.firestore_v1.vector import Vector from google.cloud.firestore_v1._helpers import GeoPoint import google.cloud.firestore_v1.pipeline_expressions as expr -from google.cloud.firestore_v1.pipeline_expressions import BooleanExpr -from google.cloud.firestore_v1.pipeline_expressions import Expr +from google.cloud.firestore_v1.pipeline_expressions import BooleanExpression +from google.cloud.firestore_v1.pipeline_expressions import Expression from google.cloud.firestore_v1.pipeline_expressions import Constant from google.cloud.firestore_v1.pipeline_expressions import Field from google.cloud.firestore_v1.pipeline_expressions import Ordering @@ -167,7 +167,7 @@ def test_equality(self, first, second, expected): class TestSelectable: """ - contains tests for each Expr class that derives from Selectable + contains tests for each Expression class that derives from Selectable """ def test_ctor(self): @@ -224,7 +224,7 @@ def test_to_map(self): assert result[0] == "field1" assert result[1] == Value(field_reference_value="field1") - class TestAliasedExpr: + class TestAliasedExpression: def test_repr(self): instance = Field.of("field1").as_("alias1") assert repr(instance) == "Field.of('field1').as_('alias1')" @@ -232,14 +232,14 @@ def test_repr(self): def test_ctor(self): arg = Field.of("field1") alias = "alias1" - instance = expr.AliasedExpr(arg, alias) + instance = expr.AliasedExpression(arg, alias) assert instance.expr == arg assert instance.alias == alias def test_to_pb(self): arg = Field.of("field1") alias = "alias1" - instance = expr.AliasedExpr(arg, alias) + instance = expr.AliasedExpression(arg, alias) result = instance._to_pb() assert result.map_value.fields.get("alias1") == arg._to_pb() @@ -249,35 +249,8 @@ def test_to_map(self): assert result[0] == "alias1" assert result[1] == Value(field_reference_value="field1") - class TestAliasedAggregate: - def test_repr(self): - instance = Field.of("field1").maximum().as_("alias1") - assert repr(instance) == "Field.of('field1').maximum().as_('alias1')" - - def test_ctor(self): - arg = Expr.minimum("field1") - alias = "alias1" - instance = expr.AliasedAggregate(arg, alias) - assert instance.expr == arg - assert instance.alias == alias - - def test_to_pb(self): - arg = Field.of("field1").average() - alias = "alias1" - instance = expr.AliasedAggregate(arg, alias) - result = instance._to_pb() - assert result.map_value.fields.get("alias1") == arg._to_pb() - - def test_to_map(self): - arg = Field.of("field1").count() - alias = "alias1" - instance = expr.AliasedAggregate(arg, alias) - result = instance._to_map() - assert result[0] == "alias1" - assert result[1] == arg._to_pb() - -class TestBooleanExpr: +class TestBooleanExpression: def test__from_query_filter_pb_composite_filter_or(self, mock_client): """ test composite OR filters @@ -305,7 +278,7 @@ def test__from_query_filter_pb_composite_filter_or(self, mock_client): composite_filter=composite_pb ) - result = BooleanExpr._from_query_filter_pb(wrapped_filter_pb, mock_client) + result = BooleanExpression._from_query_filter_pb(wrapped_filter_pb, mock_client) # should include existance checks field1 = Field.of("field1") @@ -344,7 +317,7 @@ def test__from_query_filter_pb_composite_filter_and(self, mock_client): composite_filter=composite_pb ) - result = BooleanExpr._from_query_filter_pb(wrapped_filter_pb, mock_client) + result = BooleanExpression._from_query_filter_pb(wrapped_filter_pb, mock_client) # should include existance checks field1 = Field.of("field1") @@ -391,7 +364,7 @@ def test__from_query_filter_pb_composite_filter_nested(self, mock_client): composite_filter=outer_or_pb ) - result = BooleanExpr._from_query_filter_pb(wrapped_filter_pb, mock_client) + result = BooleanExpression._from_query_filter_pb(wrapped_filter_pb, mock_client) field1 = Field.of("field1") field2 = Field.of("field2") @@ -422,23 +395,23 @@ def test__from_query_filter_pb_composite_filter_unknown_op(self, mock_client): ) with pytest.raises(TypeError, match="Unexpected CompositeFilter operator type"): - BooleanExpr._from_query_filter_pb(wrapped_filter_pb, mock_client) + BooleanExpression._from_query_filter_pb(wrapped_filter_pb, mock_client) @pytest.mark.parametrize( "op_enum, expected_expr_func", [ - (query_pb.StructuredQuery.UnaryFilter.Operator.IS_NAN, Expr.is_nan), + (query_pb.StructuredQuery.UnaryFilter.Operator.IS_NAN, Expression.is_nan), ( query_pb.StructuredQuery.UnaryFilter.Operator.IS_NOT_NAN, - Expr.is_not_nan, + Expression.is_not_nan, ), ( query_pb.StructuredQuery.UnaryFilter.Operator.IS_NULL, - Expr.is_null, + Expression.is_null, ), ( query_pb.StructuredQuery.UnaryFilter.Operator.IS_NOT_NULL, - Expr.is_not_null, + Expression.is_not_null, ), ], ) @@ -455,7 +428,7 @@ def test__from_query_filter_pb_unary_filter( ) wrapped_filter_pb = query_pb.StructuredQuery.Filter(unary_filter=filter_pb) - result = BooleanExpr._from_query_filter_pb(wrapped_filter_pb, mock_client) + result = BooleanExpression._from_query_filter_pb(wrapped_filter_pb, mock_client) field_expr_inst = Field.of(field_path) expected_condition = expected_expr_func(field_expr_inst) @@ -476,7 +449,7 @@ def test__from_query_filter_pb_unary_filter_unknown_op(self, mock_client): wrapped_filter_pb = query_pb.StructuredQuery.Filter(unary_filter=filter_pb) with pytest.raises(TypeError, match="Unexpected UnaryFilter operator type"): - BooleanExpr._from_query_filter_pb(wrapped_filter_pb, mock_client) + BooleanExpression._from_query_filter_pb(wrapped_filter_pb, mock_client) @pytest.mark.parametrize( "op_enum, value, expected_expr_func", @@ -484,48 +457,48 @@ def test__from_query_filter_pb_unary_filter_unknown_op(self, mock_client): ( query_pb.StructuredQuery.FieldFilter.Operator.LESS_THAN, 10, - Expr.less_than, + Expression.less_than, ), ( query_pb.StructuredQuery.FieldFilter.Operator.LESS_THAN_OR_EQUAL, 10, - Expr.less_than_or_equal, + Expression.less_than_or_equal, ), ( query_pb.StructuredQuery.FieldFilter.Operator.GREATER_THAN, 10, - Expr.greater_than, + Expression.greater_than, ), ( query_pb.StructuredQuery.FieldFilter.Operator.GREATER_THAN_OR_EQUAL, 10, - Expr.greater_than_or_equal, + Expression.greater_than_or_equal, ), - (query_pb.StructuredQuery.FieldFilter.Operator.EQUAL, 10, Expr.equal), + (query_pb.StructuredQuery.FieldFilter.Operator.EQUAL, 10, Expression.equal), ( query_pb.StructuredQuery.FieldFilter.Operator.NOT_EQUAL, 10, - Expr.not_equal, + Expression.not_equal, ), ( query_pb.StructuredQuery.FieldFilter.Operator.ARRAY_CONTAINS, 10, - Expr.array_contains, + Expression.array_contains, ), ( query_pb.StructuredQuery.FieldFilter.Operator.ARRAY_CONTAINS_ANY, [10, 20], - Expr.array_contains_any, + Expression.array_contains_any, ), ( query_pb.StructuredQuery.FieldFilter.Operator.IN, [10, 20], - Expr.equal_any, + Expression.equal_any, ), ( query_pb.StructuredQuery.FieldFilter.Operator.NOT_IN, [10, 20], - Expr.not_equal_any, + Expression.not_equal_any, ), ], ) @@ -544,7 +517,7 @@ def test__from_query_filter_pb_field_filter( ) wrapped_filter_pb = query_pb.StructuredQuery.Filter(field_filter=filter_pb) - result = BooleanExpr._from_query_filter_pb(wrapped_filter_pb, mock_client) + result = BooleanExpression._from_query_filter_pb(wrapped_filter_pb, mock_client) field_expr = Field.of(field_path) # convert values into constants @@ -571,7 +544,7 @@ def test__from_query_filter_pb_field_filter_unknown_op(self, mock_client): wrapped_filter_pb = query_pb.StructuredQuery.Filter(field_filter=filter_pb) with pytest.raises(TypeError, match="Unexpected FieldFilter operator type"): - BooleanExpr._from_query_filter_pb(wrapped_filter_pb, mock_client) + BooleanExpression._from_query_filter_pb(wrapped_filter_pb, mock_client) def test__from_query_filter_pb_unknown_filter_type(self, mock_client): """ @@ -579,7 +552,7 @@ def test__from_query_filter_pb_unknown_filter_type(self, mock_client): """ # Test with an unexpected protobuf type with pytest.raises(TypeError, match="Unexpected filter type"): - BooleanExpr._from_query_filter_pb(document_pb.Value(), mock_client) + BooleanExpression._from_query_filter_pb(document_pb.Value(), mock_client) class TestArray: @@ -645,9 +618,9 @@ def test_w_exprs(self): ) -class TestExpressionMethods: +class TestExpressionessionMethods: """ - contains test methods for each Expr method + contains test methods for each Expression method """ @pytest.mark.parametrize( @@ -685,11 +658,11 @@ def test_equality(self, first, second, expected): assert (first == second) is expected def _make_arg(self, name="Mock"): - class MockExpr(Constant): + class MockExpression(Constant): def __repr__(self): return self.value - arg = MockExpr(name) + arg = MockExpression(name) return arg def test_and(self): @@ -712,7 +685,7 @@ def test_or(self): def test_array_contains(self): arg1 = self._make_arg("ArrayField") arg2 = self._make_arg("Element") - instance = Expr.array_contains(arg1, arg2) + instance = Expression.array_contains(arg1, arg2) assert instance.name == "array_contains" assert instance.params == [arg1, arg2] assert repr(instance) == "ArrayField.array_contains(Element)" @@ -723,7 +696,7 @@ def test_array_contains_any(self): arg1 = self._make_arg("ArrayField") arg2 = self._make_arg("Element1") arg3 = self._make_arg("Element2") - instance = Expr.array_contains_any(arg1, [arg2, arg3]) + instance = Expression.array_contains_any(arg1, [arg2, arg3]) assert instance.name == "array_contains_any" assert isinstance(instance.params[1], expr.Array) assert instance.params[0] == arg1 @@ -737,7 +710,7 @@ def test_array_contains_any(self): def test_exists(self): arg1 = self._make_arg("Field") - instance = Expr.exists(arg1) + instance = Expression.exists(arg1) assert instance.name == "exists" assert instance.params == [arg1] assert repr(instance) == "Field.exists()" @@ -747,7 +720,7 @@ def test_exists(self): def test_equal(self): arg1 = self._make_arg("Left") arg2 = self._make_arg("Right") - instance = Expr.equal(arg1, arg2) + instance = Expression.equal(arg1, arg2) assert instance.name == "equal" assert instance.params == [arg1, arg2] assert repr(instance) == "Left.equal(Right)" @@ -757,7 +730,7 @@ def test_equal(self): def test_greater_than_or_equal(self): arg1 = self._make_arg("Left") arg2 = self._make_arg("Right") - instance = Expr.greater_than_or_equal(arg1, arg2) + instance = Expression.greater_than_or_equal(arg1, arg2) assert instance.name == "greater_than_or_equal" assert instance.params == [arg1, arg2] assert repr(instance) == "Left.greater_than_or_equal(Right)" @@ -767,7 +740,7 @@ def test_greater_than_or_equal(self): def test_greater_than(self): arg1 = self._make_arg("Left") arg2 = self._make_arg("Right") - instance = Expr.greater_than(arg1, arg2) + instance = Expression.greater_than(arg1, arg2) assert instance.name == "greater_than" assert instance.params == [arg1, arg2] assert repr(instance) == "Left.greater_than(Right)" @@ -777,7 +750,7 @@ def test_greater_than(self): def test_less_than_or_equal(self): arg1 = self._make_arg("Left") arg2 = self._make_arg("Right") - instance = Expr.less_than_or_equal(arg1, arg2) + instance = Expression.less_than_or_equal(arg1, arg2) assert instance.name == "less_than_or_equal" assert instance.params == [arg1, arg2] assert repr(instance) == "Left.less_than_or_equal(Right)" @@ -787,7 +760,7 @@ def test_less_than_or_equal(self): def test_less_than(self): arg1 = self._make_arg("Left") arg2 = self._make_arg("Right") - instance = Expr.less_than(arg1, arg2) + instance = Expression.less_than(arg1, arg2) assert instance.name == "less_than" assert instance.params == [arg1, arg2] assert repr(instance) == "Left.less_than(Right)" @@ -797,7 +770,7 @@ def test_less_than(self): def test_not_equal(self): arg1 = self._make_arg("Left") arg2 = self._make_arg("Right") - instance = Expr.not_equal(arg1, arg2) + instance = Expression.not_equal(arg1, arg2) assert instance.name == "not_equal" assert instance.params == [arg1, arg2] assert repr(instance) == "Left.not_equal(Right)" @@ -808,7 +781,7 @@ def test_equal_any(self): arg1 = self._make_arg("Field") arg2 = self._make_arg("Value1") arg3 = self._make_arg("Value2") - instance = Expr.equal_any(arg1, [arg2, arg3]) + instance = Expression.equal_any(arg1, [arg2, arg3]) assert instance.name == "equal_any" assert isinstance(instance.params[1], expr.Array) assert instance.params[0] == arg1 @@ -821,7 +794,7 @@ def test_not_equal_any(self): arg1 = self._make_arg("Field") arg2 = self._make_arg("Value1") arg3 = self._make_arg("Value2") - instance = Expr.not_equal_any(arg1, [arg2, arg3]) + instance = Expression.not_equal_any(arg1, [arg2, arg3]) assert instance.name == "not_equal_any" assert isinstance(instance.params[1], expr.Array) assert instance.params[0] == arg1 @@ -832,7 +805,7 @@ def test_not_equal_any(self): def test_is_absent(self): arg1 = self._make_arg("Field") - instance = Expr.is_absent(arg1) + instance = Expression.is_absent(arg1) assert instance.name == "is_absent" assert instance.params == [arg1] assert repr(instance) == "Field.is_absent()" @@ -841,17 +814,17 @@ def test_is_absent(self): def test_if_absent(self): arg1 = self._make_arg("Field") - arg2 = self._make_arg("ThenExpr") - instance = Expr.if_absent(arg1, arg2) + arg2 = self._make_arg("ThenExpression") + instance = Expression.if_absent(arg1, arg2) assert instance.name == "if_absent" assert instance.params == [arg1, arg2] - assert repr(instance) == "Field.if_absent(ThenExpr)" + assert repr(instance) == "Field.if_absent(ThenExpression)" infix_instance = arg1.if_absent(arg2) assert infix_instance == instance def test_is_nan(self): arg1 = self._make_arg("Value") - instance = Expr.is_nan(arg1) + instance = Expression.is_nan(arg1) assert instance.name == "is_nan" assert instance.params == [arg1] assert repr(instance) == "Value.is_nan()" @@ -860,7 +833,7 @@ def test_is_nan(self): def test_is_not_nan(self): arg1 = self._make_arg("Value") - instance = Expr.is_not_nan(arg1) + instance = Expression.is_not_nan(arg1) assert instance.name == "is_not_nan" assert instance.params == [arg1] assert repr(instance) == "Value.is_not_nan()" @@ -869,7 +842,7 @@ def test_is_not_nan(self): def test_is_null(self): arg1 = self._make_arg("Value") - instance = Expr.is_null(arg1) + instance = Expression.is_null(arg1) assert instance.name == "is_null" assert instance.params == [arg1] assert repr(instance) == "Value.is_null()" @@ -878,7 +851,7 @@ def test_is_null(self): def test_is_not_null(self): arg1 = self._make_arg("Value") - instance = Expr.is_not_null(arg1) + instance = Expression.is_not_null(arg1) assert instance.name == "is_not_null" assert instance.params == [arg1] assert repr(instance) == "Value.is_not_null()" @@ -887,7 +860,7 @@ def test_is_not_null(self): def test_is_error(self): arg1 = self._make_arg("Value") - instance = Expr.is_error(arg1) + instance = Expression.is_error(arg1) assert instance.name == "is_error" assert instance.params == [arg1] assert repr(instance) == "Value.is_error()" @@ -896,11 +869,11 @@ def test_is_error(self): def test_if_error(self): arg1 = self._make_arg("Value") - arg2 = self._make_arg("ThenExpr") - instance = Expr.if_error(arg1, arg2) + arg2 = self._make_arg("ThenExpression") + instance = Expression.if_error(arg1, arg2) assert instance.name == "if_error" assert instance.params == [arg1, arg2] - assert repr(instance) == "Value.if_error(ThenExpr)" + assert repr(instance) == "Value.if_error(ThenExpression)" infix_instance = arg1.if_error(arg2) assert infix_instance == instance @@ -915,7 +888,7 @@ def test_array_contains_all(self): arg1 = self._make_arg("ArrayField") arg2 = self._make_arg("Element1") arg3 = self._make_arg("Element2") - instance = Expr.array_contains_all(arg1, [arg2, arg3]) + instance = Expression.array_contains_all(arg1, [arg2, arg3]) assert instance.name == "array_contains_all" assert isinstance(instance.params[1], expr.Array) assert instance.params[0] == arg1 @@ -928,71 +901,71 @@ def test_array_contains_all(self): assert infix_instance == instance def test_ends_with(self): - arg1 = self._make_arg("Expr") + arg1 = self._make_arg("Expression") arg2 = self._make_arg("Postfix") - instance = Expr.ends_with(arg1, arg2) + instance = Expression.ends_with(arg1, arg2) assert instance.name == "ends_with" assert instance.params == [arg1, arg2] - assert repr(instance) == "Expr.ends_with(Postfix)" + assert repr(instance) == "Expression.ends_with(Postfix)" infix_instance = arg1.ends_with(arg2) assert infix_instance == instance def test_conditional(self): arg1 = self._make_arg("Condition") - arg2 = self._make_arg("ThenExpr") - arg3 = self._make_arg("ElseExpr") + arg2 = self._make_arg("ThenExpression") + arg3 = self._make_arg("ElseExpression") instance = expr.Conditional(arg1, arg2, arg3) assert instance.name == "conditional" assert instance.params == [arg1, arg2, arg3] - assert repr(instance) == "Conditional(Condition, ThenExpr, ElseExpr)" + assert repr(instance) == "Conditional(Condition, ThenExpression, ElseExpression)" def test_like(self): - arg1 = self._make_arg("Expr") + arg1 = self._make_arg("Expression") arg2 = self._make_arg("Pattern") - instance = Expr.like(arg1, arg2) + instance = Expression.like(arg1, arg2) assert instance.name == "like" assert instance.params == [arg1, arg2] - assert repr(instance) == "Expr.like(Pattern)" + assert repr(instance) == "Expression.like(Pattern)" infix_instance = arg1.like(arg2) assert infix_instance == instance def test_regex_contains(self): - arg1 = self._make_arg("Expr") + arg1 = self._make_arg("Expression") arg2 = self._make_arg("Regex") - instance = Expr.regex_contains(arg1, arg2) + instance = Expression.regex_contains(arg1, arg2) assert instance.name == "regex_contains" assert instance.params == [arg1, arg2] - assert repr(instance) == "Expr.regex_contains(Regex)" + assert repr(instance) == "Expression.regex_contains(Regex)" infix_instance = arg1.regex_contains(arg2) assert infix_instance == instance def test_regex_match(self): - arg1 = self._make_arg("Expr") + arg1 = self._make_arg("Expression") arg2 = self._make_arg("Regex") - instance = Expr.regex_match(arg1, arg2) + instance = Expression.regex_match(arg1, arg2) assert instance.name == "regex_match" assert instance.params == [arg1, arg2] - assert repr(instance) == "Expr.regex_match(Regex)" + assert repr(instance) == "Expression.regex_match(Regex)" infix_instance = arg1.regex_match(arg2) assert infix_instance == instance def test_starts_with(self): - arg1 = self._make_arg("Expr") + arg1 = self._make_arg("Expression") arg2 = self._make_arg("Prefix") - instance = Expr.starts_with(arg1, arg2) + instance = Expression.starts_with(arg1, arg2) assert instance.name == "starts_with" assert instance.params == [arg1, arg2] - assert repr(instance) == "Expr.starts_with(Prefix)" + assert repr(instance) == "Expression.starts_with(Prefix)" infix_instance = arg1.starts_with(arg2) assert infix_instance == instance def test_string_contains(self): - arg1 = self._make_arg("Expr") + arg1 = self._make_arg("Expression") arg2 = self._make_arg("Substring") - instance = Expr.string_contains(arg1, arg2) + instance = Expression.string_contains(arg1, arg2) assert instance.name == "string_contains" assert instance.params == [arg1, arg2] - assert repr(instance) == "Expr.string_contains(Substring)" + assert repr(instance) == "Expression.string_contains(Substring)" infix_instance = arg1.string_contains(arg2) assert infix_instance == instance @@ -1007,7 +980,7 @@ def test_xor(self): def test_divide(self): arg1 = self._make_arg("Left") arg2 = self._make_arg("Right") - instance = Expr.divide(arg1, arg2) + instance = Expression.divide(arg1, arg2) assert instance.name == "divide" assert instance.params == [arg1, arg2] assert repr(instance) == "Left.divide(Right)" @@ -1017,7 +990,7 @@ def test_divide(self): def test_logical_maximum(self): arg1 = self._make_arg("Left") arg2 = self._make_arg("Right") - instance = Expr.logical_maximum(arg1, arg2) + instance = Expression.logical_maximum(arg1, arg2) assert instance.name == "maximum" assert instance.params == [arg1, arg2] assert repr(instance) == "Left.logical_maximum(Right)" @@ -1027,7 +1000,7 @@ def test_logical_maximum(self): def test_logical_minimum(self): arg1 = self._make_arg("Left") arg2 = self._make_arg("Right") - instance = Expr.logical_minimum(arg1, arg2) + instance = Expression.logical_minimum(arg1, arg2) assert instance.name == "minimum" assert instance.params == [arg1, arg2] assert repr(instance) == "Left.logical_minimum(Right)" @@ -1036,7 +1009,7 @@ def test_logical_minimum(self): def test_to_lower(self): arg1 = self._make_arg("Input") - instance = Expr.to_lower(arg1) + instance = Expression.to_lower(arg1) assert instance.name == "to_lower" assert instance.params == [arg1] assert repr(instance) == "Input.to_lower()" @@ -1045,7 +1018,7 @@ def test_to_lower(self): def test_to_upper(self): arg1 = self._make_arg("Input") - instance = Expr.to_upper(arg1) + instance = Expression.to_upper(arg1) assert instance.name == "to_upper" assert instance.params == [arg1] assert repr(instance) == "Input.to_upper()" @@ -1054,7 +1027,7 @@ def test_to_upper(self): def test_trim(self): arg1 = self._make_arg("Input") - instance = Expr.trim(arg1) + instance = Expression.trim(arg1) assert instance.name == "trim" assert instance.params == [arg1] assert repr(instance) == "Input.trim()" @@ -1063,7 +1036,7 @@ def test_trim(self): def test_string_reverse(self): arg1 = self._make_arg("Input") - instance = Expr.string_reverse(arg1) + instance = Expression.string_reverse(arg1) assert instance.name == "string_reverse" assert instance.params == [arg1] assert repr(instance) == "Input.string_reverse()" @@ -1073,7 +1046,7 @@ def test_string_reverse(self): def test_substring(self): arg1 = self._make_arg("Input") arg2 = self._make_arg("Position") - instance = Expr.substring(arg1, arg2) + instance = Expression.substring(arg1, arg2) assert instance.name == "substring" assert instance.params == [arg1, arg2] assert repr(instance) == "Input.substring(Position)" @@ -1084,7 +1057,7 @@ def test_substring_w_length(self): arg1 = self._make_arg("Input") arg2 = self._make_arg("Position") arg3 = self._make_arg("Length") - instance = Expr.substring(arg1, arg2, arg3) + instance = Expression.substring(arg1, arg2, arg3) assert instance.name == "substring" assert instance.params == [arg1, arg2, arg3] assert repr(instance) == "Input.substring(Position, Length)" @@ -1094,7 +1067,7 @@ def test_substring_w_length(self): def test_join(self): arg1 = self._make_arg("Array") arg2 = self._make_arg("Separator") - instance = Expr.join(arg1, arg2) + instance = Expression.join(arg1, arg2) assert instance.name == "join" assert instance.params == [arg1, arg2] assert repr(instance) == "Array.join(Separator)" @@ -1104,7 +1077,7 @@ def test_join(self): def test_map_get(self): arg1 = self._make_arg("Map") arg2 = "key" - instance = Expr.map_get(arg1, arg2) + instance = Expression.map_get(arg1, arg2) assert instance.name == "map_get" assert instance.params == [arg1, Constant.of(arg2)] assert repr(instance) == "Map.map_get(Constant.of('key'))" @@ -1114,7 +1087,7 @@ def test_map_get(self): def test_map_remove(self): arg1 = self._make_arg("Map") arg2 = "key" - instance = Expr.map_remove(arg1, arg2) + instance = Expression.map_remove(arg1, arg2) assert instance.name == "map_remove" assert instance.params == [arg1, Constant.of(arg2)] assert repr(instance) == "Map.map_remove(Constant.of('key'))" @@ -1125,7 +1098,7 @@ def test_map_merge(self): arg1 = expr.Map({"a": 1}) arg2 = expr.Map({"b": 2}) arg3 = {"c": 3} - instance = Expr.map_merge(arg1, arg2, arg3) + instance = Expression.map_merge(arg1, arg2, arg3) assert instance.name == "map_merge" assert instance.params == [arg1, arg2, expr.Map(arg3)] assert repr(instance) == "Map({'a': 1}).map_merge(Map({'b': 2}), Map({'c': 3}))" @@ -1135,7 +1108,7 @@ def test_map_merge(self): def test_mod(self): arg1 = self._make_arg("Left") arg2 = self._make_arg("Right") - instance = Expr.mod(arg1, arg2) + instance = Expression.mod(arg1, arg2) assert instance.name == "mod" assert instance.params == [arg1, arg2] assert repr(instance) == "Left.mod(Right)" @@ -1145,7 +1118,7 @@ def test_mod(self): def test_multiply(self): arg1 = self._make_arg("Left") arg2 = self._make_arg("Right") - instance = Expr.multiply(arg1, arg2) + instance = Expression.multiply(arg1, arg2) assert instance.name == "multiply" assert instance.params == [arg1, arg2] assert repr(instance) == "Left.multiply(Right)" @@ -1156,7 +1129,7 @@ def test_string_concat(self): arg1 = self._make_arg("Str1") arg2 = self._make_arg("Str2") arg3 = self._make_arg("Str3") - instance = Expr.string_concat(arg1, arg2, arg3) + instance = Expression.string_concat(arg1, arg2, arg3) assert instance.name == "string_concat" assert instance.params == [arg1, arg2, arg3] assert repr(instance) == "Str1.string_concat(Str2, Str3)" @@ -1166,7 +1139,7 @@ def test_string_concat(self): def test_subtract(self): arg1 = self._make_arg("Left") arg2 = self._make_arg("Right") - instance = Expr.subtract(arg1, arg2) + instance = Expression.subtract(arg1, arg2) assert instance.name == "subtract" assert instance.params == [arg1, arg2] assert repr(instance) == "Left.subtract(Right)" @@ -1183,7 +1156,7 @@ def test_timestamp_add(self): arg1 = self._make_arg("Timestamp") arg2 = self._make_arg("Unit") arg3 = self._make_arg("Amount") - instance = Expr.timestamp_add(arg1, arg2, arg3) + instance = Expression.timestamp_add(arg1, arg2, arg3) assert instance.name == "timestamp_add" assert instance.params == [arg1, arg2, arg3] assert repr(instance) == "Timestamp.timestamp_add(Unit, Amount)" @@ -1194,7 +1167,7 @@ def test_timestamp_subtract(self): arg1 = self._make_arg("Timestamp") arg2 = self._make_arg("Unit") arg3 = self._make_arg("Amount") - instance = Expr.timestamp_subtract(arg1, arg2, arg3) + instance = Expression.timestamp_subtract(arg1, arg2, arg3) assert instance.name == "timestamp_subtract" assert instance.params == [arg1, arg2, arg3] assert repr(instance) == "Timestamp.timestamp_subtract(Unit, Amount)" @@ -1203,7 +1176,7 @@ def test_timestamp_subtract(self): def test_timestamp_to_unix_micros(self): arg1 = self._make_arg("Input") - instance = Expr.timestamp_to_unix_micros(arg1) + instance = Expression.timestamp_to_unix_micros(arg1) assert instance.name == "timestamp_to_unix_micros" assert instance.params == [arg1] assert repr(instance) == "Input.timestamp_to_unix_micros()" @@ -1212,7 +1185,7 @@ def test_timestamp_to_unix_micros(self): def test_timestamp_to_unix_millis(self): arg1 = self._make_arg("Input") - instance = Expr.timestamp_to_unix_millis(arg1) + instance = Expression.timestamp_to_unix_millis(arg1) assert instance.name == "timestamp_to_unix_millis" assert instance.params == [arg1] assert repr(instance) == "Input.timestamp_to_unix_millis()" @@ -1221,7 +1194,7 @@ def test_timestamp_to_unix_millis(self): def test_timestamp_to_unix_seconds(self): arg1 = self._make_arg("Input") - instance = Expr.timestamp_to_unix_seconds(arg1) + instance = Expression.timestamp_to_unix_seconds(arg1) assert instance.name == "timestamp_to_unix_seconds" assert instance.params == [arg1] assert repr(instance) == "Input.timestamp_to_unix_seconds()" @@ -1230,7 +1203,7 @@ def test_timestamp_to_unix_seconds(self): def test_unix_micros_to_timestamp(self): arg1 = self._make_arg("Input") - instance = Expr.unix_micros_to_timestamp(arg1) + instance = Expression.unix_micros_to_timestamp(arg1) assert instance.name == "unix_micros_to_timestamp" assert instance.params == [arg1] assert repr(instance) == "Input.unix_micros_to_timestamp()" @@ -1239,7 +1212,7 @@ def test_unix_micros_to_timestamp(self): def test_unix_millis_to_timestamp(self): arg1 = self._make_arg("Input") - instance = Expr.unix_millis_to_timestamp(arg1) + instance = Expression.unix_millis_to_timestamp(arg1) assert instance.name == "unix_millis_to_timestamp" assert instance.params == [arg1] assert repr(instance) == "Input.unix_millis_to_timestamp()" @@ -1248,7 +1221,7 @@ def test_unix_millis_to_timestamp(self): def test_unix_seconds_to_timestamp(self): arg1 = self._make_arg("Input") - instance = Expr.unix_seconds_to_timestamp(arg1) + instance = Expression.unix_seconds_to_timestamp(arg1) assert instance.name == "unix_seconds_to_timestamp" assert instance.params == [arg1] assert repr(instance) == "Input.unix_seconds_to_timestamp()" @@ -1258,7 +1231,7 @@ def test_unix_seconds_to_timestamp(self): def test_euclidean_distance(self): arg1 = self._make_arg("Vector1") arg2 = self._make_arg("Vector2") - instance = Expr.euclidean_distance(arg1, arg2) + instance = Expression.euclidean_distance(arg1, arg2) assert instance.name == "euclidean_distance" assert instance.params == [arg1, arg2] assert repr(instance) == "Vector1.euclidean_distance(Vector2)" @@ -1268,7 +1241,7 @@ def test_euclidean_distance(self): def test_cosine_distance(self): arg1 = self._make_arg("Vector1") arg2 = self._make_arg("Vector2") - instance = Expr.cosine_distance(arg1, arg2) + instance = Expression.cosine_distance(arg1, arg2) assert instance.name == "cosine_distance" assert instance.params == [arg1, arg2] assert repr(instance) == "Vector1.cosine_distance(Vector2)" @@ -1278,7 +1251,7 @@ def test_cosine_distance(self): def test_dot_product(self): arg1 = self._make_arg("Vector1") arg2 = self._make_arg("Vector2") - instance = Expr.dot_product(arg1, arg2) + instance = Expression.dot_product(arg1, arg2) assert instance.name == "dot_product" assert instance.params == [arg1, arg2] assert repr(instance) == "Vector1.dot_product(Vector2)" @@ -1305,7 +1278,7 @@ def test_vector_ctor(self, method, input): def test_vector_length(self): arg1 = self._make_arg("Array") - instance = Expr.vector_length(arg1) + instance = Expression.vector_length(arg1) assert instance.name == "vector_length" assert instance.params == [arg1] assert repr(instance) == "Array.vector_length()" @@ -1315,7 +1288,7 @@ def test_vector_length(self): def test_add(self): arg1 = self._make_arg("Left") arg2 = self._make_arg("Right") - instance = Expr.add(arg1, arg2) + instance = Expression.add(arg1, arg2) assert instance.name == "add" assert instance.params == [arg1, arg2] assert repr(instance) == "Left.add(Right)" @@ -1324,7 +1297,7 @@ def test_add(self): def test_abs(self): arg1 = self._make_arg("Value") - instance = Expr.abs(arg1) + instance = Expression.abs(arg1) assert instance.name == "abs" assert instance.params == [arg1] assert repr(instance) == "Value.abs()" @@ -1333,7 +1306,7 @@ def test_abs(self): def test_ceil(self): arg1 = self._make_arg("Value") - instance = Expr.ceil(arg1) + instance = Expression.ceil(arg1) assert instance.name == "ceil" assert instance.params == [arg1] assert repr(instance) == "Value.ceil()" @@ -1342,7 +1315,7 @@ def test_ceil(self): def test_exp(self): arg1 = self._make_arg("Value") - instance = Expr.exp(arg1) + instance = Expression.exp(arg1) assert instance.name == "exp" assert instance.params == [arg1] assert repr(instance) == "Value.exp()" @@ -1351,7 +1324,7 @@ def test_exp(self): def test_floor(self): arg1 = self._make_arg("Value") - instance = Expr.floor(arg1) + instance = Expression.floor(arg1) assert instance.name == "floor" assert instance.params == [arg1] assert repr(instance) == "Value.floor()" @@ -1360,7 +1333,7 @@ def test_floor(self): def test_ln(self): arg1 = self._make_arg("Value") - instance = Expr.ln(arg1) + instance = Expression.ln(arg1) assert instance.name == "ln" assert instance.params == [arg1] assert repr(instance) == "Value.ln()" @@ -1370,7 +1343,7 @@ def test_ln(self): def test_log(self): arg1 = self._make_arg("Value") arg2 = self._make_arg("Base") - instance = Expr.log(arg1, arg2) + instance = Expression.log(arg1, arg2) assert instance.name == "log" assert instance.params == [arg1, arg2] assert repr(instance) == "Value.log(Base)" @@ -1379,7 +1352,7 @@ def test_log(self): def test_log10(self): arg1 = self._make_arg("Value") - instance = Expr.log10(arg1) + instance = Expression.log10(arg1) assert instance.name == "log10" assert instance.params == [arg1] assert repr(instance) == "Value.log10()" @@ -1389,7 +1362,7 @@ def test_log10(self): def test_pow(self): arg1 = self._make_arg("Value") arg2 = self._make_arg("Exponent") - instance = Expr.pow(arg1, arg2) + instance = Expression.pow(arg1, arg2) assert instance.name == "pow" assert instance.params == [arg1, arg2] assert repr(instance) == "Value.pow(Exponent)" @@ -1398,7 +1371,7 @@ def test_pow(self): def test_round(self): arg1 = self._make_arg("Value") - instance = Expr.round(arg1) + instance = Expression.round(arg1) assert instance.name == "round" assert instance.params == [arg1] assert repr(instance) == "Value.round()" @@ -1407,7 +1380,7 @@ def test_round(self): def test_sqrt(self): arg1 = self._make_arg("Value") - instance = Expr.sqrt(arg1) + instance = Expression.sqrt(arg1) assert instance.name == "sqrt" assert instance.params == [arg1] assert repr(instance) == "Value.sqrt()" @@ -1416,7 +1389,7 @@ def test_sqrt(self): def test_array_length(self): arg1 = self._make_arg("Array") - instance = Expr.array_length(arg1) + instance = Expression.array_length(arg1) assert instance.name == "array_length" assert instance.params == [arg1] assert repr(instance) == "Array.array_length()" @@ -1425,7 +1398,7 @@ def test_array_length(self): def test_array_reverse(self): arg1 = self._make_arg("Array") - instance = Expr.array_reverse(arg1) + instance = Expression.array_reverse(arg1) assert instance.name == "array_reverse" assert instance.params == [arg1] assert repr(instance) == "Array.array_reverse()" @@ -1435,7 +1408,7 @@ def test_array_reverse(self): def test_array_concat(self): arg1 = self._make_arg("ArrayRef1") arg2 = self._make_arg("ArrayRef2") - instance = Expr.array_concat(arg1, arg2) + instance = Expression.array_concat(arg1, arg2) assert instance.name == "array_concat" assert instance.params == [arg1, arg2] assert repr(instance) == "ArrayRef1.array_concat(ArrayRef2)" @@ -1456,20 +1429,20 @@ def test_array_concat_multiple(self): ) def test_byte_length(self): - arg1 = self._make_arg("Expr") - instance = Expr.byte_length(arg1) + arg1 = self._make_arg("Expression") + instance = Expression.byte_length(arg1) assert instance.name == "byte_length" assert instance.params == [arg1] - assert repr(instance) == "Expr.byte_length()" + assert repr(instance) == "Expression.byte_length()" infix_instance = arg1.byte_length() assert infix_instance == instance def test_char_length(self): - arg1 = self._make_arg("Expr") - instance = Expr.char_length(arg1) + arg1 = self._make_arg("Expression") + instance = Expression.char_length(arg1) assert instance.name == "char_length" assert instance.params == [arg1] - assert repr(instance) == "Expr.char_length()" + assert repr(instance) == "Expression.char_length()" infix_instance = arg1.char_length() assert infix_instance == instance @@ -1477,7 +1450,7 @@ def test_concat(self): arg1 = self._make_arg("First") arg2 = self._make_arg("Second") arg3 = "Third" - instance = Expr.concat(arg1, arg2, arg3) + instance = Expression.concat(arg1, arg2, arg3) assert instance.name == "concat" assert instance.params == [arg1, arg2, Constant.of(arg3)] assert repr(instance) == "First.concat(Second, Constant.of('Third'))" @@ -1485,17 +1458,17 @@ def test_concat(self): assert infix_instance == instance def test_length(self): - arg1 = self._make_arg("Expr") - instance = Expr.length(arg1) + arg1 = self._make_arg("Expression") + instance = Expression.length(arg1) assert instance.name == "length" assert instance.params == [arg1] - assert repr(instance) == "Expr.length()" + assert repr(instance) == "Expression.length()" infix_instance = arg1.length() assert infix_instance == instance def test_collection_id(self): arg1 = self._make_arg("Value") - instance = Expr.collection_id(arg1) + instance = Expression.collection_id(arg1) assert instance.name == "collection_id" assert instance.params == [arg1] assert repr(instance) == "Value.collection_id()" @@ -1504,7 +1477,7 @@ def test_collection_id(self): def test_document_id(self): arg1 = self._make_arg("Value") - instance = Expr.document_id(arg1) + instance = Expression.document_id(arg1) assert instance.name == "document_id" assert instance.params == [arg1] assert repr(instance) == "Value.document_id()" @@ -1513,7 +1486,7 @@ def test_document_id(self): def test_sum(self): arg1 = self._make_arg("Value") - instance = Expr.sum(arg1) + instance = Expression.sum(arg1) assert instance.name == "sum" assert instance.params == [arg1] assert repr(instance) == "Value.sum()" @@ -1522,7 +1495,7 @@ def test_sum(self): def test_average(self): arg1 = self._make_arg("Value") - instance = Expr.average(arg1) + instance = Expression.average(arg1) assert instance.name == "average" assert instance.params == [arg1] assert repr(instance) == "Value.average()" @@ -1531,7 +1504,7 @@ def test_average(self): def test_count(self): arg1 = self._make_arg("Value") - instance = Expr.count(arg1) + instance = Expression.count(arg1) assert instance.name == "count" assert instance.params == [arg1] assert repr(instance) == "Value.count()" @@ -1546,7 +1519,7 @@ def test_base_count(self): def test_count_if(self): arg1 = self._make_arg("Value") - instance = Expr.count_if(arg1) + instance = Expression.count_if(arg1) assert instance.name == "count_if" assert instance.params == [arg1] assert repr(instance) == "Value.count_if()" @@ -1555,7 +1528,7 @@ def test_count_if(self): def test_count_distinct(self): arg1 = self._make_arg("Value") - instance = Expr.count_distinct(arg1) + instance = Expression.count_distinct(arg1) assert instance.name == "count_distinct" assert instance.params == [arg1] assert repr(instance) == "Value.count_distinct()" @@ -1564,7 +1537,7 @@ def test_count_distinct(self): def test_minimum(self): arg1 = self._make_arg("Value") - instance = Expr.minimum(arg1) + instance = Expression.minimum(arg1) assert instance.name == "minimum" assert instance.params == [arg1] assert repr(instance) == "Value.minimum()" @@ -1573,7 +1546,7 @@ def test_minimum(self): def test_maximum(self): arg1 = self._make_arg("Value") - instance = Expr.maximum(arg1) + instance = Expression.maximum(arg1) assert instance.name == "maximum" assert instance.params == [arg1] assert repr(instance) == "Value.maximum()" From 9eb32e10eb6fdcef3b3e7370dfbdce7800c05e4b Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 28 Oct 2025 16:00:48 -0700 Subject: [PATCH 30/36] renamed variables --- google/cloud/firestore_v1/base_query.py | 2 +- tests/system/pipeline_e2e/aggregates.yaml | 67 +++++--- tests/system/pipeline_e2e/array.yaml | 40 ++--- tests/system/pipeline_e2e/date_and_time.yaml | 42 ++--- tests/system/pipeline_e2e/general.yaml | 160 ++----------------- tests/system/pipeline_e2e/logical.yaml | 88 +++++----- tests/system/pipeline_e2e/map.yaml | 32 ++-- tests/system/pipeline_e2e/math.yaml | 84 +++++----- tests/system/pipeline_e2e/string.yaml | 104 ++++++------ tests/system/pipeline_e2e/vector.yaml | 22 +-- tests/unit/v1/test_aggregation.py | 12 +- tests/unit/v1/test_base_query.py | 2 +- 12 files changed, 272 insertions(+), 383 deletions(-) diff --git a/google/cloud/firestore_v1/base_query.py b/google/cloud/firestore_v1/base_query.py index 797572b1b..0f4347e5f 100644 --- a/google/cloud/firestore_v1/base_query.py +++ b/google/cloud/firestore_v1/base_query.py @@ -1151,7 +1151,7 @@ def pipeline(self): # Filters for filter_ in self._field_filters: ppl = ppl.where( - pipeline_expressions.BooleanExpr._from_query_filter_pb( + pipeline_expressions.BooleanExpression._from_query_filter_pb( filter_, self._client ) ) diff --git a/tests/system/pipeline_e2e/aggregates.yaml b/tests/system/pipeline_e2e/aggregates.yaml index 18902aff4..9593213ed 100644 --- a/tests/system/pipeline_e2e/aggregates.yaml +++ b/tests/system/pipeline_e2e/aggregates.yaml @@ -3,8 +3,8 @@ tests: pipeline: - Collection: books - Aggregate: - - AliasedExpr: - - Expr.count: + - AliasedExpression: + - Function.count: - Field: rating - "count" assert_results: @@ -29,9 +29,9 @@ tests: pipeline: - Collection: books - Aggregate: - - AliasedExpr: - - Expr.count_if: - - Expr.greater_than: + - AliasedExpression: + - Function.count_if: + - Function.greater_than: - Field: rating - Constant: 4.2 - "count_if_rating_gt_4_2" @@ -61,8 +61,8 @@ tests: pipeline: - Collection: books - Aggregate: - - AliasedExpr: - - Expr.count_distinct: + - AliasedExpression: + - Function.count_distinct: - Field: genre - "distinct_genres" assert_results: @@ -87,20 +87,20 @@ tests: pipeline: - Collection: books - Where: - - Expr.equal: + - Function.equal: - Field: genre - Constant: Science Fiction - Aggregate: - - AliasedExpr: - - Expr.count: + - AliasedExpression: + - Function.count: - Field: rating - "count" - - AliasedExpr: - - Expr.average: + - AliasedExpression: + - Function.average: - Field: rating - "avg_rating" - - AliasedExpr: - - Expr.maximum: + - AliasedExpression: + - Function.maximum: - Field: rating - "max_rating" assert_results: @@ -144,7 +144,7 @@ tests: pipeline: - Collection: books - Where: - - Expr.less_than: + - Function.less_than: - Field: published - Constant: 1900 - Aggregate: @@ -155,18 +155,18 @@ tests: pipeline: - Collection: books - Where: - - Expr.less_than: + - Function.less_than: - Field: published - Constant: 1984 - Aggregate: accumulators: - - AliasedExpr: - - Expr.average: + - AliasedExpression: + - Function.average: - Field: rating - "avg_rating" groups: [genre] - Where: - - Expr.greater_than: + - Function.greater_than: - Field: avg_rating - Constant: 4.3 - Sort: @@ -225,16 +225,16 @@ tests: pipeline: - Collection: books - Aggregate: - - AliasedExpr: - - Expr.count: + - AliasedExpression: + - Function.count: - Field: rating - "count" - - AliasedExpr: - - Expr.maximum: + - AliasedExpression: + - Function.maximum: - Field: rating - "max_rating" - - AliasedExpr: - - Expr.minimum: + - AliasedExpression: + - Function.minimum: - Field: published - "min_published" assert_results: @@ -266,4 +266,19 @@ tests: - fieldReferenceValue: published name: minimum - mapValue: {} - name: aggregate \ No newline at end of file + name: aggregate + - description: testSum + pipeline: + - Collection: books + - Where: + - Function.equal: + - Field: genre + - Constant: Science Fiction + - Aggregate: + - AliasedExpression: + - Function.sum: + - Field: rating + - "total_rating" + assert_results: + - total_rating: 8.8 + diff --git a/tests/system/pipeline_e2e/array.yaml b/tests/system/pipeline_e2e/array.yaml index 6e99f38ef..3da16264d 100644 --- a/tests/system/pipeline_e2e/array.yaml +++ b/tests/system/pipeline_e2e/array.yaml @@ -3,7 +3,7 @@ tests: pipeline: - Collection: books - Where: - - Expr.array_contains: + - Function.array_contains: - Field: tags - Constant: comedy assert_results: @@ -33,7 +33,7 @@ tests: pipeline: - Collection: books - Where: - - Expr.array_contains_any: + - Function.array_contains_any: - Field: tags - - Constant: comedy - Constant: classic @@ -81,7 +81,7 @@ tests: pipeline: - Collection: books - Where: - - Expr.array_contains_all: + - Function.array_contains_all: - Field: tags - - Constant: adventure - Constant: magic @@ -116,12 +116,12 @@ tests: pipeline: - Collection: books - Select: - - AliasedExpr: - - Expr.array_length: + - AliasedExpression: + - Function.array_length: - Field: tags - "tagsCount" - Where: - - Expr.equal: + - Function.equal: - Field: tagsCount - Constant: 3 assert_results: # All documents have 3 tags @@ -161,12 +161,12 @@ tests: pipeline: - Collection: books - Where: - - Expr.equal: + - Function.equal: - Field: title - Constant: "The Hitchhiker's Guide to the Galaxy" - Select: - - AliasedExpr: - - Expr.array_reverse: + - AliasedExpression: + - Function.array_reverse: - Field: tags - "reversedTags" assert_results: @@ -178,12 +178,12 @@ tests: pipeline: - Collection: books - Where: - - Expr.equal: + - Function.equal: - Field: title - Constant: "The Hitchhiker's Guide to the Galaxy" - Select: - - AliasedExpr: - - Expr.array_concat: + - AliasedExpression: + - Function.array_concat: - Field: tags - Constant: ["new_tag", "another_tag"] - "concatenatedTags" @@ -225,12 +225,12 @@ tests: pipeline: - Collection: books - Where: - - Expr.equal: + - Function.equal: - Field: title - Constant: "Dune" - Select: - - AliasedExpr: - - Expr.array_concat: + - AliasedExpression: + - Function.array_concat: - Field: tags - Constant: ["sci-fi"] - Constant: ["classic", "epic"] @@ -278,13 +278,13 @@ tests: pipeline: - Collection: books - AddFields: - - AliasedExpr: - - Expr.array_concat: + - AliasedExpression: + - Function.array_concat: - Field: tags - Array: ["Dystopian"] - "new_tags" - Where: - - Expr.array_contains_any: + - Function.array_contains_any: - Field: new_tags - - Constant: non_existent_tag - Field: genre @@ -351,8 +351,8 @@ tests: - Collection: books - Limit: 1 - Select: - - AliasedExpr: - - Expr.array_concat: + - AliasedExpression: + - Function.array_concat: - Array: [1, 2, 3] - Array: [4, 5] - "concatenated" diff --git a/tests/system/pipeline_e2e/date_and_time.yaml b/tests/system/pipeline_e2e/date_and_time.yaml index bbb5f34fe..cb5323dc1 100644 --- a/tests/system/pipeline_e2e/date_and_time.yaml +++ b/tests/system/pipeline_e2e/date_and_time.yaml @@ -4,15 +4,15 @@ tests: - Collection: books - Limit: 1 - Select: - - AliasedExpr: + - AliasedExpression: - And: - - Expr.greater_than_or_equal: + - Function.greater_than_or_equal: - CurrentTimestamp: [] - - Expr.unix_seconds_to_timestamp: + - Function.unix_seconds_to_timestamp: - Constant: 1735689600 # 2025-01-01 - - Expr.less_than: + - Function.less_than: - CurrentTimestamp: [] - - Expr.unix_seconds_to_timestamp: + - Function.unix_seconds_to_timestamp: - Constant: 4892438400 # 2125-01-01 - "is_between_2025_and_2125" assert_results: @@ -56,38 +56,38 @@ tests: pipeline: - Collection: timestamps - Select: - - AliasedExpr: - - Expr.timestamp_to_unix_micros: + - AliasedExpression: + - Function.timestamp_to_unix_micros: - Field: time - "micros" - - AliasedExpr: - - Expr.timestamp_to_unix_millis: + - AliasedExpression: + - Function.timestamp_to_unix_millis: - Field: time - "millis" - - AliasedExpr: - - Expr.timestamp_to_unix_seconds: + - AliasedExpression: + - Function.timestamp_to_unix_seconds: - Field: time - "seconds" - - AliasedExpr: - - Expr.unix_micros_to_timestamp: + - AliasedExpression: + - Function.unix_micros_to_timestamp: - Field: micros - "from_micros" - - AliasedExpr: - - Expr.unix_millis_to_timestamp: + - AliasedExpression: + - Function.unix_millis_to_timestamp: - Field: millis - "from_millis" - - AliasedExpr: - - Expr.unix_seconds_to_timestamp: + - AliasedExpression: + - Function.unix_seconds_to_timestamp: - Field: seconds - "from_seconds" - - AliasedExpr: - - Expr.timestamp_add: + - AliasedExpression: + - Function.timestamp_add: - Field: time - Constant: "day" - Constant: 1 - "plus_day" - - AliasedExpr: - - Expr.timestamp_subtract: + - AliasedExpression: + - Function.timestamp_subtract: - Field: time - Constant: "hour" - Constant: 1 diff --git a/tests/system/pipeline_e2e/general.yaml b/tests/system/pipeline_e2e/general.yaml index c135853d1..90b77a9a5 100644 --- a/tests/system/pipeline_e2e/general.yaml +++ b/tests/system/pipeline_e2e/general.yaml @@ -56,14 +56,14 @@ tests: pipeline: - Collection: books - AddFields: - - AliasedExpr: - - Expr.string_concat: + - AliasedExpression: + - Function.string_concat: - Field: author - Constant: _ - Field: title - "author_title" - - AliasedExpr: - - Expr.string_concat: + - AliasedExpression: + - Function.string_concat: - Field: title - Constant: _ - Field: author @@ -144,7 +144,6 @@ tests: expression: fieldReferenceValue: author_title name: sort - - description: testPipelineWithOffsetAndLimit pipeline: - Collection: books @@ -192,116 +191,6 @@ tests: title: fieldReferenceValue: title name: select - - description: testArithmeticOperations - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: title - - Constant: To Kill a Mockingbird - - Select: - - AliasedExpr: - - Expr.add: - - Field: rating - - Constant: 1 - - "ratingPlusOne" - - AliasedExpr: - - Expr.subtract: - - Field: published - - Constant: 1900 - - "yearsSince1900" - - AliasedExpr: - - Expr.multiply: - - Field: rating - - Constant: 10 - - "ratingTimesTen" - - AliasedExpr: - - Expr.divide: - - Field: rating - - Constant: 2 - - "ratingDividedByTwo" - - AliasedExpr: - - Expr.multiply: - - Field: rating - - Constant: 20 - - "ratingTimes20" - - AliasedExpr: - - Expr.add: - - Field: rating - - Constant: 3 - - "ratingPlus3" - - AliasedExpr: - - Expr.mod: - - Field: rating - - Constant: 2 - - "ratingMod2" - assert_results: - - ratingPlusOne: 5.2 - yearsSince1900: 60 - ratingTimesTen: 42.0 - ratingDividedByTwo: 2.1 - ratingTimes20: 84 - ratingPlus3: 7.2 - ratingMod2: 0.20000000000000018 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: title - - stringValue: To Kill a Mockingbird - name: equal - name: where - - args: - - mapValue: - fields: - ratingDividedByTwo: - functionValue: - args: - - fieldReferenceValue: rating - - integerValue: '2' - name: divide - ratingPlusOne: - functionValue: - args: - - fieldReferenceValue: rating - - integerValue: '1' - name: add - ratingTimesTen: - functionValue: - args: - - fieldReferenceValue: rating - - integerValue: '10' - name: multiply - yearsSince1900: - functionValue: - args: - - fieldReferenceValue: published - - integerValue: '1900' - name: subtract - ratingTimes20: - functionValue: - args: - - fieldReferenceValue: rating - - integerValue: '20' - name: multiply - ratingPlus3: - functionValue: - args: - - fieldReferenceValue: rating - - integerValue: '3' - name: add - ratingMod2: - functionValue: - args: - - fieldReferenceValue: rating - - integerValue: '2' - name: mod - name: select - description: testSampleLimit pipeline: - Collection: books @@ -338,14 +227,14 @@ tests: pipeline: - Collection: books - Where: - - Expr.equal: + - Function.equal: - Field: genre - Constant: Romance - Union: - Pipeline: - Collection: books - Where: - - Expr.equal: + - Function.equal: - Field: genre - Constant: Dystopian - Select: @@ -410,12 +299,12 @@ tests: pipeline: - Collection: books - Where: - - Expr.equal: + - Function.equal: - Field: title - Constant: "The Hitchhiker's Guide to the Galaxy" - Select: - - AliasedExpr: - - Expr.document_id: + - AliasedExpression: + - Function.document_id: - Field: __name__ - "doc_id" assert_results: @@ -442,28 +331,13 @@ tests: args: - fieldReferenceValue: __name__ name: select - - description: testSum - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: genre - - Constant: Science Fiction - - Aggregate: - - AliasedExpr: - - Expr.sum: - - Field: rating - - "total_rating" - assert_results: - - total_rating: 8.8 - - description: testCollectionId pipeline: - Collection: books - Limit: 1 - Select: - - AliasedExpr: - - Expr.collection_id: + - AliasedExpression: + - Function.collection_id: - Field: __name__ - "collectionName" assert_results: @@ -601,13 +475,13 @@ tests: - Distinct: - title - Aggregate: - - AliasedExpr: + - AliasedExpression: - Count: [] - count - Select: - - AliasedExpr: + - AliasedExpression: - Conditional: - - Expr.greater_than_or_equal: + - Function.greater_than_or_equal: - Field: count - Constant: 10 - Constant: True @@ -662,7 +536,7 @@ tests: reference_value: "/books" - GenericStage: - "where" - - Expr.equal: + - Function.equal: - Field: title - Constant: The Hitchhiker's Guide to the Galaxy - GenericStage: @@ -697,7 +571,7 @@ tests: pipeline: - Collection: books - Where: - - Expr.equal: + - Function.equal: - Field: title - Constant: The Hitchhiker's Guide to the Galaxy - Unnest: @@ -735,7 +609,7 @@ tests: pipeline: - Collection: books - Where: - - Expr.equal: + - Function.equal: - Field: title - Constant: The Hitchhiker's Guide to the Galaxy - Unnest: diff --git a/tests/system/pipeline_e2e/logical.yaml b/tests/system/pipeline_e2e/logical.yaml index 203be290d..efc91c29a 100644 --- a/tests/system/pipeline_e2e/logical.yaml +++ b/tests/system/pipeline_e2e/logical.yaml @@ -4,10 +4,10 @@ tests: - Collection: books - Where: - And: - - Expr.greater_than: + - Function.greater_than: - Field: rating - Constant: 4.5 - - Expr.equal: + - Function.equal: - Field: genre - Constant: Science Fiction assert_results: @@ -49,10 +49,10 @@ tests: - Collection: books - Where: - Or: - - Expr.equal: + - Function.equal: - Field: genre - Constant: Romance - - Expr.equal: + - Function.equal: - Field: genre - Constant: Dystopian - Select: @@ -105,13 +105,13 @@ tests: - Collection: books - Where: - And: - - Expr.greater_than: + - Function.greater_than: - Field: rating - Constant: 4.2 - - Expr.less_than_or_equal: + - Function.less_than_or_equal: - Field: rating - Constant: 4.5 - - Expr.not_equal: + - Function.not_equal: - Field: genre - Constant: Science Fiction - Select: @@ -176,13 +176,13 @@ tests: - Where: - Or: - And: - - Expr.greater_than: + - Function.greater_than: - Field: rating - Constant: 4.5 - - Expr.equal: + - Function.equal: - Field: genre - Constant: Science Fiction - - Expr.less_than: + - Function.less_than: - Field: published - Constant: 1900 - Select: @@ -243,12 +243,12 @@ tests: - Collection: books - Where: - Not: - - Expr.is_nan: + - Function.is_nan: - Field: rating - Select: - - AliasedExpr: + - AliasedExpression: - Not: - - Expr.is_nan: + - Function.is_nan: - Field: rating - "ratingIsNotNaN" - Limit: 1 @@ -288,7 +288,7 @@ tests: pipeline: - Collection: books - Where: - - Expr.is_not_null: + - Function.is_not_null: - Field: rating assert_count: 10 assert_proto: @@ -307,7 +307,7 @@ tests: pipeline: - Collection: books - Where: - - Expr.is_not_nan: + - Function.is_not_nan: - Field: rating assert_count: 10 assert_proto: @@ -326,7 +326,7 @@ tests: pipeline: - Collection: books - Where: - - Expr.is_absent: + - Function.is_absent: - Field: awards.pulitzer assert_count: 9 assert_proto: @@ -345,14 +345,14 @@ tests: pipeline: - Collection: books - Select: - - AliasedExpr: - - Expr.if_absent: + - AliasedExpression: + - Function.if_absent: - Field: awards.pulitzer - Constant: false - "pulitzer_award" - title - Where: - - Expr.equal: + - Function.equal: - Field: pulitzer_award - Constant: true assert_results: @@ -387,9 +387,9 @@ tests: pipeline: - Collection: books - Select: - - AliasedExpr: - - Expr.is_error: - - Expr.divide: + - AliasedExpression: + - Function.is_error: + - Function.divide: - Field: rating - Constant: "string" - "is_error_result" @@ -422,9 +422,9 @@ tests: pipeline: - Collection: books - Select: - - AliasedExpr: - - Expr.if_error: - - Expr.divide: + - AliasedExpression: + - Function.if_error: + - Function.divide: - Field: rating - Field: genre - Constant: "An error occurred" @@ -459,17 +459,17 @@ tests: pipeline: - Collection: books - Where: - - Expr.equal: + - Function.equal: - Field: author - Constant: Douglas Adams - Select: - - AliasedExpr: - - Expr.logical_maximum: + - AliasedExpression: + - Function.logical_maximum: - Field: rating - Constant: 4.5 - "max_rating" - - AliasedExpr: - - Expr.logical_minimum: + - AliasedExpression: + - Function.logical_minimum: - Field: published - Constant: 1900 - "min_published" @@ -509,7 +509,7 @@ tests: pipeline: - Collection: books - Where: - - Expr.greater_than_or_equal: + - Function.greater_than_or_equal: - Field: rating - Constant: 4.6 - Select: @@ -529,11 +529,11 @@ tests: - Collection: books - Where: - And: - - Expr.equal_any: + - Function.equal_any: - Field: genre - - Constant: Romance - Constant: Dystopian - - Expr.not_equal_any: + - Function.not_equal_any: - Field: author - - Constant: "George Orwell" assert_results: @@ -565,9 +565,9 @@ tests: - Collection: books - Where: - And: - - Expr.exists: + - Function.exists: - Field: awards.pulitzer - - Expr.equal: + - Function.equal: - Field: awards.pulitzer - Constant: true - Select: @@ -579,10 +579,10 @@ tests: - Collection: books - Where: - Xor: - - - Expr.equal: + - - Function.equal: - Field: genre - Constant: Romance - - Expr.greater_than: + - Function.greater_than: - Field: published - Constant: 1980 - Select: @@ -605,9 +605,9 @@ tests: - Collection: books - Select: - title - - AliasedExpr: + - AliasedExpression: - Conditional: - - Expr.greater_than: + - Function.greater_than: - Field: published - Constant: 1950 - Constant: "Modern" @@ -631,7 +631,7 @@ tests: pipeline: - Collection: books - Where: - - Expr.greater_than: + - Function.greater_than: - Field: published - Field: rating - Select: @@ -641,22 +641,22 @@ tests: pipeline: - Collection: books - Where: - - Expr.exists: + - Function.exists: - Field: non_existent_field assert_count: 0 - description: testConditionalWithFields pipeline: - Collection: books - Where: - - Expr.equal_any: + - Function.equal_any: - Field: title - - Constant: "Dune" - Constant: "1984" - Select: - title - - AliasedExpr: + - AliasedExpression: - Conditional: - - Expr.greater_than: + - Function.greater_than: - Field: published - Constant: 1950 - Field: author diff --git a/tests/system/pipeline_e2e/map.yaml b/tests/system/pipeline_e2e/map.yaml index 638fe0798..546af1351 100644 --- a/tests/system/pipeline_e2e/map.yaml +++ b/tests/system/pipeline_e2e/map.yaml @@ -7,14 +7,14 @@ tests: - Field: published - DESCENDING - Select: - - AliasedExpr: - - Expr.map_get: + - AliasedExpression: + - Function.map_get: - Field: awards - hugo - "hugoAward" - Field: title - Where: - - Expr.equal: + - Function.equal: - Field: hugoAward - Constant: true assert_results: @@ -59,16 +59,16 @@ tests: pipeline: - Collection: books - Where: - - Expr.equal: + - Function.equal: - Field: title - Constant: "Dune" - AddFields: - - AliasedExpr: + - AliasedExpression: - Constant: "hugo" - "award_name" - Select: - - AliasedExpr: - - Expr.map_get: + - AliasedExpression: + - Function.map_get: - Field: awards - Field: award_name - "hugoAward" @@ -111,12 +111,12 @@ tests: pipeline: - Collection: books - Where: - - Expr.equal: + - Function.equal: - Field: title - Constant: "Dune" - Select: - - AliasedExpr: - - Expr.map_remove: + - AliasedExpression: + - Function.map_remove: - Field: awards - "nebula" - "awards_removed" @@ -150,12 +150,12 @@ tests: pipeline: - Collection: books - Where: - - Expr.equal: + - Function.equal: - Field: title - Constant: "Dune" - Select: - - AliasedExpr: - - Expr.map_merge: + - AliasedExpression: + - Function.map_merge: - Field: awards - Map: elements: {"new_award": true, "hugo": false} @@ -206,7 +206,7 @@ tests: pipeline: - Collection: books - Where: - - Expr.equal: + - Function.equal: - Field: awards.hugo - Constant: true - Sort: @@ -255,8 +255,8 @@ tests: - Collection: books - Limit: 1 - Select: - - AliasedExpr: - - Expr.map_merge: + - AliasedExpression: + - Function.map_merge: - Map: elements: {"a": "orig", "b": "orig"} - Map: diff --git a/tests/system/pipeline_e2e/math.yaml b/tests/system/pipeline_e2e/math.yaml index a5a47d4c0..b62c0510b 100644 --- a/tests/system/pipeline_e2e/math.yaml +++ b/tests/system/pipeline_e2e/math.yaml @@ -3,61 +3,61 @@ tests: pipeline: - Collection: books - Where: - - Expr.equal: + - Function.equal: - Field: title - Constant: "Dune" - Select: - - AliasedExpr: - - Expr.add: + - AliasedExpression: + - Function.add: - Field: published - Field: rating - "pub_plus_rating" assert_results: - pub_plus_rating: 1969.6 - - description: testMathExpressions + - description: testMathFunctionessions pipeline: - Collection: books - Where: - - Expr.equal: + - Function.equal: - Field: title - Constant: To Kill a Mockingbird - Select: - - AliasedExpr: - - Expr.abs: + - AliasedExpression: + - Function.abs: - Field: rating - "abs_rating" - - AliasedExpr: - - Expr.ceil: + - AliasedExpression: + - Function.ceil: - Field: rating - "ceil_rating" - - AliasedExpr: - - Expr.exp: + - AliasedExpression: + - Function.exp: - Field: rating - "exp_rating" - - AliasedExpr: - - Expr.floor: + - AliasedExpression: + - Function.floor: - Field: rating - "floor_rating" - - AliasedExpr: - - Expr.ln: + - AliasedExpression: + - Function.ln: - Field: rating - "ln_rating" - - AliasedExpr: - - Expr.log10: + - AliasedExpression: + - Function.log10: - Field: rating - "log_rating_base10" - - AliasedExpr: - - Expr.log: + - AliasedExpression: + - Function.log: - Field: rating - Constant: 2 - "log_rating_base2" - - AliasedExpr: - - Expr.pow: + - AliasedExpression: + - Function.pow: - Field: rating - Constant: 2 - "pow_rating" - - AliasedExpr: - - Expr.sqrt: + - AliasedExpression: + - Function.sqrt: - Field: rating - "sqrt_rating" assert_results_approximate: @@ -134,19 +134,19 @@ tests: - fieldReferenceValue: rating name: sqrt name: select - - description: testRoundExpressions + - description: testRoundFunctionessions pipeline: - Collection: books - Where: - - Expr.equal_any: + - Function.equal_any: - Field: title - - Constant: "To Kill a Mockingbird" # rating 4.2 - Constant: "Pride and Prejudice" # rating 4.5 - Constant: "The Lord of the Rings" # rating 4.7 - Select: - title - - AliasedExpr: - - Expr.round: + - AliasedExpression: + - Function.round: - Field: rating - "round_rating" - Sort: @@ -201,42 +201,42 @@ tests: pipeline: - Collection: books - Where: - - Expr.equal: + - Function.equal: - Field: title - Constant: To Kill a Mockingbird - Select: - - AliasedExpr: - - Expr.add: + - AliasedExpression: + - Function.add: - Field: rating - Constant: 1 - "ratingPlusOne" - - AliasedExpr: - - Expr.subtract: + - AliasedExpression: + - Function.subtract: - Field: published - Constant: 1900 - "yearsSince1900" - - AliasedExpr: - - Expr.multiply: + - AliasedExpression: + - Function.multiply: - Field: rating - Constant: 10 - "ratingTimesTen" - - AliasedExpr: - - Expr.divide: + - AliasedExpression: + - Function.divide: - Field: rating - Constant: 2 - "ratingDividedByTwo" - - AliasedExpr: - - Expr.multiply: + - AliasedExpression: + - Function.multiply: - Field: rating - Constant: 20 - "ratingTimes20" - - AliasedExpr: - - Expr.add: + - AliasedExpression: + - Function.add: - Field: rating - Constant: 3 - "ratingPlus3" - - AliasedExpr: - - Expr.mod: + - AliasedExpression: + - Function.mod: - Field: rating - Constant: 2 - "ratingMod2" diff --git a/tests/system/pipeline_e2e/string.yaml b/tests/system/pipeline_e2e/string.yaml index b1e3a0b64..d612483e1 100644 --- a/tests/system/pipeline_e2e/string.yaml +++ b/tests/system/pipeline_e2e/string.yaml @@ -7,8 +7,8 @@ tests: - Field: author - ASCENDING - Select: - - AliasedExpr: - - Expr.string_concat: + - AliasedExpression: + - Function.string_concat: - Field: author - Constant: " - " - Field: title @@ -48,7 +48,7 @@ tests: pipeline: - Collection: books - Where: - - Expr.starts_with: + - Function.starts_with: - Field: title - Constant: The - Select: @@ -93,7 +93,7 @@ tests: pipeline: - Collection: books - Where: - - Expr.ends_with: + - Function.ends_with: - Field: title - Constant: y - Select: @@ -136,18 +136,18 @@ tests: pipeline: - Collection: books - Where: - - Expr.equal: + - Function.equal: - Field: title - Constant: "The Hitchhiker's Guide to the Galaxy" - Select: - - AliasedExpr: - - Expr.concat: + - AliasedExpression: + - Function.concat: - Field: author - Constant: ": " - Field: title - "author_title" - - AliasedExpr: - - Expr.concat: + - AliasedExpression: + - Function.concat: - Field: tags - - Constant: "new_tag" - "concatenatedTags" @@ -162,20 +162,20 @@ tests: pipeline: - Collection: books - Where: - - Expr.equal: + - Function.equal: - Field: title - Constant: "The Hitchhiker's Guide to the Galaxy" - Select: - - AliasedExpr: - - Expr.length: + - AliasedExpression: + - Function.length: - Field: title - "titleLength" - - AliasedExpr: - - Expr.length: + - AliasedExpression: + - Function.length: - Field: tags - "tagsLength" - - AliasedExpr: - - Expr.length: + - AliasedExpression: + - Function.length: - Field: awards - "awardsLength" assert_results: @@ -186,13 +186,13 @@ tests: pipeline: - Collection: books - Select: - - AliasedExpr: - - Expr.char_length: + - AliasedExpression: + - Function.char_length: - Field: title - "titleLength" - title - Where: - - Expr.greater_than: + - Function.greater_than: - Field: titleLength - Constant: 20 - Sort: @@ -244,12 +244,12 @@ tests: pipeline: - Collection: books - Where: - - Expr.equal: + - Function.equal: - Field: author - Constant: "Douglas Adams" - Select: - - AliasedExpr: - - Expr.char_length: + - AliasedExpression: + - Function.char_length: - Field: title - "title_length" assert_results: @@ -280,13 +280,13 @@ tests: pipeline: - Collection: books - Where: - - Expr.equal: + - Function.equal: - Field: author - Constant: Douglas Adams - Select: - - AliasedExpr: - - Expr.byte_length: - - Expr.string_concat: + - AliasedExpression: + - Function.byte_length: + - Function.string_concat: - Field: title - Constant: _银河系漫游指南 - "title_byte_length" @@ -322,7 +322,7 @@ tests: pipeline: - Collection: books - Where: - - Expr.like: + - Function.like: - Field: title - Constant: "%Guide%" - Select: @@ -334,7 +334,7 @@ tests: pipeline: - Collection: books - Where: - - Expr.regex_contains: + - Function.regex_contains: - Field: title - Constant: "(?i)(the|of)" assert_count: 5 @@ -356,7 +356,7 @@ tests: pipeline: - Collection: books - Where: - - Expr.regex_match: + - Function.regex_match: - Field: title - Constant: ".*(?i)(the|of).*" assert_count: 5 @@ -377,7 +377,7 @@ tests: pipeline: - Collection: books - Where: - - Expr.string_contains: + - Function.string_contains: - Field: title - Constant: "Hitchhiker's" - Select: @@ -388,12 +388,12 @@ tests: pipeline: - Collection: books - Where: - - Expr.equal: + - Function.equal: - Field: author - Constant: "Douglas Adams" - Select: - - AliasedExpr: - - Expr.to_lower: + - AliasedExpression: + - Function.to_lower: - Field: title - "lower_title" assert_results: @@ -424,12 +424,12 @@ tests: pipeline: - Collection: books - Where: - - Expr.equal: + - Function.equal: - Field: author - Constant: "Douglas Adams" - Select: - - AliasedExpr: - - Expr.to_upper: + - AliasedExpression: + - Function.to_upper: - Field: title - "upper_title" assert_results: @@ -460,13 +460,13 @@ tests: pipeline: - Collection: books - Where: - - Expr.equal: + - Function.equal: - Field: author - Constant: "Douglas Adams" - Select: - - AliasedExpr: - - Expr.trim: - - Expr.string_concat: + - AliasedExpression: + - Function.trim: + - Function.string_concat: - Constant: " " - Field: title - Constant: " " @@ -504,12 +504,12 @@ tests: pipeline: - Collection: books - Where: - - Expr.equal: + - Function.equal: - Field: author - Constant: "Jane Austen" - Select: - - AliasedExpr: - - Expr.string_reverse: + - AliasedExpression: + - Function.string_reverse: - Field: title - "reversed_title" assert_results: @@ -540,12 +540,12 @@ tests: pipeline: - Collection: books - Where: - - Expr.equal: + - Function.equal: - Field: author - Constant: "Douglas Adams" - Select: - - AliasedExpr: - - Expr.substring: + - AliasedExpression: + - Function.substring: - Field: title - Constant: 4 - Constant: 11 @@ -580,12 +580,12 @@ tests: pipeline: - Collection: books - Where: - - Expr.equal: + - Function.equal: - Field: author - Constant: "Fyodor Dostoevsky" - Select: - - AliasedExpr: - - Expr.substring: + - AliasedExpression: + - Function.substring: - Field: title - Constant: 10 - "substring_title" @@ -618,12 +618,12 @@ tests: pipeline: - Collection: books - Where: - - Expr.equal: + - Function.equal: - Field: author - Constant: "Douglas Adams" - Select: - - AliasedExpr: - - Expr.join: + - AliasedExpression: + - Function.join: - Field: tags - Constant: ", " - "joined_tags" diff --git a/tests/system/pipeline_e2e/vector.yaml b/tests/system/pipeline_e2e/vector.yaml index 15fc9bcaa..85d265c2d 100644 --- a/tests/system/pipeline_e2e/vector.yaml +++ b/tests/system/pipeline_e2e/vector.yaml @@ -3,8 +3,8 @@ tests: pipeline: - Collection: vectors - Select: - - AliasedExpr: - - Expr.vector_length: + - AliasedExpression: + - Function.vector_length: - Field: embedding - "embedding_length" - Sort: @@ -117,12 +117,12 @@ tests: pipeline: - Collection: vectors - Where: - - Expr.equal: + - Function.equal: - Field: embedding - Vector: [1.0, 2.0, 3.0] - Select: - - AliasedExpr: - - Expr.dot_product: + - AliasedExpression: + - Function.dot_product: - Field: embedding - Vector: [1.0, 1.0, 1.0] - "dot_product_result" @@ -132,12 +132,12 @@ tests: pipeline: - Collection: vectors - Where: - - Expr.equal: + - Function.equal: - Field: embedding - Vector: [1.0, 2.0, 3.0] - Select: - - AliasedExpr: - - Expr.euclidean_distance: + - AliasedExpression: + - Function.euclidean_distance: - Field: embedding - Vector: [1.0, 2.0, 3.0] - "euclidean_distance_result" @@ -147,12 +147,12 @@ tests: pipeline: - Collection: vectors - Where: - - Expr.equal: + - Function.equal: - Field: embedding - Vector: [1.0, 2.0, 3.0] - Select: - - AliasedExpr: - - Expr.cosine_distance: + - AliasedExpression: + - Function.cosine_distance: - Field: embedding - Vector: [1.0, 2.0, 3.0] - "cosine_distance_result" diff --git a/tests/unit/v1/test_aggregation.py b/tests/unit/v1/test_aggregation.py index 66239f9ea..59c8eccea 100644 --- a/tests/unit/v1/test_aggregation.py +++ b/tests/unit/v1/test_aggregation.py @@ -127,12 +127,12 @@ def test_avg_aggregation_no_alias_to_pb(): "in_alias,expected_alias", [("total", "total"), (None, "field_1")] ) def test_count_aggregation_to_pipeline_expr(in_alias, expected_alias): - from google.cloud.firestore_v1.pipeline_expressions import AliasedAggregate + from google.cloud.firestore_v1.pipeline_expressions import AliasedExpression from google.cloud.firestore_v1.pipeline_expressions import Count count_aggregation = CountAggregation(alias=in_alias) got = count_aggregation._to_pipeline_expr(iter([1])) - assert isinstance(got, AliasedAggregate) + assert isinstance(got, AliasedExpression) assert got.alias == expected_alias assert isinstance(got.expr, Count) assert len(got.expr.params) == 0 @@ -143,11 +143,11 @@ def test_count_aggregation_to_pipeline_expr(in_alias, expected_alias): [("total", "path", "total"), (None, "some_ref", "field_1")], ) def test_sum_aggregation_to_pipeline_expr(in_alias, expected_path, expected_alias): - from google.cloud.firestore_v1.pipeline_expressions import AliasedAggregate + from google.cloud.firestore_v1.pipeline_expressions import AliasedExpression count_aggregation = SumAggregation(expected_path, alias=in_alias) got = count_aggregation._to_pipeline_expr(iter([1])) - assert isinstance(got, AliasedAggregate) + assert isinstance(got, AliasedExpression) assert got.alias == expected_alias assert got.expr.name == "sum" assert got.expr.params[0].path == expected_path @@ -158,11 +158,11 @@ def test_sum_aggregation_to_pipeline_expr(in_alias, expected_path, expected_alia [("total", "path", "total"), (None, "some_ref", "field_1")], ) def test_avg_aggregation_to_pipeline_expr(in_alias, expected_path, expected_alias): - from google.cloud.firestore_v1.pipeline_expressions import AliasedAggregate + from google.cloud.firestore_v1.pipeline_expressions import AliasedExpression count_aggregation = AvgAggregation(expected_path, alias=in_alias) got = count_aggregation._to_pipeline_expr(iter([1])) - assert isinstance(got, AliasedAggregate) + assert isinstance(got, AliasedExpression) assert got.alias == expected_alias assert got.expr.name == "average" assert got.expr.params[0].path == expected_path diff --git a/tests/unit/v1/test_base_query.py b/tests/unit/v1/test_base_query.py index 7efa0dacf..2338fab71 100644 --- a/tests/unit/v1/test_base_query.py +++ b/tests/unit/v1/test_base_query.py @@ -2040,7 +2040,7 @@ def test__query_pipeline_composite_filter(): client = make_client() in_filter = FieldFilter("field_a", "==", "value_a") query = client.collection("my_col").where(filter=in_filter) - with mock.patch.object(expr.BooleanExpr, "_from_query_filter_pb") as convert_mock: + with mock.patch.object(expr.BooleanExpression, "_from_query_filter_pb") as convert_mock: pipeline = query.pipeline() convert_mock.assert_called_once_with(in_filter._to_pb(), client) assert len(pipeline.stages) == 2 From adbfd215bae595a6655295ee9164415457099bda Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 28 Oct 2025 16:20:03 -0700 Subject: [PATCH 31/36] renamed GenericStage to RawStage --- google/cloud/firestore_v1/_pipeline_stages.py | 2 +- google/cloud/firestore_v1/base_pipeline.py | 17 +++++------ tests/system/pipeline_e2e/general.yaml | 8 +++--- tests/unit/v1/test_async_pipeline.py | 28 +++++++++---------- tests/unit/v1/test_pipeline.py | 28 +++++++++---------- tests/unit/v1/test_pipeline_stages.py | 10 +++---- 6 files changed, 47 insertions(+), 46 deletions(-) diff --git a/google/cloud/firestore_v1/_pipeline_stages.py b/google/cloud/firestore_v1/_pipeline_stages.py index 4a271ad02..8fd0c812a 100644 --- a/google/cloud/firestore_v1/_pipeline_stages.py +++ b/google/cloud/firestore_v1/_pipeline_stages.py @@ -298,7 +298,7 @@ def _pb_options(self) -> dict[str, Value]: return options -class GenericStage(Stage): +class RawStage(Stage): """Represents a generic, named stage with parameters.""" def __init__( diff --git a/google/cloud/firestore_v1/base_pipeline.py b/google/cloud/firestore_v1/base_pipeline.py index 608a3b74b..15eee8d57 100644 --- a/google/cloud/firestore_v1/base_pipeline.py +++ b/google/cloud/firestore_v1/base_pipeline.py @@ -458,28 +458,29 @@ def unnest( """ return self._append(stages.Unnest(field, alias, options)) - def generic_stage(self, name: str, *params: Expression) -> "_BasePipeline": + def raw_stage(self, name: str, *params: Expression) -> "_BasePipeline": """ - Adds a generic, named stage to the pipeline with specified parameters. + Adds a stage to the pipeline by specifying the stage name as an argument. This does not offer any + type safety on the stage params and requires the caller to know the order (and optionally names) + of parameters accepted by the stage. - This method provides a flexible way to extend the pipeline's functionality - by adding custom stages. Each generic stage is defined by a unique `name` - and a set of `params` that control its behavior. + This class provides a way to call stages that are supported by the Firestore backend but that + are not implemented in the SDK version being used. Example: >>> # Assume we don't have a built-in "where" stage >>> pipeline = client.pipeline().collection("books") - >>> pipeline = pipeline.generic_stage("where", [Field.of("published").lt(900)]) + >>> pipeline = pipeline.raw_stage("where", Field.of("published").lt(900)) >>> pipeline = pipeline.select("title", "author") Args: - name: The name of the generic stage. + name: The name of the stage. *params: A sequence of `Expression` objects representing the parameters for the stage. Returns: A new Pipeline object with this stage appended to the stage list """ - return self._append(stages.GenericStage(name, *params)) + return self._append(stages.RawStage(name, *params)) def offset(self, offset: int) -> "_BasePipeline": """ diff --git a/tests/system/pipeline_e2e/general.yaml b/tests/system/pipeline_e2e/general.yaml index 90b77a9a5..23e98cf3d 100644 --- a/tests/system/pipeline_e2e/general.yaml +++ b/tests/system/pipeline_e2e/general.yaml @@ -528,18 +528,18 @@ tests: - booleanValue: true - booleanValue: false name: conditional - - description: testGenericStage + - description: testRawStage pipeline: - - GenericStage: + - RawStage: - "collection" - Value: reference_value: "/books" - - GenericStage: + - RawStage: - "where" - Function.equal: - Field: title - Constant: The Hitchhiker's Guide to the Galaxy - - GenericStage: + - RawStage: - "select" - Value: map_value: diff --git a/tests/unit/v1/test_async_pipeline.py b/tests/unit/v1/test_async_pipeline.py index a11a2951b..402287b02 100644 --- a/tests/unit/v1/test_async_pipeline.py +++ b/tests/unit/v1/test_async_pipeline.py @@ -67,33 +67,33 @@ def test_async_pipeline_repr_single_stage(): def test_async_pipeline_repr_multiple_stage(): stage_1 = stages.Collection("path") - stage_2 = stages.GenericStage("second", 2) - stage_3 = stages.GenericStage("third", 3) + stage_2 = stages.RawStage("second", 2) + stage_3 = stages.RawStage("third", 3) ppl = _make_async_pipeline(stage_1, stage_2, stage_3) repr_str = repr(ppl) assert repr_str == ( "AsyncPipeline(\n" " Collection(path='/path'),\n" - " GenericStage(name='second'),\n" - " GenericStage(name='third')\n" + " RawStage(name='second'),\n" + " RawStage(name='third')\n" ")" ) def test_async_pipeline_repr_long(): num_stages = 100 - stage_list = [stages.GenericStage("custom", i) for i in range(num_stages)] + stage_list = [stages.RawStage("custom", i) for i in range(num_stages)] ppl = _make_async_pipeline(*stage_list) repr_str = repr(ppl) - assert repr_str.count("GenericStage") == num_stages + assert repr_str.count("RawStage") == num_stages assert repr_str.count("\n") == num_stages + 1 def test_async_pipeline__to_pb(): from google.cloud.firestore_v1.types.pipeline import StructuredPipeline - stage_1 = stages.GenericStage("first") - stage_2 = stages.GenericStage("second") + stage_1 = stages.RawStage("first") + stage_2 = stages.RawStage("second") ppl = _make_async_pipeline(stage_1, stage_2) pb = ppl._to_pb() assert isinstance(pb, StructuredPipeline) @@ -103,9 +103,9 @@ def test_async_pipeline__to_pb(): def test_async_pipeline_append(): """append should create a new pipeline with the additional stage""" - stage_1 = stages.GenericStage("first") + stage_1 = stages.RawStage("first") ppl_1 = _make_async_pipeline(stage_1, client=object()) - stage_2 = stages.GenericStage("second") + stage_2 = stages.RawStage("second") ppl_2 = ppl_1._append(stage_2) assert ppl_1 != ppl_2 assert len(ppl_1.stages) == 1 @@ -130,7 +130,7 @@ async def test_async_pipeline_stream_empty(): mock_rpc = mock.AsyncMock() client._firestore_api.execute_pipeline = mock_rpc mock_rpc.return_value = _async_it([ExecutePipelineResponse()]) - ppl_1 = _make_async_pipeline(stages.GenericStage("s"), client=client) + ppl_1 = _make_async_pipeline(stages.RawStage("s"), client=client) results = [r async for r in ppl_1.stream()] assert results == [] @@ -159,7 +159,7 @@ async def test_async_pipeline_stream_no_doc_ref(): mock_rpc.return_value = _async_it( [ExecutePipelineResponse(results=[Document()], execution_time={"seconds": 9})] ) - ppl_1 = _make_async_pipeline(stages.GenericStage("s"), client=client) + ppl_1 = _make_async_pipeline(stages.RawStage("s"), client=client) results = [r async for r in ppl_1.stream()] assert len(results) == 1 @@ -401,8 +401,8 @@ async def test_async_pipeline_stream_stream_equivalence_mocked(): ("unnest", ("field_name", "alias"), stages.Unnest), ("unnest", (Field.of("n"), Field.of("alias")), stages.Unnest), ("unnest", ("n", "a", stages.UnnestOptions("idx")), stages.Unnest), - ("generic_stage", ("stage_name",), stages.GenericStage), - ("generic_stage", ("stage_name", Field.of("n")), stages.GenericStage), + ("raw_stage", ("stage_name",), stages.RawStage), + ("raw_stage", ("stage_name", Field.of("n")), stages.RawStage), ("offset", (1,), stages.Offset), ("limit", (1,), stages.Limit), ("aggregate", (Field.of("n").as_("alias"),), stages.Aggregate), diff --git a/tests/unit/v1/test_pipeline.py b/tests/unit/v1/test_pipeline.py index 161eef1cc..fcda4ac7c 100644 --- a/tests/unit/v1/test_pipeline.py +++ b/tests/unit/v1/test_pipeline.py @@ -62,33 +62,33 @@ def test_pipeline_repr_single_stage(): def test_pipeline_repr_multiple_stage(): stage_1 = stages.Collection("path") - stage_2 = stages.GenericStage("second", 2) - stage_3 = stages.GenericStage("third", 3) + stage_2 = stages.RawStage("second", 2) + stage_3 = stages.RawStage("third", 3) ppl = _make_pipeline(stage_1, stage_2, stage_3) repr_str = repr(ppl) assert repr_str == ( "Pipeline(\n" " Collection(path='/path'),\n" - " GenericStage(name='second'),\n" - " GenericStage(name='third')\n" + " RawStage(name='second'),\n" + " RawStage(name='third')\n" ")" ) def test_pipeline_repr_long(): num_stages = 100 - stage_list = [stages.GenericStage("custom", i) for i in range(num_stages)] + stage_list = [stages.RawStage("custom", i) for i in range(num_stages)] ppl = _make_pipeline(*stage_list) repr_str = repr(ppl) - assert repr_str.count("GenericStage") == num_stages + assert repr_str.count("RawStage") == num_stages assert repr_str.count("\n") == num_stages + 1 def test_pipeline__to_pb(): from google.cloud.firestore_v1.types.pipeline import StructuredPipeline - stage_1 = stages.GenericStage("first") - stage_2 = stages.GenericStage("second") + stage_1 = stages.RawStage("first") + stage_2 = stages.RawStage("second") ppl = _make_pipeline(stage_1, stage_2) pb = ppl._to_pb() assert isinstance(pb, StructuredPipeline) @@ -99,9 +99,9 @@ def test_pipeline__to_pb(): def test_pipeline_append(): """append should create a new pipeline with the additional stage""" - stage_1 = stages.GenericStage("first") + stage_1 = stages.RawStage("first") ppl_1 = _make_pipeline(stage_1, client=object()) - stage_2 = stages.GenericStage("second") + stage_2 = stages.RawStage("second") ppl_2 = ppl_1._append(stage_2) assert ppl_1 != ppl_2 assert len(ppl_1.stages) == 1 @@ -124,7 +124,7 @@ def test_pipeline_stream_empty(): client._database = "B" mock_rpc = client._firestore_api.execute_pipeline mock_rpc.return_value = [ExecutePipelineResponse()] - ppl_1 = _make_pipeline(stages.GenericStage("s"), client=client) + ppl_1 = _make_pipeline(stages.RawStage("s"), client=client) results = list(ppl_1.stream()) assert results == [] @@ -151,7 +151,7 @@ def test_pipeline_stream_no_doc_ref(): mock_rpc.return_value = [ ExecutePipelineResponse(results=[Document()], execution_time={"seconds": 9}) ] - ppl_1 = _make_pipeline(stages.GenericStage("s"), client=client) + ppl_1 = _make_pipeline(stages.RawStage("s"), client=client) results = list(ppl_1.stream()) assert len(results) == 1 @@ -378,8 +378,8 @@ def test_pipeline_execute_stream_equivalence_mocked(): ("unnest", ("field_name", "alias"), stages.Unnest), ("unnest", (Field.of("n"), Field.of("alias")), stages.Unnest), ("unnest", ("n", "a", stages.UnnestOptions("idx")), stages.Unnest), - ("generic_stage", ("stage_name",), stages.GenericStage), - ("generic_stage", ("stage_name", Field.of("n")), stages.GenericStage), + ("raw_stage", ("stage_name",), stages.RawStage), + ("raw_stage", ("stage_name", Field.of("n")), stages.RawStage), ("offset", (1,), stages.Offset), ("limit", (1,), stages.Limit), ("aggregate", (Field.of("n").as_("alias"),), stages.Aggregate), diff --git a/tests/unit/v1/test_pipeline_stages.py b/tests/unit/v1/test_pipeline_stages.py index 1d2ff8760..367b9ce26 100644 --- a/tests/unit/v1/test_pipeline_stages.py +++ b/tests/unit/v1/test_pipeline_stages.py @@ -420,9 +420,9 @@ def test_to_pb_no_options(self): assert len(result.args) == 3 -class TestGenericStage: +class TestRawStage: def _make_one(self, *args, **kwargs): - return stages.GenericStage(*args, **kwargs) + return stages.RawStage(*args, **kwargs) @pytest.mark.parametrize( "input_args,expected_params", @@ -471,7 +471,7 @@ def test_ctor_with_options(self): standard_unnest = stages.Unnest( field, alias, options=stages.UnnestOptions(**options) ) - generic_unnest = stages.GenericStage("unnest", field, alias, options=options) + generic_unnest = stages.RawStage("unnest", field, alias, options=options) assert standard_unnest._pb_args() == generic_unnest._pb_args() assert standard_unnest._pb_options() == generic_unnest._pb_options() assert standard_unnest._to_pb() == generic_unnest._to_pb() @@ -479,8 +479,8 @@ def test_ctor_with_options(self): @pytest.mark.parametrize( "input_args,expected", [ - (("name",), "GenericStage(name='name')"), - (("custom", Value(string_value="val")), "GenericStage(name='custom')"), + (("name",), "RawStage(name='name')"), + (("custom", Value(string_value="val")), "RawStage(name='custom')"), ], ) def test_repr(self, input_args, expected): From 5d4d6b0e86043f28d6e3a98c1f855d75876d2015 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 28 Oct 2025 16:32:12 -0700 Subject: [PATCH 32/36] ran blacken --- google/cloud/firestore_v1/_pipeline_stages.py | 8 +++- .../firestore_v1/pipeline_expressions.py | 44 ++++++++++++++----- tests/unit/v1/test_base_query.py | 4 +- tests/unit/v1/test_pipeline_expressions.py | 4 +- 4 files changed, 45 insertions(+), 15 deletions(-) diff --git a/google/cloud/firestore_v1/_pipeline_stages.py b/google/cloud/firestore_v1/_pipeline_stages.py index 8fd0c812a..95ce32021 100644 --- a/google/cloud/firestore_v1/_pipeline_stages.py +++ b/google/cloud/firestore_v1/_pipeline_stages.py @@ -302,14 +302,18 @@ class RawStage(Stage): """Represents a generic, named stage with parameters.""" def __init__( - self, name: str, *params: Expression | Value, options: dict[str, Expression | Value] = {} + self, + name: str, + *params: Expression | Value, + options: dict[str, Expression | Value] = {}, ): super().__init__(name) self.params: list[Value] = [ p._to_pb() if isinstance(p, Expression) else p for p in params ] self.options: dict[str, Value] = { - k: v._to_pb() if isinstance(v, Expression) else v for k, v in options.items() + k: v._to_pb() if isinstance(v, Expression) else v + for k, v in options.items() } def _pb_args(self): diff --git a/google/cloud/firestore_v1/pipeline_expressions.py b/google/cloud/firestore_v1/pipeline_expressions.py index d9e4c81b7..a6e17055a 100644 --- a/google/cloud/firestore_v1/pipeline_expressions.py +++ b/google/cloud/firestore_v1/pipeline_expressions.py @@ -110,7 +110,9 @@ def _to_pb(self) -> Value: raise NotImplementedError @staticmethod - def _cast_to_expr_or_convert_to_constant(o: Any, include_vector=False) -> "Expression": + def _cast_to_expr_or_convert_to_constant( + o: Any, include_vector=False + ) -> "Expression": """Convert arbitrary object to an Expression.""" if isinstance(o, Constant) and isinstance(o.value, list): o = o.value @@ -146,7 +148,9 @@ def static_func(self, first_arg, *other_args, **kwargs): f"`expressions must be called on an Expression or a string representing a field name. got {type(first_arg)}." ) first_expr = ( - Field.of(first_arg) if not isinstance(first_arg, Expression) else first_arg + Field.of(first_arg) + if not isinstance(first_arg, Expression) + else first_arg ) return self.instance_func(first_expr, *other_args, **kwargs) @@ -509,7 +513,9 @@ def greater_than(self, other: Expression | CONSTANT_TYPE) -> "BooleanExpression" ) @expose_as_static - def greater_than_or_equal(self, other: Expression | CONSTANT_TYPE) -> "BooleanExpression": + def greater_than_or_equal( + self, other: Expression | CONSTANT_TYPE + ) -> "BooleanExpression": """Creates an expression that checks if this expression is greater than or equal to another expression or constant value. @@ -552,7 +558,9 @@ def less_than(self, other: Expression | CONSTANT_TYPE) -> "BooleanExpression": ) @expose_as_static - def less_than_or_equal(self, other: Expression | CONSTANT_TYPE) -> "BooleanExpression": + def less_than_or_equal( + self, other: Expression | CONSTANT_TYPE + ) -> "BooleanExpression": """Creates an expression that checks if this expression is less than or equal to another expression or constant value. @@ -624,7 +632,9 @@ def not_equal_any( ) @expose_as_static - def array_contains(self, element: Expression | CONSTANT_TYPE) -> "BooleanExpression": + def array_contains( + self, element: Expression | CONSTANT_TYPE + ) -> "BooleanExpression": """Creates an expression that checks if an array contains a specific element or value. Example: @@ -1298,7 +1308,10 @@ def map_remove(self, key: str | Constant[str]) -> "Expression": @expose_as_static def map_merge( - self, *other_maps: Map | dict[str | Constant[str], Expression | CONSTANT_TYPE] | Expression + self, + *other_maps: Map + | dict[str | Constant[str], Expression | CONSTANT_TYPE] + | Expression, ) -> "Expression": """Creates an expression that merges one or more dicts into a single map. @@ -1342,7 +1355,9 @@ def cosine_distance(self, other: Expression | list[float] | Vector) -> "Expressi ) @expose_as_static - def euclidean_distance(self, other: Expression | list[float] | Vector) -> "Expression": + def euclidean_distance( + self, other: Expression | list[float] | Vector + ) -> "Expression": """Calculates the Euclidean distance between two vectors. Example: @@ -1493,7 +1508,9 @@ def unix_seconds_to_timestamp(self) -> "Expression": return Function("unix_seconds_to_timestamp", [self]) @expose_as_static - def timestamp_add(self, unit: Expression | str, amount: Expression | float) -> "Expression": + def timestamp_add( + self, unit: Expression | str, amount: Expression | float + ) -> "Expression": """Creates an expression that adds a specified amount of time to this timestamp expression. Example: @@ -1520,7 +1537,9 @@ def timestamp_add(self, unit: Expression | str, amount: Expression | float) -> " ) @expose_as_static - def timestamp_subtract(self, unit: Expression | str, amount: Expression | float) -> "Expression": + def timestamp_subtract( + self, unit: Expression | str, amount: Expression | float + ) -> "Expression": """Creates an expression that subtracts a specified amount of time from this timestamp expression. Example: @@ -1796,7 +1815,8 @@ class BooleanExpression(Function): def _from_query_filter_pb(filter_pb, client): if isinstance(filter_pb, Query_pb.CompositeFilter): sub_filters = [ - BooleanExpression._from_query_filter_pb(f, client) for f in filter_pb.filters + BooleanExpression._from_query_filter_pb(f, client) + for f in filter_pb.filters ] if filter_pb.op == Query_pb.CompositeFilter.Operator.OR: return Or(*sub_filters) @@ -1986,7 +2006,9 @@ class Conditional(BooleanExpression): else_expr: The expression to return if the condition is false """ - def __init__(self, condition: BooleanExpression, then_expr: Expression, else_expr: Expression): + def __init__( + self, condition: BooleanExpression, then_expr: Expression, else_expr: Expression + ): super().__init__( "conditional", [condition, then_expr, else_expr], use_infix_repr=False ) diff --git a/tests/unit/v1/test_base_query.py b/tests/unit/v1/test_base_query.py index 2338fab71..7ecbd1966 100644 --- a/tests/unit/v1/test_base_query.py +++ b/tests/unit/v1/test_base_query.py @@ -2040,7 +2040,9 @@ def test__query_pipeline_composite_filter(): client = make_client() in_filter = FieldFilter("field_a", "==", "value_a") query = client.collection("my_col").where(filter=in_filter) - with mock.patch.object(expr.BooleanExpression, "_from_query_filter_pb") as convert_mock: + with mock.patch.object( + expr.BooleanExpression, "_from_query_filter_pb" + ) as convert_mock: pipeline = query.pipeline() convert_mock.assert_called_once_with(in_filter._to_pb(), client) assert len(pipeline.stages) == 2 diff --git a/tests/unit/v1/test_pipeline_expressions.py b/tests/unit/v1/test_pipeline_expressions.py index 00d636ca1..b8be41e15 100644 --- a/tests/unit/v1/test_pipeline_expressions.py +++ b/tests/unit/v1/test_pipeline_expressions.py @@ -917,7 +917,9 @@ def test_conditional(self): instance = expr.Conditional(arg1, arg2, arg3) assert instance.name == "conditional" assert instance.params == [arg1, arg2, arg3] - assert repr(instance) == "Conditional(Condition, ThenExpression, ElseExpression)" + assert ( + repr(instance) == "Conditional(Condition, ThenExpression, ElseExpression)" + ) def test_like(self): arg1 = self._make_arg("Expression") From ea3aaa53f1f9ccea426c216252b5f55275c9e169 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 28 Oct 2025 16:39:24 -0700 Subject: [PATCH 33/36] made pipeline_stages public --- google/cloud/firestore_v1/async_pipeline.py | 2 +- google/cloud/firestore_v1/base_pipeline.py | 2 +- google/cloud/firestore_v1/pipeline.py | 2 +- google/cloud/firestore_v1/pipeline_source.py | 2 +- .../{_pipeline_stages.py => pipeline_stages.py} | 0 tests/system/test_pipeline_acceptance.py | 2 +- tests/unit/v1/test_aggregation.py | 8 ++++---- tests/unit/v1/test_async_aggregation.py | 8 ++++---- tests/unit/v1/test_async_collection.py | 2 +- tests/unit/v1/test_async_pipeline.py | 2 +- tests/unit/v1/test_base_query.py | 2 +- tests/unit/v1/test_collection.py | 2 +- tests/unit/v1/test_pipeline.py | 2 +- tests/unit/v1/test_pipeline_source.py | 2 +- tests/unit/v1/test_pipeline_stages.py | 2 +- 15 files changed, 20 insertions(+), 20 deletions(-) rename google/cloud/firestore_v1/{_pipeline_stages.py => pipeline_stages.py} (100%) diff --git a/google/cloud/firestore_v1/async_pipeline.py b/google/cloud/firestore_v1/async_pipeline.py index 471c33093..9fe0c8756 100644 --- a/google/cloud/firestore_v1/async_pipeline.py +++ b/google/cloud/firestore_v1/async_pipeline.py @@ -14,7 +14,7 @@ from __future__ import annotations from typing import AsyncIterable, TYPE_CHECKING -from google.cloud.firestore_v1 import _pipeline_stages as stages +from google.cloud.firestore_v1 import pipeline_stages as stages from google.cloud.firestore_v1.base_pipeline import _BasePipeline if TYPE_CHECKING: # pragma: NO COVER diff --git a/google/cloud/firestore_v1/base_pipeline.py b/google/cloud/firestore_v1/base_pipeline.py index 15eee8d57..63fee19fa 100644 --- a/google/cloud/firestore_v1/base_pipeline.py +++ b/google/cloud/firestore_v1/base_pipeline.py @@ -14,7 +14,7 @@ from __future__ import annotations from typing import Iterable, Sequence, TYPE_CHECKING -from google.cloud.firestore_v1 import _pipeline_stages as stages +from google.cloud.firestore_v1 import pipeline_stages as stages from google.cloud.firestore_v1.types.pipeline import ( StructuredPipeline as StructuredPipeline_pb, ) diff --git a/google/cloud/firestore_v1/pipeline.py b/google/cloud/firestore_v1/pipeline.py index 9f568f925..f578e00b6 100644 --- a/google/cloud/firestore_v1/pipeline.py +++ b/google/cloud/firestore_v1/pipeline.py @@ -14,7 +14,7 @@ from __future__ import annotations from typing import Iterable, TYPE_CHECKING -from google.cloud.firestore_v1 import _pipeline_stages as stages +from google.cloud.firestore_v1 import pipeline_stages as stages from google.cloud.firestore_v1.base_pipeline import _BasePipeline if TYPE_CHECKING: # pragma: NO COVER diff --git a/google/cloud/firestore_v1/pipeline_source.py b/google/cloud/firestore_v1/pipeline_source.py index 6d83ae533..f4328afa4 100644 --- a/google/cloud/firestore_v1/pipeline_source.py +++ b/google/cloud/firestore_v1/pipeline_source.py @@ -14,7 +14,7 @@ from __future__ import annotations from typing import Generic, TypeVar, TYPE_CHECKING -from google.cloud.firestore_v1 import _pipeline_stages as stages +from google.cloud.firestore_v1 import pipeline_stages as stages from google.cloud.firestore_v1.base_pipeline import _BasePipeline from google.cloud.firestore_v1._helpers import DOCUMENT_PATH_DELIMITER diff --git a/google/cloud/firestore_v1/_pipeline_stages.py b/google/cloud/firestore_v1/pipeline_stages.py similarity index 100% rename from google/cloud/firestore_v1/_pipeline_stages.py rename to google/cloud/firestore_v1/pipeline_stages.py diff --git a/tests/system/test_pipeline_acceptance.py b/tests/system/test_pipeline_acceptance.py index 3083b07a5..232d350e5 100644 --- a/tests/system/test_pipeline_acceptance.py +++ b/tests/system/test_pipeline_acceptance.py @@ -25,7 +25,7 @@ from google.protobuf.json_format import MessageToDict -from google.cloud.firestore_v1 import _pipeline_stages as stages +from google.cloud.firestore_v1 import pipeline_stages as stages from google.cloud.firestore_v1 import pipeline_expressions from google.cloud.firestore_v1.vector import Vector from google.cloud.firestore_v1 import pipeline_expressions as expr diff --git a/tests/unit/v1/test_aggregation.py b/tests/unit/v1/test_aggregation.py index 59c8eccea..299283564 100644 --- a/tests/unit/v1/test_aggregation.py +++ b/tests/unit/v1/test_aggregation.py @@ -1033,7 +1033,7 @@ def test_aggregation_from_query(): ) def test_aggreation_to_pipeline_sum(field, in_alias, out_alias): from google.cloud.firestore_v1.pipeline import Pipeline - from google.cloud.firestore_v1._pipeline_stages import Collection, Aggregate + from google.cloud.firestore_v1.pipeline_stages import Collection, Aggregate client = make_client() parent = client.collection("dee") @@ -1064,7 +1064,7 @@ def test_aggreation_to_pipeline_sum(field, in_alias, out_alias): ) def test_aggreation_to_pipeline_avg(field, in_alias, out_alias): from google.cloud.firestore_v1.pipeline import Pipeline - from google.cloud.firestore_v1._pipeline_stages import Collection, Aggregate + from google.cloud.firestore_v1.pipeline_stages import Collection, Aggregate client = make_client() parent = client.collection("dee") @@ -1094,7 +1094,7 @@ def test_aggreation_to_pipeline_avg(field, in_alias, out_alias): ) def test_aggreation_to_pipeline_count(in_alias, out_alias): from google.cloud.firestore_v1.pipeline import Pipeline - from google.cloud.firestore_v1._pipeline_stages import Collection, Aggregate + from google.cloud.firestore_v1.pipeline_stages import Collection, Aggregate from google.cloud.firestore_v1.pipeline_expressions import Count client = make_client() @@ -1137,7 +1137,7 @@ def test_aggreation_to_pipeline_count_increment(): def test_aggreation_to_pipeline_complex(): from google.cloud.firestore_v1.pipeline import Pipeline - from google.cloud.firestore_v1._pipeline_stages import Collection, Aggregate, Select + from google.cloud.firestore_v1.pipeline_stages import Collection, Aggregate, Select client = make_client() query = client.collection("my_col").select(["field_a", "field_b.c"]) diff --git a/tests/unit/v1/test_async_aggregation.py b/tests/unit/v1/test_async_aggregation.py index f51db482d..eca2ecef1 100644 --- a/tests/unit/v1/test_async_aggregation.py +++ b/tests/unit/v1/test_async_aggregation.py @@ -709,7 +709,7 @@ async def test_aggregation_query_stream_w_explain_options_analyze_false(): ) def test_async_aggreation_to_pipeline_sum(field, in_alias, out_alias): from google.cloud.firestore_v1.async_pipeline import AsyncPipeline - from google.cloud.firestore_v1._pipeline_stages import Collection, Aggregate + from google.cloud.firestore_v1.pipeline_stages import Collection, Aggregate client = make_async_client() parent = client.collection("dee") @@ -740,7 +740,7 @@ def test_async_aggreation_to_pipeline_sum(field, in_alias, out_alias): ) def test_async_aggreation_to_pipeline_avg(field, in_alias, out_alias): from google.cloud.firestore_v1.async_pipeline import AsyncPipeline - from google.cloud.firestore_v1._pipeline_stages import Collection, Aggregate + from google.cloud.firestore_v1.pipeline_stages import Collection, Aggregate client = make_async_client() parent = client.collection("dee") @@ -770,7 +770,7 @@ def test_async_aggreation_to_pipeline_avg(field, in_alias, out_alias): ) def test_async_aggreation_to_pipeline_count(in_alias, out_alias): from google.cloud.firestore_v1.async_pipeline import AsyncPipeline - from google.cloud.firestore_v1._pipeline_stages import Collection, Aggregate + from google.cloud.firestore_v1.pipeline_stages import Collection, Aggregate from google.cloud.firestore_v1.pipeline_expressions import Count client = make_async_client() @@ -813,7 +813,7 @@ def test_aggreation_to_pipeline_count_increment(): def test_async_aggreation_to_pipeline_complex(): from google.cloud.firestore_v1.async_pipeline import AsyncPipeline - from google.cloud.firestore_v1._pipeline_stages import Collection, Aggregate, Select + from google.cloud.firestore_v1.pipeline_stages import Collection, Aggregate, Select client = make_async_client() query = client.collection("my_col").select(["field_a", "field_b.c"]) diff --git a/tests/unit/v1/test_async_collection.py b/tests/unit/v1/test_async_collection.py index 353997b8e..5b4df059a 100644 --- a/tests/unit/v1/test_async_collection.py +++ b/tests/unit/v1/test_async_collection.py @@ -605,7 +605,7 @@ def test_asynccollectionreference_recursive(): def test_asynccollectionreference_pipeline(): from google.cloud.firestore_v1.async_pipeline import AsyncPipeline - from google.cloud.firestore_v1._pipeline_stages import Collection + from google.cloud.firestore_v1.pipeline_stages import Collection client = make_async_client() collection = _make_async_collection_reference("collection", client=client) diff --git a/tests/unit/v1/test_async_pipeline.py b/tests/unit/v1/test_async_pipeline.py index 402287b02..2fc39a906 100644 --- a/tests/unit/v1/test_async_pipeline.py +++ b/tests/unit/v1/test_async_pipeline.py @@ -15,7 +15,7 @@ import mock import pytest -from google.cloud.firestore_v1 import _pipeline_stages as stages +from google.cloud.firestore_v1 import pipeline_stages as stages from google.cloud.firestore_v1.pipeline_expressions import Field diff --git a/tests/unit/v1/test_base_query.py b/tests/unit/v1/test_base_query.py index 7ecbd1966..925010070 100644 --- a/tests/unit/v1/test_base_query.py +++ b/tests/unit/v1/test_base_query.py @@ -18,7 +18,7 @@ import pytest from tests.unit.v1._test_helpers import make_client -from google.cloud.firestore_v1 import _pipeline_stages as stages +from google.cloud.firestore_v1 import pipeline_stages as stages def _make_base_query(*args, **kwargs): diff --git a/tests/unit/v1/test_collection.py b/tests/unit/v1/test_collection.py index 9e615541a..76418204b 100644 --- a/tests/unit/v1/test_collection.py +++ b/tests/unit/v1/test_collection.py @@ -514,7 +514,7 @@ def test_stream_w_read_time(query_class): def test_collectionreference_pipeline(): from tests.unit.v1 import _test_helpers from google.cloud.firestore_v1.pipeline import Pipeline - from google.cloud.firestore_v1._pipeline_stages import Collection + from google.cloud.firestore_v1.pipeline_stages import Collection client = _test_helpers.make_client() collection = _make_collection_reference("collection", client=client) diff --git a/tests/unit/v1/test_pipeline.py b/tests/unit/v1/test_pipeline.py index fcda4ac7c..b6d353f1a 100644 --- a/tests/unit/v1/test_pipeline.py +++ b/tests/unit/v1/test_pipeline.py @@ -15,7 +15,7 @@ import mock import pytest -from google.cloud.firestore_v1 import _pipeline_stages as stages +from google.cloud.firestore_v1 import pipeline_stages as stages from google.cloud.firestore_v1.pipeline_expressions import Field diff --git a/tests/unit/v1/test_pipeline_source.py b/tests/unit/v1/test_pipeline_source.py index bed1bd05a..e29b763e2 100644 --- a/tests/unit/v1/test_pipeline_source.py +++ b/tests/unit/v1/test_pipeline_source.py @@ -17,7 +17,7 @@ from google.cloud.firestore_v1.async_pipeline import AsyncPipeline from google.cloud.firestore_v1.client import Client from google.cloud.firestore_v1.async_client import AsyncClient -from google.cloud.firestore_v1 import _pipeline_stages as stages +from google.cloud.firestore_v1 import pipeline_stages as stages from google.cloud.firestore_v1.base_document import BaseDocumentReference diff --git a/tests/unit/v1/test_pipeline_stages.py b/tests/unit/v1/test_pipeline_stages.py index 367b9ce26..18c9d6790 100644 --- a/tests/unit/v1/test_pipeline_stages.py +++ b/tests/unit/v1/test_pipeline_stages.py @@ -16,7 +16,7 @@ from unittest import mock from google.cloud.firestore_v1.base_pipeline import _BasePipeline -import google.cloud.firestore_v1._pipeline_stages as stages +import google.cloud.firestore_v1.pipeline_stages as stages from google.cloud.firestore_v1.pipeline_expressions import ( Constant, Field, From 7059435cfd3508f3849ea87b96f5392530b2c79a Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 29 Oct 2025 14:30:42 -0700 Subject: [PATCH 34/36] skip pipeline verification on kokoro --- tests/system/test_pipeline_acceptance.py | 6 ++++++ tests/system/test_system.py | 7 +++++++ tests/system/test_system_async.py | 6 ++++++ 3 files changed, 19 insertions(+) diff --git a/tests/system/test_pipeline_acceptance.py b/tests/system/test_pipeline_acceptance.py index 3083b07a5..aa0886f43 100644 --- a/tests/system/test_pipeline_acceptance.py +++ b/tests/system/test_pipeline_acceptance.py @@ -37,6 +37,12 @@ FIRESTORE_PROJECT = os.environ.get("GCLOUD_PROJECT") +# TODO: enable kokoro tests when internal test project is whitelisted +pytestmark = pytest.mark.skipif( + condition=os.getenv("KOKORO_JOB_NAME") is not None, + reason="Pipeline tests are currently not supported by kokoro" +) + test_dir_name = os.path.dirname(__file__) id_format = ( diff --git a/tests/system/test_system.py b/tests/system/test_system.py index c2bd93ef8..c92cd7008 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -90,8 +90,15 @@ def verify_pipeline(query): It can be attached to existing query tests to check both modalities at the same time """ + import os + import warnings from google.cloud.firestore_v1.base_aggregation import BaseAggregationQuery + # return early on kokoro. Test project doesn't currently support pipelines + # TODO: enable pipeline verification when kokoro test project is whitelisted + if os.getenv("KOKORO_JOB_NAME"): + pytest.skip("skipping pipeline verification on kokoro") + def _clean_results(results): if isinstance(results, dict): return {k: _clean_results(v) for k, v in results.items()} diff --git a/tests/system/test_system_async.py b/tests/system/test_system_async.py index d053cbd7a..a0eae1f74 100644 --- a/tests/system/test_system_async.py +++ b/tests/system/test_system_async.py @@ -170,8 +170,14 @@ async def verify_pipeline(query): It can be attached to existing query tests to check both modalities at the same time """ + import os from google.cloud.firestore_v1.base_aggregation import BaseAggregationQuery + # return early on kokoro. Test project doesn't currently support pipelines + # TODO: enable pipeline verification when kokoro test project is whitelisted + if os.getenv("KOKORO_JOB_NAME"): + pytest.skip("skipping pipeline verification on kokoro") + def _clean_results(results): if isinstance(results, dict): return {k: _clean_results(v) for k, v in results.items()} From 52e11bf548bf4a6f18a6c8364000d08bb83a9ee1 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 29 Oct 2025 14:37:22 -0700 Subject: [PATCH 35/36] fixed lint --- tests/system/test_pipeline_acceptance.py | 2 +- tests/system/test_system.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/system/test_pipeline_acceptance.py b/tests/system/test_pipeline_acceptance.py index aa0886f43..81ee0a8a6 100644 --- a/tests/system/test_pipeline_acceptance.py +++ b/tests/system/test_pipeline_acceptance.py @@ -40,7 +40,7 @@ # TODO: enable kokoro tests when internal test project is whitelisted pytestmark = pytest.mark.skipif( condition=os.getenv("KOKORO_JOB_NAME") is not None, - reason="Pipeline tests are currently not supported by kokoro" + reason="Pipeline tests are currently not supported by kokoro", ) test_dir_name = os.path.dirname(__file__) diff --git a/tests/system/test_system.py b/tests/system/test_system.py index c92cd7008..99de91254 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -91,7 +91,6 @@ def verify_pipeline(query): modalities at the same time """ import os - import warnings from google.cloud.firestore_v1.base_aggregation import BaseAggregationQuery # return early on kokoro. Test project doesn't currently support pipelines From c994ad6d89121ec4ec5e42f79bd68d168e665b99 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Wed, 29 Oct 2025 16:06:31 -0700 Subject: [PATCH 36/36] fixed test coverage --- .../firestore_v1/pipeline_expressions.py | 4 +--- tests/unit/v1/test_pipeline_expressions.py | 24 +++++++++++++++++++ 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/google/cloud/firestore_v1/pipeline_expressions.py b/google/cloud/firestore_v1/pipeline_expressions.py index 302cee28e..30f3de995 100644 --- a/google/cloud/firestore_v1/pipeline_expressions.py +++ b/google/cloud/firestore_v1/pipeline_expressions.py @@ -112,8 +112,6 @@ def _to_pb(self) -> Value: @staticmethod def _cast_to_expr_or_convert_to_constant(o: Any, include_vector=False) -> "Expr": """Convert arbitrary object to an Expr.""" - if isinstance(o, Constant) and isinstance(o.value, list): - o = o.value if isinstance(o, Expr): return o if isinstance(o, dict): @@ -143,7 +141,7 @@ def __init__(self, instance_func): def static_func(self, first_arg, *other_args, **kwargs): if not isinstance(first_arg, (Expr, str)): raise TypeError( - f"`expressions must be called on an Expr or a string representing a field name. got {type(first_arg)}." + f"'{self.instance_func.__name__}' must be called on an Expression or a string representing a field. got {type(first_arg)}." ) first_expr = ( Field.of(first_arg) if not isinstance(first_arg, Expr) else first_arg diff --git a/tests/unit/v1/test_pipeline_expressions.py b/tests/unit/v1/test_pipeline_expressions.py index aec721e7d..522b51c84 100644 --- a/tests/unit/v1/test_pipeline_expressions.py +++ b/tests/unit/v1/test_pipeline_expressions.py @@ -645,6 +645,14 @@ def test_w_exprs(self): ) +class TestFunction: + def test_equals(self): + assert expr.Function.sqrt("1") == expr.Function.sqrt("1") + assert expr.Function.sqrt("1") != expr.Function.sqrt("2") + assert expr.Function.sqrt("1") != expr.Function.sum("1") + assert expr.Function.sqrt("1") != object() + + class TestExpressionMethods: """ contains test methods for each Expr method @@ -692,6 +700,22 @@ def __repr__(self): arg = MockExpr(name) return arg + def test_expression_wrong_first_type(self): + """The first argument should always be an expression or field name""" + expected_message = "must be called on an Expression or a string representing a field. got ." + with pytest.raises(TypeError) as e1: + Expr.logical_minimum(5, 1) + assert str(e1.value) == f"'logical_minimum' {expected_message}" + with pytest.raises(TypeError) as e2: + Expr.sqrt(9) + assert str(e2.value) == f"'sqrt' {expected_message}" + + def test_expression_w_string(self): + """should be able to use string for first argument. Should be interpreted as Field""" + instance = Expr.logical_minimum("first", "second") + assert isinstance(instance.params[0], Field) + assert instance.params[0].path == "first" + def test_and(self): arg1 = self._make_arg() arg2 = self._make_arg()