diff --git a/google/cloud/firestore_v1/_pipeline_stages.py b/google/cloud/firestore_v1/_pipeline_stages.py index c63b748ac..62503404e 100644 --- a/google/cloud/firestore_v1/_pipeline_stages.py +++ b/google/cloud/firestore_v1/_pipeline_stages.py @@ -112,11 +112,13 @@ class UnnestOptions: storing the original 0-based index of the element within the array. """ - def __init__(self, index_field: str): - self.index_field = index_field + def __init__(self, index_field: Field | str): + self.index_field = ( + index_field if isinstance(index_field, Field) else Field.of(index_field) + ) def __repr__(self): - return f"{self.__class__.__name__}(index_field={self.index_field!r})" + return f"{self.__class__.__name__}(index_field={self.index_field.path!r})" class Stage(ABC): @@ -258,13 +260,7 @@ def of(*documents: "BaseDocumentReference") -> "Documents": return Documents(*doc_paths) def _pb_args(self): - return [ - Value( - array_value={ - "values": [Value(string_value=path) for path in self.paths] - } - ) - ] + return [Value(reference_value=path) for path in self.paths] class FindNearest(Stage): @@ -306,15 +302,23 @@ def _pb_options(self) -> dict[str, Value]: class GenericStage(Stage): """Represents a generic, named stage with parameters.""" - def __init__(self, name: str, *params: Expr | Value): + def __init__( + self, name: str, *params: Expr | Value, options: dict[str, Expr | Value] = {} + ): super().__init__(name) self.params: list[Value] = [ p._to_pb() if isinstance(p, Expr) else p for p in params ] + self.options: dict[str, Value] = { + k: v._to_pb() if isinstance(v, Expr) else v for k, v in options.items() + } def _pb_args(self): return self.params + def _pb_options(self): + return self.options + def __repr__(self): return f"{self.__class__.__name__}(name='{self.name}')" @@ -437,7 +441,7 @@ def _pb_args(self): def _pb_options(self): options = {} if self.options is not None: - options["index_field"] = Value(string_value=self.options.index_field) + options["index_field"] = self.options.index_field._to_pb() return options diff --git a/google/cloud/firestore_v1/pipeline_expressions.py b/google/cloud/firestore_v1/pipeline_expressions.py index b113e2874..30f3de995 100644 --- a/google/cloud/firestore_v1/pipeline_expressions.py +++ b/google/cloud/firestore_v1/pipeline_expressions.py @@ -112,8 +112,6 @@ def _to_pb(self) -> Value: @staticmethod def _cast_to_expr_or_convert_to_constant(o: Any, include_vector=False) -> "Expr": """Convert arbitrary object to an Expr.""" - if isinstance(o, Constant) and isinstance(o.value, list): - o = o.value if isinstance(o, Expr): return o if isinstance(o, dict): @@ -143,7 +141,7 @@ def __init__(self, instance_func): def static_func(self, first_arg, *other_args, **kwargs): if not isinstance(first_arg, (Expr, str)): raise TypeError( - f"`expressions must be called on an Expr or a string representing a field name. got {type(first_arg)}." + f"'{self.instance_func.__name__}' must be called on an Expression or a string representing a field. got {type(first_arg)}." ) first_expr = ( Field.of(first_arg) if not isinstance(first_arg, Expr) else first_arg @@ -152,7 +150,7 @@ def static_func(self, first_arg, *other_args, **kwargs): def __get__(self, instance, owner): if instance is None: - return self.static_func.__get__(instance, owner) + return self.static_func else: return self.instance_func.__get__(instance, owner) @@ -1280,7 +1278,7 @@ def map_get(self, key: str | Constant[str]) -> "Expr": @expose_as_static def map_remove(self, key: str | Constant[str]) -> "Expr": - """Remove a key from the map produced by evaluating this expression. + """Remove a key from a the map produced by evaluating this expression. Example: >>> Map({"city": "London"}).map_remove("city") diff --git a/pytest.ini b/pytest.ini index eac8ea123..308d1b494 100644 --- a/pytest.ini +++ b/pytest.ini @@ -22,4 +22,5 @@ filterwarnings = ignore:.*The \`credentials_file\` argument is deprecated.*:DeprecationWarning # Remove after updating test dependencies that use asyncio.iscoroutinefunction ignore:.*\'asyncio.iscoroutinefunction\' is deprecated.*:DeprecationWarning - ignore:.*\'asyncio.get_event_loop_policy\' is deprecated.*:DeprecationWarning \ No newline at end of file + ignore:.*\'asyncio.get_event_loop_policy\' is deprecated.*:DeprecationWarning + ignore:.*Please upgrade to the latest Python version.*:FutureWarning diff --git a/tests/system/pipeline_e2e.yaml b/tests/system/pipeline_e2e.yaml deleted file mode 100644 index 38595224a..000000000 --- a/tests/system/pipeline_e2e.yaml +++ /dev/null @@ -1,3303 +0,0 @@ -data: - books: - book1: - title: "The Hitchhiker's Guide to the Galaxy" - author: "Douglas Adams" - genre: "Science Fiction" - published: 1979 - rating: 4.2 - tags: - - comedy - - space - - adventure - awards: - hugo: true - nebula: false - book2: - title: "Pride and Prejudice" - author: "Jane Austen" - genre: "Romance" - published: 1813 - rating: 4.5 - tags: - - classic - - social commentary - - love - awards: - none: true - book3: - title: "One Hundred Years of Solitude" - author: "Gabriel García Márquez" - genre: "Magical Realism" - published: 1967 - rating: 4.3 - tags: - - family - - history - - fantasy - awards: - nobel: true - nebula: false - book4: - title: "The Lord of the Rings" - author: "J.R.R. Tolkien" - genre: "Fantasy" - published: 1954 - rating: 4.7 - tags: - - adventure - - magic - - epic - awards: - hugo: false - nebula: false - book5: - title: "The Handmaid's Tale" - author: "Margaret Atwood" - genre: "Dystopian" - published: 1985 - rating: 4.1 - tags: - - feminism - - totalitarianism - - resistance - awards: - arthur c. clarke: true - booker prize: false - book6: - title: "Crime and Punishment" - author: "Fyodor Dostoevsky" - genre: "Psychological Thriller" - published: 1866 - rating: 4.3 - tags: - - philosophy - - crime - - redemption - awards: - none: true - book7: - title: "To Kill a Mockingbird" - author: "Harper Lee" - genre: "Southern Gothic" - published: 1960 - rating: 4.2 - tags: - - racism - - injustice - - coming-of-age - awards: - pulitzer: true - book8: - title: "1984" - author: "George Orwell" - genre: "Dystopian" - published: 1949 - rating: 4.2 - tags: - - surveillance - - totalitarianism - - propaganda - awards: - prometheus: true - book9: - title: "The Great Gatsby" - author: "F. Scott Fitzgerald" - genre: "Modernist" - published: 1925 - rating: 4.0 - tags: - - wealth - - american dream - - love - awards: - none: true - book10: - title: "Dune" - author: "Frank Herbert" - genre: "Science Fiction" - published: 1965 - rating: 4.6 - tags: - - politics - - desert - - ecology - awards: - hugo: true - nebula: true - timestamps: - ts1: - time: "1993-04-28T12:01:00.654321+00:00" - micros: 735998460654321 - millis: 735998460654 - seconds: 735998460 - vectors: - vec1: - embedding: [1.0, 2.0, 3.0] - vec2: - embedding: [4.0, 5.0, 6.0, 7.0] - vec3: - embedding: [5.0, 6.0, 7.0] - vec4: - embedding: [1.0, 2.0, 4.0] -tests: - - description: "testAggregates - count" - pipeline: - - Collection: books - - Aggregate: - - AliasedExpr: - - Expr.count: - - Field: rating - - "count" - assert_results: - - count: 10 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - mapValue: - fields: - count: - functionValue: - name: count - args: - - fieldReferenceValue: rating - - mapValue: {} - name: aggregate - - description: "testAggregates - count_if" - pipeline: - - Collection: books - - Aggregate: - - AliasedExpr: - - Expr.count_if: - - Expr.greater_than: - - Field: rating - - Constant: 4.2 - - "count_if_rating_gt_4_2" - assert_results: - - count_if_rating_gt_4_2: 5 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - mapValue: - fields: - count_if_rating_gt_4_2: - functionValue: - name: count_if - args: - - functionValue: - name: greater_than - args: - - fieldReferenceValue: rating - - doubleValue: 4.2 - - mapValue: {} - name: aggregate - - description: "testAggregates - count_distinct" - pipeline: - - Collection: books - - Aggregate: - - AliasedExpr: - - Expr.count_distinct: - - Field: genre - - "distinct_genres" - assert_results: - - distinct_genres: 8 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - mapValue: - fields: - distinct_genres: - functionValue: - name: count_distinct - args: - - fieldReferenceValue: genre - - mapValue: {} - name: aggregate - - description: "testAggregates - avg, count, max" - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: genre - - Constant: Science Fiction - - Aggregate: - - AliasedExpr: - - Expr.count: - - Field: rating - - "count" - - AliasedExpr: - - Expr.average: - - Field: rating - - "avg_rating" - - AliasedExpr: - - Expr.maximum: - - Field: rating - - "max_rating" - assert_results: - - count: 2 - avg_rating: 4.4 - max_rating: 4.6 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: genre - - stringValue: Science Fiction - name: equal - name: where - - args: - - mapValue: - fields: - avg_rating: - functionValue: - args: - - fieldReferenceValue: rating - name: average - count: - functionValue: - name: count - args: - - fieldReferenceValue: rating - max_rating: - functionValue: - args: - - fieldReferenceValue: rating - name: maximum - - mapValue: {} - name: aggregate - - description: testGroupBysWithoutAccumulators - pipeline: - - Collection: books - - Where: - - Expr.less_than: - - Field: published - - Constant: 1900 - - Aggregate: - accumulators: [] - groups: [genre] - assert_error: ".* requires at least one accumulator" - - description: testGroupBysAndAggregate - pipeline: - - Collection: books - - Where: - - Expr.less_than: - - Field: published - - Constant: 1984 - - Aggregate: - accumulators: - - AliasedExpr: - - Expr.average: - - Field: rating - - "avg_rating" - groups: [genre] - - Where: - - Expr.greater_than: - - Field: avg_rating - - Constant: 4.3 - - Sort: - - Ordering: - - Field: avg_rating - - ASCENDING - assert_results: - - avg_rating: 4.4 - genre: Science Fiction - - avg_rating: 4.5 - genre: Romance - - avg_rating: 4.7 - genre: Fantasy - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: published - - integerValue: '1984' - name: less_than - name: where - - args: - - mapValue: - fields: - avg_rating: - functionValue: - args: - - fieldReferenceValue: rating - name: average - - mapValue: - fields: - genre: - fieldReferenceValue: genre - name: aggregate - - args: - - functionValue: - args: - - fieldReferenceValue: avg_rating - - doubleValue: 4.3 - name: greater_than - name: where - - args: - - mapValue: - fields: - direction: - stringValue: ascending - expression: - fieldReferenceValue: avg_rating - name: sort - - description: testMinMax - pipeline: - - Collection: books - - Aggregate: - - AliasedExpr: - - Expr.count: - - Field: rating - - "count" - - AliasedExpr: - - Expr.maximum: - - Field: rating - - "max_rating" - - AliasedExpr: - - Expr.minimum: - - Field: published - - "min_published" - assert_results: - - count: 10 - max_rating: 4.7 - min_published: 1813 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - mapValue: - fields: - count: - functionValue: - args: - - fieldReferenceValue: rating - name: count - max_rating: - functionValue: - args: - - fieldReferenceValue: rating - name: maximum - min_published: - functionValue: - args: - - fieldReferenceValue: published - name: minimum - - mapValue: {} - name: aggregate - - description: selectSpecificFields - pipeline: - - Collection: books - - Select: - - title - - author - - Sort: - - Ordering: - - Field: author - - ASCENDING - assert_results: - - title: "The Hitchhiker's Guide to the Galaxy" - author: "Douglas Adams" - - title: "The Great Gatsby" - author: "F. Scott Fitzgerald" - - title: "Dune" - author: "Frank Herbert" - - title: "Crime and Punishment" - author: "Fyodor Dostoevsky" - - title: "One Hundred Years of Solitude" - author: "Gabriel García Márquez" - - title: "1984" - author: "George Orwell" - - title: "To Kill a Mockingbird" - author: "Harper Lee" - - title: "The Lord of the Rings" - author: "J.R.R. Tolkien" - - title: "Pride and Prejudice" - author: "Jane Austen" - - title: "The Handmaid's Tale" - author: "Margaret Atwood" - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - mapValue: - fields: - author: - fieldReferenceValue: author - title: - fieldReferenceValue: title - name: select - - args: - - mapValue: - fields: - direction: - stringValue: ascending - expression: - fieldReferenceValue: author - name: sort - - description: addAndRemoveFields - pipeline: - - Collection: books - - AddFields: - - AliasedExpr: - - Expr.string_concat: - - Field: author - - Constant: _ - - Field: title - - "author_title" - - AliasedExpr: - - Expr.string_concat: - - Field: title - - Constant: _ - - Field: author - - "title_author" - - RemoveFields: - - title_author - - tags - - awards - - rating - - title - - Field: published - - Field: genre - - Field: nestedField # Field does not exist, should be ignored - - Sort: - - Ordering: - - Field: author_title - - ASCENDING - assert_results: - - author: Douglas Adams - author_title: Douglas Adams_The Hitchhiker's Guide to the Galaxy - - author: F. Scott Fitzgerald - author_title: F. Scott Fitzgerald_The Great Gatsby - - author: Frank Herbert - author_title: Frank Herbert_Dune - - author: Fyodor Dostoevsky - author_title: Fyodor Dostoevsky_Crime and Punishment - - author: Gabriel García Márquez - author_title: Gabriel García Márquez_One Hundred Years of Solitude - - author: George Orwell - author_title: George Orwell_1984 - - author: Harper Lee - author_title: Harper Lee_To Kill a Mockingbird - - author: J.R.R. Tolkien - author_title: J.R.R. Tolkien_The Lord of the Rings - - author: Jane Austen - author_title: Jane Austen_Pride and Prejudice - - author: Margaret Atwood - author_title: Margaret Atwood_The Handmaid's Tale - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - mapValue: - fields: - author_title: - functionValue: - args: - - fieldReferenceValue: author - - stringValue: _ - - fieldReferenceValue: title - name: string_concat - title_author: - functionValue: - args: - - fieldReferenceValue: title - - stringValue: _ - - fieldReferenceValue: author - name: string_concat - name: add_fields - - args: - - fieldReferenceValue: title_author - - fieldReferenceValue: tags - - fieldReferenceValue: awards - - fieldReferenceValue: rating - - fieldReferenceValue: title - - fieldReferenceValue: published - - fieldReferenceValue: genre - - fieldReferenceValue: nestedField - name: remove_fields - - args: - - mapValue: - fields: - direction: - stringValue: ascending - expression: - fieldReferenceValue: author_title - name: sort - - description: whereByMultipleConditions - pipeline: - - Collection: books - - Where: - - And: - - Expr.greater_than: - - Field: rating - - Constant: 4.5 - - Expr.equal: - - Field: genre - - Constant: Science Fiction - assert_results: - - title: Dune - author: Frank Herbert - genre: Science Fiction - published: 1965 - rating: 4.6 - tags: - - politics - - desert - - ecology - awards: - hugo: true - nebula: true - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - functionValue: - args: - - fieldReferenceValue: rating - - doubleValue: 4.5 - name: greater_than - - functionValue: - args: - - fieldReferenceValue: genre - - stringValue: Science Fiction - name: equal - name: and - name: where - - description: whereByOrCondition - pipeline: - - Collection: books - - Where: - - Or: - - Expr.equal: - - Field: genre - - Constant: Romance - - Expr.equal: - - Field: genre - - Constant: Dystopian - - Select: - - title - - Sort: - - Ordering: - - Field: title - - ASCENDING - assert_results: - - title: "1984" - - title: Pride and Prejudice - - title: The Handmaid's Tale - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - functionValue: - args: - - fieldReferenceValue: genre - - stringValue: Romance - name: equal - - functionValue: - args: - - fieldReferenceValue: genre - - stringValue: Dystopian - name: equal - name: or - name: where - - args: - - mapValue: - fields: - title: - fieldReferenceValue: title - name: select - - args: - - mapValue: - fields: - direction: - stringValue: ascending - expression: - fieldReferenceValue: title - name: sort - - description: testPipelineWithOffsetAndLimit - pipeline: - - Collection: books - - Sort: - - Ordering: - - Field: author - - ASCENDING - - Offset: 5 - - Limit: 3 - - Select: - - title - - author - assert_results: - - title: "1984" - author: George Orwell - - title: To Kill a Mockingbird - author: Harper Lee - - title: The Lord of the Rings - author: J.R.R. Tolkien - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - mapValue: - fields: - direction: - stringValue: ascending - expression: - fieldReferenceValue: author - name: sort - - args: - - integerValue: '5' - name: offset - - args: - - integerValue: '3' - name: limit - - args: - - mapValue: - fields: - author: - fieldReferenceValue: author - title: - fieldReferenceValue: title - name: select - - description: testArrayContains - pipeline: - - Collection: books - - Where: - - Expr.array_contains: - - Field: tags - - Constant: comedy - assert_results: - - title: The Hitchhiker's Guide to the Galaxy - author: Douglas Adams - awards: - hugo: true - nebula: false - genre: Science Fiction - published: 1979 - rating: 4.2 - tags: ["comedy", "space", "adventure"] - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: tags - - stringValue: comedy - name: array_contains - name: where - - description: testArrayContainsAny - pipeline: - - Collection: books - - Where: - - Expr.array_contains_any: - - Field: tags - - - Constant: comedy - - Constant: classic - - Select: - - title - - Sort: - - Ordering: - - Field: title - - ASCENDING - assert_results: - - title: Pride and Prejudice - - title: The Hitchhiker's Guide to the Galaxy - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: tags - - functionValue: - args: - - stringValue: comedy - - stringValue: classic - name: array - name: array_contains_any - name: where - - args: - - mapValue: - fields: - title: - fieldReferenceValue: title - name: select - - args: - - mapValue: - fields: - direction: - stringValue: ascending - expression: - fieldReferenceValue: title - name: sort - - description: testArrayContainsAll - pipeline: - - Collection: books - - Where: - - Expr.array_contains_all: - - Field: tags - - - Constant: adventure - - Constant: magic - - Select: - - title - assert_results: - - title: The Lord of the Rings - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: tags - - functionValue: - args: - - stringValue: adventure - - stringValue: magic - name: array - name: array_contains_all - name: where - - args: - - mapValue: - fields: - title: - fieldReferenceValue: title - name: select - - description: testArrayLength - pipeline: - - Collection: books - - Select: - - AliasedExpr: - - Expr.array_length: - - Field: tags - - "tagsCount" - - Where: - - Expr.equal: - - Field: tagsCount - - Constant: 3 - assert_results: # All documents have 3 tags - - tagsCount: 3 - - tagsCount: 3 - - tagsCount: 3 - - tagsCount: 3 - - tagsCount: 3 - - tagsCount: 3 - - tagsCount: 3 - - tagsCount: 3 - - tagsCount: 3 - - tagsCount: 3 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - mapValue: - fields: - tagsCount: - functionValue: - args: - - fieldReferenceValue: tags - name: array_length - name: select - - args: - - functionValue: - args: - - fieldReferenceValue: tagsCount - - integerValue: '3' - name: equal - name: where - - description: testStringConcat - pipeline: - - Collection: books - - Sort: - - Ordering: - - Field: author - - ASCENDING - - Select: - - AliasedExpr: - - Expr.string_concat: - - Field: author - - Constant: " - " - - Field: title - - "bookInfo" - - Limit: 1 - assert_results: - - bookInfo: Douglas Adams - The Hitchhiker's Guide to the Galaxy - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - mapValue: - fields: - direction: - stringValue: ascending - expression: - fieldReferenceValue: author - name: sort - - args: - - mapValue: - fields: - bookInfo: - functionValue: - args: - - fieldReferenceValue: author - - stringValue: ' - ' - - fieldReferenceValue: title - name: string_concat - name: select - - args: - - integerValue: '1' - name: limit - - description: testStartsWith - pipeline: - - Collection: books - - Where: - - Expr.starts_with: - - Field: title - - Constant: The - - Select: - - title - - Sort: - - Ordering: - - Field: title - - ASCENDING - assert_results: - - title: The Great Gatsby - - title: The Handmaid's Tale - - title: The Hitchhiker's Guide to the Galaxy - - title: The Lord of the Rings - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: title - - stringValue: The - name: starts_with - name: where - - args: - - mapValue: - fields: - title: - fieldReferenceValue: title - name: select - - args: - - mapValue: - fields: - direction: - stringValue: ascending - expression: - fieldReferenceValue: title - name: sort - - description: testEndsWith - pipeline: - - Collection: books - - Where: - - Expr.ends_with: - - Field: title - - Constant: y - - Select: - - title - - Sort: - - Ordering: - - Field: title - - DESCENDING - assert_results: - - title: The Hitchhiker's Guide to the Galaxy - - title: The Great Gatsby - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: title - - stringValue: y - name: ends_with - name: where - - args: - - mapValue: - fields: - title: - fieldReferenceValue: title - name: select - - args: - - mapValue: - fields: - direction: - stringValue: descending - expression: - fieldReferenceValue: title - name: sort - - description: testConcat - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: title - - Constant: "The Hitchhiker's Guide to the Galaxy" - - Select: - - AliasedExpr: - - Expr.concat: - - Field: author - - Constant: ": " - - Field: title - - "author_title" - - AliasedExpr: - - Expr.concat: - - Field: tags - - - Constant: "new_tag" - - "concatenatedTags" - assert_results: - - author_title: "Douglas Adams: The Hitchhiker's Guide to the Galaxy" - concatenatedTags: - - comedy - - space - - adventure - - new_tag - - description: testLength - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: title - - Constant: "The Hitchhiker's Guide to the Galaxy" - - Select: - - AliasedExpr: - - Expr.length: - - Field: title - - "titleLength" - - AliasedExpr: - - Expr.length: - - Field: tags - - "tagsLength" - - AliasedExpr: - - Expr.length: - - Field: awards - - "awardsLength" - assert_results: - - titleLength: 36 - tagsLength: 3 - awardsLength: 2 - - description: testCharLength - pipeline: - - Collection: books - - Select: - - AliasedExpr: - - Expr.char_length: - - Field: title - - "titleLength" - - title - - Where: - - Expr.greater_than: - - Field: titleLength - - Constant: 20 - - Sort: - - Ordering: - - Field: title - - ASCENDING - assert_results: - - titleLength: 29 - title: One Hundred Years of Solitude - - titleLength: 36 - title: The Hitchhiker's Guide to the Galaxy - - titleLength: 21 - title: The Lord of the Rings - - titleLength: 21 - title: To Kill a Mockingbird - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - mapValue: - fields: - title: - fieldReferenceValue: title - titleLength: - functionValue: - args: - - fieldReferenceValue: title - name: char_length - name: select - - args: - - functionValue: - args: - - fieldReferenceValue: titleLength - - integerValue: '20' - name: greater_than - name: where - - args: - - mapValue: - fields: - direction: - stringValue: ascending - expression: - fieldReferenceValue: title - name: sort - - description: testStringFunctions - CharLength - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: author - - Constant: "Douglas Adams" - - Select: - - AliasedExpr: - - Expr.char_length: - - Field: title - - "title_length" - assert_results: - - title_length: 36 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: author - - stringValue: Douglas Adams - name: equal - name: where - - args: - - mapValue: - fields: - title_length: - functionValue: - args: - - fieldReferenceValue: title - name: char_length - name: select - - description: testStringFunctions - ByteLength - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: author - - Constant: Douglas Adams - - Select: - - AliasedExpr: - - Expr.byte_length: - - Expr.string_concat: - - Field: title - - Constant: _银河系漫游指南 - - "title_byte_length" - assert_results: - - title_byte_length: 58 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: author - - stringValue: Douglas Adams - name: equal - name: where - - args: - - mapValue: - fields: - title_byte_length: - functionValue: - args: - - functionValue: - args: - - fieldReferenceValue: title - - stringValue: "_\u94F6\u6CB3\u7CFB\u6F2B\u6E38\u6307\u5357" - name: string_concat - name: byte_length - name: select - - description: testLike - pipeline: - - Collection: books - - Where: - - Expr.like: - - Field: title - - Constant: "%Guide%" - - Select: - - title - assert_results: - - title: The Hitchhiker's Guide to the Galaxy - - description: testRegexContains - # Find titles that contain either "the" or "of" (case-insensitive) - pipeline: - - Collection: books - - Where: - - Expr.regex_contains: - - Field: title - - Constant: "(?i)(the|of)" - assert_count: 5 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: title - - stringValue: "(?i)(the|of)" - name: regex_contains - name: where - - description: testRegexMatches - # Find titles that contain either "the" or "of" (case-insensitive) - pipeline: - - Collection: books - - Where: - - Expr.regex_match: - - Field: title - - Constant: ".*(?i)(the|of).*" - assert_count: 5 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: title - - stringValue: ".*(?i)(the|of).*" - name: regex_match - name: where - - description: testArithmeticOperations - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: title - - Constant: To Kill a Mockingbird - - Select: - - AliasedExpr: - - Expr.add: - - Field: rating - - Constant: 1 - - "ratingPlusOne" - - AliasedExpr: - - Expr.subtract: - - Field: published - - Constant: 1900 - - "yearsSince1900" - - AliasedExpr: - - Expr.multiply: - - Field: rating - - Constant: 10 - - "ratingTimesTen" - - AliasedExpr: - - Expr.divide: - - Field: rating - - Constant: 2 - - "ratingDividedByTwo" - - AliasedExpr: - - Expr.multiply: - - Field: rating - - Constant: 20 - - "ratingTimes20" - - AliasedExpr: - - Expr.add: - - Field: rating - - Constant: 3 - - "ratingPlus3" - - AliasedExpr: - - Expr.mod: - - Field: rating - - Constant: 2 - - "ratingMod2" - assert_results: - - ratingPlusOne: 5.2 - yearsSince1900: 60 - ratingTimesTen: 42.0 - ratingDividedByTwo: 2.1 - ratingTimes20: 84 - ratingPlus3: 7.2 - ratingMod2: 0.20000000000000018 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: title - - stringValue: To Kill a Mockingbird - name: equal - name: where - - args: - - mapValue: - fields: - ratingDividedByTwo: - functionValue: - args: - - fieldReferenceValue: rating - - integerValue: '2' - name: divide - ratingPlusOne: - functionValue: - args: - - fieldReferenceValue: rating - - integerValue: '1' - name: add - ratingTimesTen: - functionValue: - args: - - fieldReferenceValue: rating - - integerValue: '10' - name: multiply - yearsSince1900: - functionValue: - args: - - fieldReferenceValue: published - - integerValue: '1900' - name: subtract - ratingTimes20: - functionValue: - args: - - fieldReferenceValue: rating - - integerValue: '20' - name: multiply - ratingPlus3: - functionValue: - args: - - fieldReferenceValue: rating - - integerValue: '3' - name: add - ratingMod2: - functionValue: - args: - - fieldReferenceValue: rating - - integerValue: '2' - name: mod - name: select - - description: testComparisonOperators - pipeline: - - Collection: books - - Where: - - And: - - Expr.greater_than: - - Field: rating - - Constant: 4.2 - - Expr.less_than_or_equal: - - Field: rating - - Constant: 4.5 - - Expr.not_equal: - - Field: genre - - Constant: Science Fiction - - Select: - - rating - - title - - Sort: - - Ordering: - - title - - ASCENDING - assert_results: - - rating: 4.3 - title: Crime and Punishment - - rating: 4.3 - title: One Hundred Years of Solitude - - rating: 4.5 - title: Pride and Prejudice - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - functionValue: - args: - - fieldReferenceValue: rating - - doubleValue: 4.2 - name: greater_than - - functionValue: - args: - - fieldReferenceValue: rating - - doubleValue: 4.5 - name: less_than_or_equal - - functionValue: - args: - - fieldReferenceValue: genre - - stringValue: Science Fiction - name: not_equal - name: and - name: where - - args: - - mapValue: - fields: - rating: - fieldReferenceValue: rating - title: - fieldReferenceValue: title - name: select - - args: - - mapValue: - fields: - direction: - stringValue: ascending - expression: - fieldReferenceValue: title - name: sort - - description: testLogicalOperators - pipeline: - - Collection: books - - Where: - - Or: - - And: - - Expr.greater_than: - - Field: rating - - Constant: 4.5 - - Expr.equal: - - Field: genre - - Constant: Science Fiction - - Expr.less_than: - - Field: published - - Constant: 1900 - - Select: - - title - - Sort: - - Ordering: - - Field: title - - ASCENDING - assert_results: - - title: Crime and Punishment - - title: Dune - - title: Pride and Prejudice - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - functionValue: - args: - - functionValue: - args: - - fieldReferenceValue: rating - - doubleValue: 4.5 - name: greater_than - - functionValue: - args: - - fieldReferenceValue: genre - - stringValue: Science Fiction - name: equal - name: and - - functionValue: - args: - - fieldReferenceValue: published - - integerValue: '1900' - name: less_than - name: or - name: where - - args: - - mapValue: - fields: - title: - fieldReferenceValue: title - name: select - - args: - - mapValue: - fields: - direction: - stringValue: ascending - expression: - fieldReferenceValue: title - name: sort - - description: testChecks - pipeline: - - Collection: books - - Where: - - Not: - - Expr.is_nan: - - Field: rating - - Select: - - AliasedExpr: - - Not: - - Expr.is_nan: - - Field: rating - - "ratingIsNotNaN" - - Limit: 1 - assert_results: - - ratingIsNotNaN: true - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - functionValue: - args: - - fieldReferenceValue: rating - name: is_nan - name: not - name: where - - args: - - mapValue: - fields: - ratingIsNotNaN: - functionValue: - args: - - functionValue: - args: - - fieldReferenceValue: rating - name: is_nan - name: not - name: select - - args: - - integerValue: '1' - name: limit - - description: testIsNotNull - pipeline: - - Collection: books - - Where: - - Expr.is_not_null: - - Field: rating - assert_count: 10 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: rating - name: is_not_null - name: where - - description: testIsNotNaN - pipeline: - - Collection: books - - Where: - - Expr.is_not_nan: - - Field: rating - assert_count: 10 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: rating - name: is_not_nan - name: where - - description: testIsAbsent - pipeline: - - Collection: books - - Where: - - Expr.is_absent: - - Field: awards.pulitzer - assert_count: 9 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: awards.pulitzer - name: is_absent - name: where - - description: testIfAbsent - pipeline: - - Collection: books - - Select: - - AliasedExpr: - - Expr.if_absent: - - Field: awards.pulitzer - - Constant: false - - "pulitzer_award" - - title - - Where: - - Expr.equal: - - Field: pulitzer_award - - Constant: true - assert_results: - - pulitzer_award: true - title: To Kill a Mockingbird - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - mapValue: - fields: - pulitzer_award: - functionValue: - name: if_absent - args: - - fieldReferenceValue: awards.pulitzer - - booleanValue: false - title: - fieldReferenceValue: title - name: select - - args: - - functionValue: - args: - - fieldReferenceValue: pulitzer_award - - booleanValue: true - name: equal - name: where - - description: testIsError - pipeline: - - Collection: books - - Select: - - AliasedExpr: - - Expr.is_error: - - Expr.divide: - - Field: rating - - Constant: "string" - - "is_error_result" - - Limit: 1 - assert_results: - - is_error_result: true - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - mapValue: - fields: - is_error_result: - functionValue: - name: is_error - args: - - functionValue: - name: divide - args: - - fieldReferenceValue: rating - - stringValue: "string" - name: select - - args: - - integerValue: '1' - name: limit - - description: testIfError - pipeline: - - Collection: books - - Select: - - AliasedExpr: - - Expr.if_error: - - Expr.divide: - - Field: rating - - Field: genre - - Constant: "An error occurred" - - "if_error_result" - - Limit: 1 - assert_results: - - if_error_result: "An error occurred" - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - mapValue: - fields: - if_error_result: - functionValue: - name: if_error - args: - - functionValue: - name: divide - args: - - fieldReferenceValue: rating - - fieldReferenceValue: genre - - stringValue: "An error occurred" - name: select - - args: - - integerValue: '1' - name: limit - - description: testLogicalMinMax - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: author - - Constant: Douglas Adams - - Select: - - AliasedExpr: - - Expr.logical_maximum: - - Field: rating - - Constant: 4.5 - - "max_rating" - - AliasedExpr: - - Expr.logical_minimum: - - Field: published - - Constant: 1900 - - "min_published" - assert_results: - - max_rating: 4.5 - min_published: 1900 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: author - - stringValue: Douglas Adams - name: equal - name: where - - args: - - mapValue: - fields: - min_published: - functionValue: - args: - - fieldReferenceValue: published - - integerValue: '1900' - name: minimum - max_rating: - functionValue: - args: - - fieldReferenceValue: rating - - doubleValue: 4.5 - name: maximum - name: select - - description: testMapGet - pipeline: - - Collection: books - - Sort: - - Ordering: - - Field: published - - DESCENDING - - Select: - - AliasedExpr: - - Expr.map_get: - - Field: awards - - hugo - - "hugoAward" - - Field: title - - Where: - - Expr.equal: - - Field: hugoAward - - Constant: true - assert_results: - - hugoAward: true - title: The Hitchhiker's Guide to the Galaxy - - hugoAward: true - title: Dune - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - mapValue: - fields: - direction: - stringValue: descending - expression: - fieldReferenceValue: published - name: sort - - args: - - mapValue: - fields: - hugoAward: - functionValue: - args: - - fieldReferenceValue: awards - - stringValue: hugo - name: map_get - title: - fieldReferenceValue: title - name: select - - args: - - functionValue: - args: - - fieldReferenceValue: hugoAward - - booleanValue: true - name: equal - name: where - - description: testMapGetWithField - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: title - - Constant: "Dune" - - AddFields: - - AliasedExpr: - - Constant: "hugo" - - "award_name" - - Select: - - AliasedExpr: - - Expr.map_get: - - Field: awards - - Field: award_name - - "hugoAward" - - Field: title - assert_results: - - hugoAward: true - title: Dune - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: title - - stringValue: "Dune" - name: equal - name: where - - args: - - mapValue: - fields: - award_name: - stringValue: "hugo" - name: add_fields - - args: - - mapValue: - fields: - hugoAward: - functionValue: - name: map_get - args: - - fieldReferenceValue: awards - - fieldReferenceValue: award_name - title: - fieldReferenceValue: title - name: select - - description: testMapRemove - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: title - - Constant: "Dune" - - Select: - - AliasedExpr: - - Expr.map_remove: - - Field: awards - - "nebula" - - "awards_removed" - assert_results: - - awards_removed: - hugo: true - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: title - - stringValue: "Dune" - name: equal - name: where - - args: - - mapValue: - fields: - awards_removed: - functionValue: - name: map_remove - args: - - fieldReferenceValue: awards - - stringValue: "nebula" - name: select - - description: testMapMerge - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: title - - Constant: "Dune" - - Select: - - AliasedExpr: - - Expr.map_merge: - - Field: awards - - Map: - elements: {"new_award": true, "hugo": false} - - Map: - elements: {"another_award": "yes"} - - "awards_merged" - assert_results: - - awards_merged: - hugo: false - nebula: true - new_award: true - another_award: "yes" - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: title - - stringValue: "Dune" - name: equal - name: where - - args: - - mapValue: - fields: - awards_merged: - functionValue: - name: map_merge - args: - - fieldReferenceValue: awards - - functionValue: - name: map - args: - - stringValue: "new_award" - - booleanValue: true - - stringValue: "hugo" - - booleanValue: false - - functionValue: - name: map - args: - - stringValue: "another_award" - - stringValue: "yes" - name: select - - description: testNestedFields - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: awards.hugo - - Constant: true - - Sort: - - Ordering: - - Field: title - - DESCENDING - - Select: - - title - - Field: awards.hugo - assert_results: - - title: The Hitchhiker's Guide to the Galaxy - awards.hugo: true - - title: Dune - awards.hugo: true - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: awards.hugo - - booleanValue: true - name: equal - name: where - - args: - - mapValue: - fields: - direction: - stringValue: descending - expression: - fieldReferenceValue: title - name: sort - - args: - - mapValue: - fields: - awards.hugo: - fieldReferenceValue: awards.hugo - title: - fieldReferenceValue: title - name: select - - description: testSampleLimit - pipeline: - - Collection: books - - Sample: 3 - assert_count: 3 # Results will vary due to randomness - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - integerValue: '3' - - stringValue: documents - name: sample - - description: testSamplePercentage - pipeline: - - Collection: books - - Sample: - - SampleOptions: - - 0.6 - - percent - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - doubleValue: 0.6 - - stringValue: percent - name: sample - - description: testUnion - pipeline: - - Collection: books - - Union: - - Pipeline: - - Collection: books - assert_count: 20 # Results will be duplicated - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - pipelineValue: - stages: - - args: - - referenceValue: /books - name: collection - name: union - - description: testUnnest - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: title - - Constant: The Hitchhiker's Guide to the Galaxy - - Unnest: - - tags - - tags_alias - - Select: tags_alias - assert_results: - - tags_alias: comedy - - tags_alias: space - - tags_alias: adventure - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: title - - stringValue: The Hitchhiker's Guide to the Galaxy - name: equal - name: where - - args: - - fieldReferenceValue: tags - - fieldReferenceValue: tags_alias - name: unnest - - args: - - mapValue: - fields: - tags_alias: - fieldReferenceValue: tags_alias - name: select - - description: testGreaterThanOrEqual - pipeline: - - Collection: books - - Where: - - Expr.greater_than_or_equal: - - Field: rating - - Constant: 4.6 - - Select: - - title - - rating - - Sort: - - Ordering: - - Field: rating - - ASCENDING - assert_results: - - title: Dune - rating: 4.6 - - title: The Lord of the Rings - rating: 4.7 - - description: testInAndNotIn - pipeline: - - Collection: books - - Where: - - And: - - Expr.equal_any: - - Field: genre - - - Constant: Romance - - Constant: Dystopian - - Expr.not_equal_any: - - Field: author - - - Constant: "George Orwell" - assert_results: - - title: "Pride and Prejudice" - author: "Jane Austen" - genre: "Romance" - published: 1813 - rating: 4.5 - tags: - - classic - - social commentary - - love - awards: - none: true - - title: "The Handmaid's Tale" - author: "Margaret Atwood" - genre: "Dystopian" - published: 1985 - rating: 4.1 - tags: - - feminism - - totalitarianism - - resistance - awards: - "arthur c. clarke": true - "booker prize": false - - description: testArrayReverse - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: title - - Constant: "The Hitchhiker's Guide to the Galaxy" - - Select: - - AliasedExpr: - - Expr.array_reverse: - - Field: tags - - "reversedTags" - assert_results: - - reversedTags: - - adventure - - space - - comedy - - description: testDocumentId - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: title - - Constant: "The Hitchhiker's Guide to the Galaxy" - - Select: - - AliasedExpr: - - Expr.document_id: - - Field: __name__ - - "doc_id" - assert_results: - - doc_id: "book1" - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: title - - stringValue: "The Hitchhiker's Guide to the Galaxy" - name: equal - name: where - - args: - - mapValue: - fields: - doc_id: - functionValue: - name: document_id - args: - - fieldReferenceValue: __name__ - name: select - - description: testCurrentTimestamp - pipeline: - - Collection: books - - Limit: 1 - - Select: - - AliasedExpr: - - And: - - Expr.greater_than_or_equal: - - CurrentTimestamp: [] - - Expr.unix_seconds_to_timestamp: - - Constant: 1735689600 # 2025-01-01 - - Expr.less_than: - - CurrentTimestamp: [] - - Expr.unix_seconds_to_timestamp: - - Constant: 4892438400 # 2125-01-01 - - "is_between_2025_and_2125" - assert_results: - - is_between_2025_and_2125: true - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - integerValue: '1' - name: limit - - args: - - mapValue: - fields: - is_between_2025_and_2125: - functionValue: - name: and - args: - - functionValue: - name: greater_than_or_equal - args: - - functionValue: - name: current_timestamp - - functionValue: - name: unix_seconds_to_timestamp - args: - - integerValue: '1735689600' - - functionValue: - name: less_than - args: - - functionValue: - name: current_timestamp - - functionValue: - name: unix_seconds_to_timestamp - args: - - integerValue: '4892438400' - name: select - - description: testArrayConcat - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: title - - Constant: "The Hitchhiker's Guide to the Galaxy" - - Select: - - AliasedExpr: - - Expr.array_concat: - - Field: tags - - Constant: ["new_tag", "another_tag"] - - "concatenatedTags" - assert_results: - - concatenatedTags: - - comedy - - space - - adventure - - new_tag - - another_tag - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: title - - stringValue: "The Hitchhiker's Guide to the Galaxy" - name: equal - name: where - - args: - - mapValue: - fields: - concatenatedTags: - functionValue: - args: - - fieldReferenceValue: tags - - functionValue: - args: - - stringValue: "new_tag" - - stringValue: "another_tag" - name: array - name: array_concat - name: select - - description: testArrayConcatMultiple - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: title - - Constant: "Dune" - - Select: - - AliasedExpr: - - Expr.array_concat: - - Field: tags - - Constant: ["sci-fi"] - - Constant: ["classic", "epic"] - - "concatenatedTags" - assert_results: - - concatenatedTags: - - politics - - desert - - ecology - - sci-fi - - classic - - epic - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: title - - stringValue: "Dune" - name: equal - name: where - - args: - - mapValue: - fields: - concatenatedTags: - functionValue: - args: - - fieldReferenceValue: tags - - functionValue: - args: - - stringValue: "sci-fi" - name: array - - functionValue: - args: - - stringValue: "classic" - - stringValue: "epic" - name: array - name: array_concat - name: select - - description: testMapMergeLiterals - pipeline: - - Collection: books - - Limit: 1 - - Select: - - AliasedExpr: - - Expr.map_merge: - - Map: - elements: {"a": "orig", "b": "orig"} - - Map: - elements: {"b": "new", "c": "new"} - - "merged" - assert_results: - - merged: - a: "orig" - b: "new" - c: "new" - - description: testArrayContainsAnyWithField - pipeline: - - Collection: books - - AddFields: - - AliasedExpr: - - Expr.array_concat: - - Field: tags - - Array: ["Dystopian"] - - "new_tags" - - Where: - - Expr.array_contains_any: - - Field: new_tags - - - Constant: non_existent_tag - - Field: genre - - Select: - - title - - genre - - Sort: - - Ordering: - - Field: title - - ASCENDING - assert_results: - - title: "1984" - genre: "Dystopian" - - title: "The Handmaid's Tale" - genre: "Dystopian" - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - mapValue: - fields: - new_tags: - functionValue: - args: - - fieldReferenceValue: tags - - functionValue: - args: - - stringValue: "Dystopian" - name: array - name: array_concat - name: add_fields - - args: - - functionValue: - args: - - fieldReferenceValue: new_tags - - functionValue: - args: - - stringValue: "non_existent_tag" - - fieldReferenceValue: genre - name: array - name: array_contains_any - name: where - - args: - - mapValue: - fields: - title: - fieldReferenceValue: title - genre: - fieldReferenceValue: genre - name: select - - args: - - mapValue: - fields: - direction: - stringValue: ascending - expression: - fieldReferenceValue: title - name: sort - - description: testArrayConcatLiterals - pipeline: - - Collection: books - - Limit: 1 - - Select: - - AliasedExpr: - - Expr.array_concat: - - Array: [1, 2, 3] - - Array: [4, 5] - - "concatenated" - assert_results: - - concatenated: [1, 2, 3, 4, 5] - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - integerValue: '1' - name: limit - - args: - - mapValue: - fields: - concatenated: - functionValue: - args: - - functionValue: - args: - - integerValue: '1' - - integerValue: '2' - - integerValue: '3' - name: array - - functionValue: - args: - - integerValue: '4' - - integerValue: '5' - name: array - name: array_concat - name: select - - description: testExists - pipeline: - - Collection: books - - Where: - - And: - - Expr.exists: - - Field: awards.pulitzer - - Expr.equal: - - Field: awards.pulitzer - - Constant: true - - Select: - - title - assert_results: - - title: To Kill a Mockingbird - - description: testSum - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: genre - - Constant: Science Fiction - - Aggregate: - - AliasedExpr: - - Expr.sum: - - Field: rating - - "total_rating" - assert_results: - - total_rating: 8.8 - - description: testStringContains - pipeline: - - Collection: books - - Where: - - Expr.string_contains: - - Field: title - - Constant: "Hitchhiker's" - - Select: - - title - assert_results: - - title: "The Hitchhiker's Guide to the Galaxy" - - description: testVectorLength - pipeline: - - Collection: vectors - - Select: - - AliasedExpr: - - Expr.vector_length: - - Field: embedding - - "embedding_length" - - Sort: - - Ordering: - - Field: embedding_length - - ASCENDING - assert_results: - - embedding_length: 3 - - embedding_length: 3 - - embedding_length: 3 - - embedding_length: 4 - - description: testTimestampFunctions - pipeline: - - Collection: timestamps - - Select: - - AliasedExpr: - - Expr.timestamp_to_unix_micros: - - Field: time - - "micros" - - AliasedExpr: - - Expr.timestamp_to_unix_millis: - - Field: time - - "millis" - - AliasedExpr: - - Expr.timestamp_to_unix_seconds: - - Field: time - - "seconds" - - AliasedExpr: - - Expr.unix_micros_to_timestamp: - - Field: micros - - "from_micros" - - AliasedExpr: - - Expr.unix_millis_to_timestamp: - - Field: millis - - "from_millis" - - AliasedExpr: - - Expr.unix_seconds_to_timestamp: - - Field: seconds - - "from_seconds" - - AliasedExpr: - - Expr.timestamp_add: - - Field: time - - Constant: "day" - - Constant: 1 - - "plus_day" - - AliasedExpr: - - Expr.timestamp_subtract: - - Field: time - - Constant: "hour" - - Constant: 1 - - "minus_hour" - assert_results: - - micros: 735998460654321 - millis: 735998460654 - seconds: 735998460 - from_micros: "1993-04-28T12:01:00.654321+00:00" - from_millis: "1993-04-28T12:01:00.654000+00:00" - from_seconds: "1993-04-28T12:01:00.000000+00:00" - plus_day: "1993-04-29T12:01:00.654321+00:00" - minus_hour: "1993-04-28T11:01:00.654321+00:00" - - description: testCollectionId - pipeline: - - Collection: books - - Limit: 1 - - Select: - - AliasedExpr: - - Expr.collection_id: - - Field: __name__ - - "collectionName" - assert_results: - - collectionName: "books" - - description: testXor - pipeline: - - Collection: books - - Where: - - Xor: - - - Expr.equal: - - Field: genre - - Constant: Romance - - Expr.greater_than: - - Field: published - - Constant: 1980 - - Select: - - title - - genre - - published - - Sort: - - Ordering: - - Field: title - - ASCENDING - assert_results: - - title: "Pride and Prejudice" - genre: "Romance" - published: 1813 - - title: "The Handmaid's Tale" - genre: "Dystopian" - published: 1985 - - description: testConditional - pipeline: - - Collection: books - - Select: - - title - - AliasedExpr: - - Conditional: - - Expr.greater_than: - - Field: published - - Constant: 1950 - - Constant: "Modern" - - Constant: "Classic" - - "era" - - Sort: - - Ordering: - - Field: title - - ASCENDING - - Limit: 4 - assert_results: - - title: "1984" - era: "Classic" - - title: "Crime and Punishment" - era: "Classic" - - title: "Dune" - era: "Modern" - - title: "One Hundred Years of Solitude" - era: "Modern" - - description: testFieldToFieldArithmetic - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: title - - Constant: "Dune" - - Select: - - AliasedExpr: - - Expr.add: - - Field: published - - Field: rating - - "pub_plus_rating" - assert_results: - - pub_plus_rating: 1969.6 - - description: testFieldToFieldComparison - pipeline: - - Collection: books - - Where: - - Expr.greater_than: - - Field: published - - Field: rating - - Select: - - title - assert_count: 10 # All books were published after year 4.7 - - description: testExistsNegative - pipeline: - - Collection: books - - Where: - - Expr.exists: - - Field: non_existent_field - assert_count: 0 - - description: testConditionalWithFields - pipeline: - - Collection: books - - Where: - - Expr.equal_any: - - Field: title - - - Constant: "Dune" - - Constant: "1984" - - Select: - - title - - AliasedExpr: - - Conditional: - - Expr.greater_than: - - Field: published - - Constant: 1950 - - Field: author - - Field: genre - - "conditional_field" - - Sort: - - Ordering: - - Field: title - - ASCENDING - assert_results: - - title: "1984" - conditional_field: "Dystopian" - - title: "Dune" - conditional_field: "Frank Herbert" - - description: testFindNearestEuclidean - pipeline: - - Collection: vectors - - FindNearest: - field: embedding - vector: [1.0, 2.0, 3.0] - distance_measure: EUCLIDEAN - options: - FindNearestOptions: - limit: 2 - distance_field: - Field: distance - - Select: - - distance - assert_results: - - distance: 0.0 - - distance: 1.0 - assert_proto: - pipeline: - stages: - - name: collection - args: - - referenceValue: /vectors - - name: find_nearest - args: - - fieldReferenceValue: embedding - - mapValue: - fields: - __type__: - stringValue: __vector__ - value: - arrayValue: - values: - - doubleValue: 1.0 - - doubleValue: 2.0 - - doubleValue: 3.0 - - stringValue: euclidean - options: - limit: - integerValue: '2' - distance_field: - fieldReferenceValue: distance - - name: select - args: - - mapValue: - fields: - distance: - fieldReferenceValue: distance - - description: testMathExpressions - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: title - - Constant: To Kill a Mockingbird - - Select: - - AliasedExpr: - - Expr.abs: - - Field: rating - - "abs_rating" - - AliasedExpr: - - Expr.ceil: - - Field: rating - - "ceil_rating" - - AliasedExpr: - - Expr.exp: - - Field: rating - - "exp_rating" - - AliasedExpr: - - Expr.floor: - - Field: rating - - "floor_rating" - - AliasedExpr: - - Expr.ln: - - Field: rating - - "ln_rating" - - AliasedExpr: - - Expr.log10: - - Field: rating - - "log_rating_base10" - - AliasedExpr: - - Expr.log: - - Field: rating - - Constant: 2 - - "log_rating_base2" - - AliasedExpr: - - Expr.pow: - - Field: rating - - Constant: 2 - - "pow_rating" - - AliasedExpr: - - Expr.sqrt: - - Field: rating - - "sqrt_rating" - assert_results_approximate: - - abs_rating: 4.2 - ceil_rating: 5.0 - exp_rating: 66.686331 - floor_rating: 4.0 - ln_rating: 1.4350845 - log_rating_base10: 0.623249 - log_rating_base2: 2.0704 - pow_rating: 17.64 - sqrt_rating: 2.049390 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: title - - stringValue: To Kill a Mockingbird - name: equal - name: where - - args: - - mapValue: - fields: - abs_rating: - functionValue: - args: - - fieldReferenceValue: rating - name: abs - ceil_rating: - functionValue: - args: - - fieldReferenceValue: rating - name: ceil - exp_rating: - functionValue: - args: - - fieldReferenceValue: rating - name: exp - floor_rating: - functionValue: - args: - - fieldReferenceValue: rating - name: floor - ln_rating: - functionValue: - args: - - fieldReferenceValue: rating - name: ln - log_rating_base10: - functionValue: - args: - - fieldReferenceValue: rating - name: log10 - log_rating_base2: - functionValue: - args: - - fieldReferenceValue: rating - - integerValue: '2' - name: log - pow_rating: - functionValue: - args: - - fieldReferenceValue: rating - - integerValue: '2' - name: pow - sqrt_rating: - functionValue: - args: - - fieldReferenceValue: rating - name: sqrt - name: select - - description: testRoundExpressions - pipeline: - - Collection: books - - Where: - - Expr.equal_any: - - Field: title - - - Constant: "To Kill a Mockingbird" # rating 4.2 - - Constant: "Pride and Prejudice" # rating 4.5 - - Constant: "The Lord of the Rings" # rating 4.7 - - Select: - - title - - AliasedExpr: - - Expr.round: - - Field: rating - - "round_rating" - - Sort: - - Ordering: - - Field: title - - ASCENDING - assert_results: - - title: "Pride and Prejudice" - round_rating: 5.0 - - title: "The Lord of the Rings" - round_rating: 5.0 - - title: "To Kill a Mockingbird" - round_rating: 4.0 - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: title - - functionValue: - args: - - stringValue: "To Kill a Mockingbird" - - stringValue: "Pride and Prejudice" - - stringValue: "The Lord of the Rings" - name: array - name: equal_any - name: where - - args: - - mapValue: - fields: - title: - fieldReferenceValue: title - round_rating: - functionValue: - args: - - fieldReferenceValue: rating - name: round - name: select - - args: - - mapValue: - fields: - direction: - stringValue: ascending - expression: - fieldReferenceValue: title - name: sort - - description: testFindNearestDotProduct - pipeline: - - Collection: vectors - - FindNearest: - field: embedding - vector: [1.0, 2.0, 3.0] - distance_measure: DOT_PRODUCT - options: - FindNearestOptions: - limit: 3 - distance_field: - Field: distance - - Select: - - distance - assert_results: - - distance: 38.0 - - distance: 17.0 - - distance: 14.0 - assert_proto: - pipeline: - stages: - - name: collection - args: - - referenceValue: /vectors - - name: find_nearest - args: - - fieldReferenceValue: embedding - - mapValue: - fields: - __type__: - stringValue: __vector__ - value: - arrayValue: - values: - - doubleValue: 1.0 - - doubleValue: 2.0 - - doubleValue: 3.0 - - stringValue: dot_product - options: - limit: - integerValue: '3' - distance_field: - fieldReferenceValue: distance - - name: select - args: - - mapValue: - fields: - distance: - fieldReferenceValue: distance - - description: testDotProductWithConstant - pipeline: - - Collection: vectors - - Where: - - Expr.equal: - - Field: embedding - - Vector: [1.0, 2.0, 3.0] - - Select: - - AliasedExpr: - - Expr.dot_product: - - Field: embedding - - Vector: [1.0, 1.0, 1.0] - - "dot_product_result" - assert_results: - - dot_product_result: 6.0 - - description: testEuclideanDistanceWithConstant - pipeline: - - Collection: vectors - - Where: - - Expr.equal: - - Field: embedding - - Vector: [1.0, 2.0, 3.0] - - Select: - - AliasedExpr: - - Expr.euclidean_distance: - - Field: embedding - - Vector: [1.0, 2.0, 3.0] - - "euclidean_distance_result" - assert_results: - - euclidean_distance_result: 0.0 - - description: testCosineDistanceWithConstant - pipeline: - - Collection: vectors - - Where: - - Expr.equal: - - Field: embedding - - Vector: [1.0, 2.0, 3.0] - - Select: - - AliasedExpr: - - Expr.cosine_distance: - - Field: embedding - - Vector: [1.0, 2.0, 3.0] - - "cosine_distance_result" - assert_results: - - cosine_distance_result: 0.0 - - description: testStringFunctions - ToLower - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: author - - Constant: "Douglas Adams" - - Select: - - AliasedExpr: - - Expr.to_lower: - - Field: title - - "lower_title" - assert_results: - - lower_title: "the hitchhiker's guide to the galaxy" - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: author - - stringValue: Douglas Adams - name: equal - name: where - - args: - - mapValue: - fields: - lower_title: - functionValue: - args: - - fieldReferenceValue: title - name: to_lower - name: select - - description: testStringFunctions - ToUpper - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: author - - Constant: "Douglas Adams" - - Select: - - AliasedExpr: - - Expr.to_upper: - - Field: title - - "upper_title" - assert_results: - - upper_title: "THE HITCHHIKER'S GUIDE TO THE GALAXY" - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: author - - stringValue: Douglas Adams - name: equal - name: where - - args: - - mapValue: - fields: - upper_title: - functionValue: - args: - - fieldReferenceValue: title - name: to_upper - name: select - - description: testStringFunctions - Trim - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: author - - Constant: "Douglas Adams" - - Select: - - AliasedExpr: - - Expr.trim: - - Expr.string_concat: - - Constant: " " - - Field: title - - Constant: " " - - "trimmed_title" - assert_results: - - trimmed_title: "The Hitchhiker's Guide to the Galaxy" - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: author - - stringValue: Douglas Adams - name: equal - name: where - - args: - - mapValue: - fields: - trimmed_title: - functionValue: - args: - - functionValue: - args: - - stringValue: " " - - fieldReferenceValue: title - - stringValue: " " - name: string_concat - name: trim - name: select - - description: testStringFunctions - StringReverse - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: author - - Constant: "Jane Austen" - - Select: - - AliasedExpr: - - Expr.string_reverse: - - Field: title - - "reversed_title" - assert_results: - - reversed_title: "ecidujerP dna edirP" - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: author - - stringValue: "Jane Austen" - name: equal - name: where - - args: - - mapValue: - fields: - reversed_title: - functionValue: - args: - - fieldReferenceValue: title - name: string_reverse - name: select - - description: testStringFunctions - Substring - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: author - - Constant: "Douglas Adams" - - Select: - - AliasedExpr: - - Expr.substring: - - Field: title - - Constant: 4 - - Constant: 11 - - "substring_title" - assert_results: - - substring_title: "Hitchhiker'" - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: author - - stringValue: "Douglas Adams" - name: equal - name: where - - args: - - mapValue: - fields: - substring_title: - functionValue: - args: - - fieldReferenceValue: title - - integerValue: '4' - - integerValue: '11' - name: substring - name: select - - description: testStringFunctions - Substring without length - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: author - - Constant: "Fyodor Dostoevsky" - - Select: - - AliasedExpr: - - Expr.substring: - - Field: title - - Constant: 10 - - "substring_title" - assert_results: - - substring_title: "Punishment" - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: author - - stringValue: "Fyodor Dostoevsky" - name: equal - name: where - - args: - - mapValue: - fields: - substring_title: - functionValue: - args: - - fieldReferenceValue: title - - integerValue: '10' - name: substring - name: select - - description: testStringFunctions - Join - pipeline: - - Collection: books - - Where: - - Expr.equal: - - Field: author - - Constant: "Douglas Adams" - - Select: - - AliasedExpr: - - Expr.join: - - Field: tags - - Constant: ", " - - "joined_tags" - assert_results: - - joined_tags: "comedy, space, adventure" - assert_proto: - pipeline: - stages: - - args: - - referenceValue: /books - name: collection - - args: - - functionValue: - args: - - fieldReferenceValue: author - - stringValue: "Douglas Adams" - name: equal - name: where - - args: - - mapValue: - fields: - joined_tags: - functionValue: - args: - - fieldReferenceValue: tags - - stringValue: ", " - name: join - name: select diff --git a/tests/system/pipeline_e2e/aggregates.yaml b/tests/system/pipeline_e2e/aggregates.yaml new file mode 100644 index 000000000..18902aff4 --- /dev/null +++ b/tests/system/pipeline_e2e/aggregates.yaml @@ -0,0 +1,269 @@ +tests: + - description: "testAggregates - count" + pipeline: + - Collection: books + - Aggregate: + - AliasedExpr: + - Expr.count: + - Field: rating + - "count" + assert_results: + - count: 10 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + count: + functionValue: + name: count + args: + - fieldReferenceValue: rating + - mapValue: {} + name: aggregate + - description: "testAggregates - count_if" + pipeline: + - Collection: books + - Aggregate: + - AliasedExpr: + - Expr.count_if: + - Expr.greater_than: + - Field: rating + - Constant: 4.2 + - "count_if_rating_gt_4_2" + assert_results: + - count_if_rating_gt_4_2: 5 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + count_if_rating_gt_4_2: + functionValue: + name: count_if + args: + - functionValue: + name: greater_than + args: + - fieldReferenceValue: rating + - doubleValue: 4.2 + - mapValue: {} + name: aggregate + - description: "testAggregates - count_distinct" + pipeline: + - Collection: books + - Aggregate: + - AliasedExpr: + - Expr.count_distinct: + - Field: genre + - "distinct_genres" + assert_results: + - distinct_genres: 8 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + distinct_genres: + functionValue: + name: count_distinct + args: + - fieldReferenceValue: genre + - mapValue: {} + name: aggregate + - description: "testAggregates - avg, count, max" + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: genre + - Constant: Science Fiction + - Aggregate: + - AliasedExpr: + - Expr.count: + - Field: rating + - "count" + - AliasedExpr: + - Expr.average: + - Field: rating + - "avg_rating" + - AliasedExpr: + - Expr.maximum: + - Field: rating + - "max_rating" + assert_results: + - count: 2 + avg_rating: 4.4 + max_rating: 4.6 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: genre + - stringValue: Science Fiction + name: equal + name: where + - args: + - mapValue: + fields: + avg_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: average + count: + functionValue: + name: count + args: + - fieldReferenceValue: rating + max_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: maximum + - mapValue: {} + name: aggregate + - description: testGroupBysWithoutAccumulators + pipeline: + - Collection: books + - Where: + - Expr.less_than: + - Field: published + - Constant: 1900 + - Aggregate: + accumulators: [] + groups: [genre] + assert_error: ".* requires at least one accumulator" + - description: testGroupBysAndAggregate + pipeline: + - Collection: books + - Where: + - Expr.less_than: + - Field: published + - Constant: 1984 + - Aggregate: + accumulators: + - AliasedExpr: + - Expr.average: + - Field: rating + - "avg_rating" + groups: [genre] + - Where: + - Expr.greater_than: + - Field: avg_rating + - Constant: 4.3 + - Sort: + - Ordering: + - Field: avg_rating + - ASCENDING + assert_results: + - avg_rating: 4.4 + genre: Science Fiction + - avg_rating: 4.5 + genre: Romance + - avg_rating: 4.7 + genre: Fantasy + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: published + - integerValue: '1984' + name: less_than + name: where + - args: + - mapValue: + fields: + avg_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: average + - mapValue: + fields: + genre: + fieldReferenceValue: genre + name: aggregate + - args: + - functionValue: + args: + - fieldReferenceValue: avg_rating + - doubleValue: 4.3 + name: greater_than + name: where + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: avg_rating + name: sort + - description: testMinMax + pipeline: + - Collection: books + - Aggregate: + - AliasedExpr: + - Expr.count: + - Field: rating + - "count" + - AliasedExpr: + - Expr.maximum: + - Field: rating + - "max_rating" + - AliasedExpr: + - Expr.minimum: + - Field: published + - "min_published" + assert_results: + - count: 10 + max_rating: 4.7 + min_published: 1813 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + count: + functionValue: + args: + - fieldReferenceValue: rating + name: count + max_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: maximum + min_published: + functionValue: + args: + - fieldReferenceValue: published + name: minimum + - mapValue: {} + name: aggregate \ No newline at end of file diff --git a/tests/system/pipeline_e2e/array.yaml b/tests/system/pipeline_e2e/array.yaml new file mode 100644 index 000000000..d63a63402 --- /dev/null +++ b/tests/system/pipeline_e2e/array.yaml @@ -0,0 +1,388 @@ +tests: + - description: testArrayContains + pipeline: + - Collection: books + - Where: + - Expr.array_contains: + - Field: tags + - Constant: comedy + assert_results: + - title: The Hitchhiker's Guide to the Galaxy + author: Douglas Adams + awards: + hugo: true + nebula: false + genre: Science Fiction + published: 1979 + rating: 4.2 + tags: ["comedy", "space", "adventure"] + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: tags + - stringValue: comedy + name: array_contains + name: where + - description: testArrayContainsAny + pipeline: + - Collection: books + - Where: + - Expr.array_contains_any: + - Field: tags + - - Constant: comedy + - Constant: classic + - Select: + - title + - Sort: + - Ordering: + - Field: title + - ASCENDING + assert_results: + - title: Pride and Prejudice + - title: The Hitchhiker's Guide to the Galaxy + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: tags + - functionValue: + args: + - stringValue: comedy + - stringValue: classic + name: array + name: array_contains_any + name: where + - args: + - mapValue: + fields: + title: + fieldReferenceValue: title + name: select + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: title + name: sort + - description: testArrayContainsAll + pipeline: + - Collection: books + - Where: + - Expr.array_contains_all: + - Field: tags + - - Constant: adventure + - Constant: magic + - Select: + - title + assert_results: + - title: The Lord of the Rings + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: tags + - functionValue: + args: + - stringValue: adventure + - stringValue: magic + name: array + name: array_contains_all + name: where + - args: + - mapValue: + fields: + title: + fieldReferenceValue: title + name: select + - description: testArrayLength + pipeline: + - Collection: books + - Select: + - AliasedExpr: + - Expr.array_length: + - Field: tags + - "tagsCount" + - Where: + - Expr.equal: + - Field: tagsCount + - Constant: 3 + assert_results: # All documents have 3 tags + - tagsCount: 3 + - tagsCount: 3 + - tagsCount: 3 + - tagsCount: 3 + - tagsCount: 3 + - tagsCount: 3 + - tagsCount: 3 + - tagsCount: 3 + - tagsCount: 3 + - tagsCount: 3 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + tagsCount: + functionValue: + args: + - fieldReferenceValue: tags + name: array_length + name: select + - args: + - functionValue: + args: + - fieldReferenceValue: tagsCount + - integerValue: '3' + name: equal + name: where + - description: testArrayReverse + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: "The Hitchhiker's Guide to the Galaxy" + - Select: + - AliasedExpr: + - Expr.array_reverse: + - Field: tags + - "reversedTags" + assert_results: + - reversedTags: + - adventure + - space + - comedy + - description: testArrayConcat + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: "The Hitchhiker's Guide to the Galaxy" + - Select: + - AliasedExpr: + - Expr.array_concat: + - Field: tags + - Array: ["new_tag", "another_tag"] + - "concatenatedTags" + assert_results: + - concatenatedTags: + - comedy + - space + - adventure + - new_tag + - another_tag + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: "The Hitchhiker's Guide to the Galaxy" + name: equal + name: where + - args: + - mapValue: + fields: + concatenatedTags: + functionValue: + args: + - fieldReferenceValue: tags + - functionValue: + args: + - stringValue: "new_tag" + - stringValue: "another_tag" + name: array + name: array_concat + name: select + - description: testArrayConcatMultiple + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: "Dune" + - Select: + - AliasedExpr: + - Expr.array_concat: + - Field: tags + - ["sci-fi"] + - ["classic", "epic"] + - "concatenatedTags" + assert_results: + - concatenatedTags: + - politics + - desert + - ecology + - sci-fi + - classic + - epic + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: "Dune" + name: equal + name: where + - args: + - mapValue: + fields: + concatenatedTags: + functionValue: + args: + - fieldReferenceValue: tags + - functionValue: + args: + - stringValue: "sci-fi" + name: array + - functionValue: + args: + - stringValue: "classic" + - stringValue: "epic" + name: array + name: array_concat + name: select + - description: testArrayContainsAnyWithField + pipeline: + - Collection: books + - AddFields: + - AliasedExpr: + - Expr.array_concat: + - Field: tags + - Array: ["Dystopian"] + - "new_tags" + - Where: + - Expr.array_contains_any: + - Field: new_tags + - - Constant: non_existent_tag + - Field: genre + - Select: + - title + - genre + - Sort: + - Ordering: + - Field: title + - ASCENDING + assert_results: + - title: "1984" + genre: "Dystopian" + - title: "The Handmaid's Tale" + genre: "Dystopian" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + new_tags: + functionValue: + args: + - fieldReferenceValue: tags + - functionValue: + args: + - stringValue: "Dystopian" + name: array + name: array_concat + name: add_fields + - args: + - functionValue: + args: + - fieldReferenceValue: new_tags + - functionValue: + args: + - stringValue: "non_existent_tag" + - fieldReferenceValue: genre + name: array + name: array_contains_any + name: where + - args: + - mapValue: + fields: + title: + fieldReferenceValue: title + genre: + fieldReferenceValue: genre + name: select + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: title + name: sort + - description: testArrayConcatLiterals + pipeline: + - Collection: books + - Limit: 1 + - Select: + - AliasedExpr: + - Expr.array_concat: + - Array: [1, 2, 3] + - Array: [4, 5] + - "concatenated" + assert_results: + - concatenated: [1, 2, 3, 4, 5] + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - integerValue: '1' + name: limit + - args: + - mapValue: + fields: + concatenated: + functionValue: + args: + - functionValue: + args: + - integerValue: '1' + - integerValue: '2' + - integerValue: '3' + name: array + - functionValue: + args: + - integerValue: '4' + - integerValue: '5' + name: array + name: array_concat + name: select diff --git a/tests/system/pipeline_e2e/data.yaml b/tests/system/pipeline_e2e/data.yaml new file mode 100644 index 000000000..902f7782d --- /dev/null +++ b/tests/system/pipeline_e2e/data.yaml @@ -0,0 +1,142 @@ +data: + books: + book1: + title: "The Hitchhiker's Guide to the Galaxy" + author: "Douglas Adams" + genre: "Science Fiction" + published: 1979 + rating: 4.2 + tags: + - comedy + - space + - adventure + awards: + hugo: true + nebula: false + book2: + title: "Pride and Prejudice" + author: "Jane Austen" + genre: "Romance" + published: 1813 + rating: 4.5 + tags: + - classic + - social commentary + - love + awards: + none: true + book3: + title: "One Hundred Years of Solitude" + author: "Gabriel García Márquez" + genre: "Magical Realism" + published: 1967 + rating: 4.3 + tags: + - family + - history + - fantasy + awards: + nobel: true + nebula: false + book4: + title: "The Lord of the Rings" + author: "J.R.R. Tolkien" + genre: "Fantasy" + published: 1954 + rating: 4.7 + tags: + - adventure + - magic + - epic + awards: + hugo: false + nebula: false + book5: + title: "The Handmaid's Tale" + author: "Margaret Atwood" + genre: "Dystopian" + published: 1985 + rating: 4.1 + tags: + - feminism + - totalitarianism + - resistance + awards: + arthur c. clarke: true + booker prize: false + book6: + title: "Crime and Punishment" + author: "Fyodor Dostoevsky" + genre: "Psychological Thriller" + published: 1866 + rating: 4.3 + tags: + - philosophy + - crime + - redemption + awards: + none: true + book7: + title: "To Kill a Mockingbird" + author: "Harper Lee" + genre: "Southern Gothic" + published: 1960 + rating: 4.2 + tags: + - racism + - injustice + - coming-of-age + awards: + pulitzer: true + book8: + title: "1984" + author: "George Orwell" + genre: "Dystopian" + published: 1949 + rating: 4.2 + tags: + - surveillance + - totalitarianism + - propaganda + awards: + prometheus: true + book9: + title: "The Great Gatsby" + author: "F. Scott Fitzgerald" + genre: "Modernist" + published: 1925 + rating: 4.0 + tags: + - wealth + - american dream + - love + awards: + none: true + book10: + title: "Dune" + author: "Frank Herbert" + genre: "Science Fiction" + published: 1965 + rating: 4.6 + tags: + - politics + - desert + - ecology + awards: + hugo: true + nebula: true + timestamps: + ts1: + time: "1993-04-28T12:01:00.654321+00:00" + micros: 735998460654321 + millis: 735998460654 + seconds: 735998460 + vectors: + vec1: + embedding: [1.0, 2.0, 3.0] + vec2: + embedding: [4.0, 5.0, 6.0, 7.0] + vec3: + embedding: [5.0, 6.0, 7.0] + vec4: + embedding: [1.0, 2.0, 4.0] \ No newline at end of file diff --git a/tests/system/pipeline_e2e/date_and_time.yaml b/tests/system/pipeline_e2e/date_and_time.yaml new file mode 100644 index 000000000..bbb5f34fe --- /dev/null +++ b/tests/system/pipeline_e2e/date_and_time.yaml @@ -0,0 +1,103 @@ +tests: + - description: testCurrentTimestamp + pipeline: + - Collection: books + - Limit: 1 + - Select: + - AliasedExpr: + - And: + - Expr.greater_than_or_equal: + - CurrentTimestamp: [] + - Expr.unix_seconds_to_timestamp: + - Constant: 1735689600 # 2025-01-01 + - Expr.less_than: + - CurrentTimestamp: [] + - Expr.unix_seconds_to_timestamp: + - Constant: 4892438400 # 2125-01-01 + - "is_between_2025_and_2125" + assert_results: + - is_between_2025_and_2125: true + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - integerValue: '1' + name: limit + - args: + - mapValue: + fields: + is_between_2025_and_2125: + functionValue: + name: and + args: + - functionValue: + name: greater_than_or_equal + args: + - functionValue: + name: current_timestamp + - functionValue: + name: unix_seconds_to_timestamp + args: + - integerValue: '1735689600' + - functionValue: + name: less_than + args: + - functionValue: + name: current_timestamp + - functionValue: + name: unix_seconds_to_timestamp + args: + - integerValue: '4892438400' + name: select + - description: testTimestampFunctions + pipeline: + - Collection: timestamps + - Select: + - AliasedExpr: + - Expr.timestamp_to_unix_micros: + - Field: time + - "micros" + - AliasedExpr: + - Expr.timestamp_to_unix_millis: + - Field: time + - "millis" + - AliasedExpr: + - Expr.timestamp_to_unix_seconds: + - Field: time + - "seconds" + - AliasedExpr: + - Expr.unix_micros_to_timestamp: + - Field: micros + - "from_micros" + - AliasedExpr: + - Expr.unix_millis_to_timestamp: + - Field: millis + - "from_millis" + - AliasedExpr: + - Expr.unix_seconds_to_timestamp: + - Field: seconds + - "from_seconds" + - AliasedExpr: + - Expr.timestamp_add: + - Field: time + - Constant: "day" + - Constant: 1 + - "plus_day" + - AliasedExpr: + - Expr.timestamp_subtract: + - Field: time + - Constant: "hour" + - Constant: 1 + - "minus_hour" + assert_results: + - micros: 735998460654321 + millis: 735998460654 + seconds: 735998460 + from_micros: "1993-04-28T12:01:00.654321+00:00" + from_millis: "1993-04-28T12:01:00.654000+00:00" + from_seconds: "1993-04-28T12:01:00.000000+00:00" + plus_day: "1993-04-29T12:01:00.654321+00:00" + minus_hour: "1993-04-28T11:01:00.654321+00:00" diff --git a/tests/system/pipeline_e2e/general.yaml b/tests/system/pipeline_e2e/general.yaml new file mode 100644 index 000000000..c135853d1 --- /dev/null +++ b/tests/system/pipeline_e2e/general.yaml @@ -0,0 +1,784 @@ +tests: + - description: selectSpecificFields + pipeline: + - Collection: books + - Select: + - title + - author + - Sort: + - Ordering: + - Field: author + - ASCENDING + assert_results: + - title: "The Hitchhiker's Guide to the Galaxy" + author: "Douglas Adams" + - title: "The Great Gatsby" + author: "F. Scott Fitzgerald" + - title: "Dune" + author: "Frank Herbert" + - title: "Crime and Punishment" + author: "Fyodor Dostoevsky" + - title: "One Hundred Years of Solitude" + author: "Gabriel García Márquez" + - title: "1984" + author: "George Orwell" + - title: "To Kill a Mockingbird" + author: "Harper Lee" + - title: "The Lord of the Rings" + author: "J.R.R. Tolkien" + - title: "Pride and Prejudice" + author: "Jane Austen" + - title: "The Handmaid's Tale" + author: "Margaret Atwood" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + author: + fieldReferenceValue: author + title: + fieldReferenceValue: title + name: select + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: author + name: sort + - description: addAndRemoveFields + pipeline: + - Collection: books + - AddFields: + - AliasedExpr: + - Expr.string_concat: + - Field: author + - Constant: _ + - Field: title + - "author_title" + - AliasedExpr: + - Expr.string_concat: + - Field: title + - Constant: _ + - Field: author + - "title_author" + - RemoveFields: + - title_author + - tags + - awards + - rating + - title + - Field: published + - Field: genre + - Field: nestedField # Field does not exist, should be ignored + - Sort: + - Ordering: + - Field: author_title + - ASCENDING + assert_results: + - author: Douglas Adams + author_title: Douglas Adams_The Hitchhiker's Guide to the Galaxy + - author: F. Scott Fitzgerald + author_title: F. Scott Fitzgerald_The Great Gatsby + - author: Frank Herbert + author_title: Frank Herbert_Dune + - author: Fyodor Dostoevsky + author_title: Fyodor Dostoevsky_Crime and Punishment + - author: Gabriel García Márquez + author_title: Gabriel García Márquez_One Hundred Years of Solitude + - author: George Orwell + author_title: George Orwell_1984 + - author: Harper Lee + author_title: Harper Lee_To Kill a Mockingbird + - author: J.R.R. Tolkien + author_title: J.R.R. Tolkien_The Lord of the Rings + - author: Jane Austen + author_title: Jane Austen_Pride and Prejudice + - author: Margaret Atwood + author_title: Margaret Atwood_The Handmaid's Tale + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + author_title: + functionValue: + args: + - fieldReferenceValue: author + - stringValue: _ + - fieldReferenceValue: title + name: string_concat + title_author: + functionValue: + args: + - fieldReferenceValue: title + - stringValue: _ + - fieldReferenceValue: author + name: string_concat + name: add_fields + - args: + - fieldReferenceValue: title_author + - fieldReferenceValue: tags + - fieldReferenceValue: awards + - fieldReferenceValue: rating + - fieldReferenceValue: title + - fieldReferenceValue: published + - fieldReferenceValue: genre + - fieldReferenceValue: nestedField + name: remove_fields + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: author_title + name: sort + + - description: testPipelineWithOffsetAndLimit + pipeline: + - Collection: books + - Sort: + - Ordering: + - Field: author + - ASCENDING + - Offset: 5 + - Limit: 3 + - Select: + - title + - author + assert_results: + - title: "1984" + author: George Orwell + - title: To Kill a Mockingbird + author: Harper Lee + - title: The Lord of the Rings + author: J.R.R. Tolkien + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: author + name: sort + - args: + - integerValue: '5' + name: offset + - args: + - integerValue: '3' + name: limit + - args: + - mapValue: + fields: + author: + fieldReferenceValue: author + title: + fieldReferenceValue: title + name: select + - description: testArithmeticOperations + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: To Kill a Mockingbird + - Select: + - AliasedExpr: + - Expr.add: + - Field: rating + - Constant: 1 + - "ratingPlusOne" + - AliasedExpr: + - Expr.subtract: + - Field: published + - Constant: 1900 + - "yearsSince1900" + - AliasedExpr: + - Expr.multiply: + - Field: rating + - Constant: 10 + - "ratingTimesTen" + - AliasedExpr: + - Expr.divide: + - Field: rating + - Constant: 2 + - "ratingDividedByTwo" + - AliasedExpr: + - Expr.multiply: + - Field: rating + - Constant: 20 + - "ratingTimes20" + - AliasedExpr: + - Expr.add: + - Field: rating + - Constant: 3 + - "ratingPlus3" + - AliasedExpr: + - Expr.mod: + - Field: rating + - Constant: 2 + - "ratingMod2" + assert_results: + - ratingPlusOne: 5.2 + yearsSince1900: 60 + ratingTimesTen: 42.0 + ratingDividedByTwo: 2.1 + ratingTimes20: 84 + ratingPlus3: 7.2 + ratingMod2: 0.20000000000000018 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: To Kill a Mockingbird + name: equal + name: where + - args: + - mapValue: + fields: + ratingDividedByTwo: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '2' + name: divide + ratingPlusOne: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '1' + name: add + ratingTimesTen: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '10' + name: multiply + yearsSince1900: + functionValue: + args: + - fieldReferenceValue: published + - integerValue: '1900' + name: subtract + ratingTimes20: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '20' + name: multiply + ratingPlus3: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '3' + name: add + ratingMod2: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '2' + name: mod + name: select + - description: testSampleLimit + pipeline: + - Collection: books + - Sample: 3 + assert_count: 3 # Results will vary due to randomness + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - integerValue: '3' + - stringValue: documents + name: sample + - description: testSamplePercentage + pipeline: + - Collection: books + - Sample: + - SampleOptions: + - 0.6 + - percent + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - doubleValue: 0.6 + - stringValue: percent + name: sample + - description: testUnion + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: genre + - Constant: Romance + - Union: + - Pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: genre + - Constant: Dystopian + - Select: + - title + - Sort: + - Ordering: + - Field: title + - ASCENDING + assert_results: + - title: "1984" + - title: Pride and Prejudice + - title: "The Handmaid's Tale" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: genre + - stringValue: Romance + name: equal + name: where + - args: + - pipelineValue: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: genre + - stringValue: Dystopian + name: equal + name: where + name: union + - args: + - mapValue: + fields: + title: + fieldReferenceValue: title + name: select + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: title + name: sort + - description: testUnionFullCollection + pipeline: + - Collection: books + - Union: + - Pipeline: + - Collection: books + assert_count: 20 # Results will be duplicated + - description: testDocumentId + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: "The Hitchhiker's Guide to the Galaxy" + - Select: + - AliasedExpr: + - Expr.document_id: + - Field: __name__ + - "doc_id" + assert_results: + - doc_id: "book1" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: "The Hitchhiker's Guide to the Galaxy" + name: equal + name: where + - args: + - mapValue: + fields: + doc_id: + functionValue: + name: document_id + args: + - fieldReferenceValue: __name__ + name: select + - description: testSum + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: genre + - Constant: Science Fiction + - Aggregate: + - AliasedExpr: + - Expr.sum: + - Field: rating + - "total_rating" + assert_results: + - total_rating: 8.8 + + - description: testCollectionId + pipeline: + - Collection: books + - Limit: 1 + - Select: + - AliasedExpr: + - Expr.collection_id: + - Field: __name__ + - "collectionName" + assert_results: + - collectionName: "books" + - description: testCollectionGroup + pipeline: + - CollectionGroup: books + - Select: + - title + - Distinct: + - title + - Sort: + - Ordering: + - Field: title + - ASCENDING + assert_results: + - title: "1984" + - title: "Crime and Punishment" + - title: "Dune" + - title: "One Hundred Years of Solitude" + - title: "Pride and Prejudice" + - title: "The Great Gatsby" + - title: "The Handmaid's Tale" + - title: "The Hitchhiker's Guide to the Galaxy" + - title: "The Lord of the Rings" + - title: "To Kill a Mockingbird" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: '' + - stringValue: books + name: collection_group + - args: + - mapValue: + fields: + title: + fieldReferenceValue: title + name: select + - args: + - mapValue: + fields: + title: + fieldReferenceValue: title + name: distinct + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: title + name: sort + + - description: testDistinct + pipeline: + - Collection: books + - Distinct: + - genre + - Sort: + - Ordering: + - Field: genre + - ASCENDING + assert_results: + - genre: Dystopian + - genre: Fantasy + - genre: Magical Realism + - genre: Modernist + - genre: Psychological Thriller + - genre: Romance + - genre: Science Fiction + - genre: Southern Gothic + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + genre: + fieldReferenceValue: genre + name: distinct + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: genre + name: sort + + - description: testDocuments + pipeline: + - Documents: + - /books/book1 + - /books/book10 + - Select: + - title + - Sort: + - Ordering: + - Field: title + - ASCENDING + assert_results: + - title: "Dune" + - title: "The Hitchhiker's Guide to the Galaxy" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books/book1 + - referenceValue: /books/book10 + name: documents + - args: + - mapValue: + fields: + title: + fieldReferenceValue: title + name: select + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: title + name: sort + + - description: testDatabase + pipeline: + - Database + - Select: + - title + - Distinct: + - title + - Aggregate: + - AliasedExpr: + - Count: [] + - count + - Select: + - AliasedExpr: + - Conditional: + - Expr.greater_than_or_equal: + - Field: count + - Constant: 10 + - Constant: True + - Constant: False + - result + assert_results: + - result: True + assert_proto: + pipeline: + stages: + - name: database + - args: + - mapValue: + fields: + title: + fieldReferenceValue: title + name: select + - args: + - mapValue: + fields: + title: + fieldReferenceValue: title + name: distinct + - args: + - mapValue: + fields: + count: + functionValue: + name: count + - mapValue: {} + name: aggregate + - name: select + args: + - mapValue: + fields: + result: + functionValue: + args: + - functionValue: + args: + - fieldReferenceValue: count + - integerValue: '10' + name: greater_than_or_equal + - booleanValue: true + - booleanValue: false + name: conditional + - description: testGenericStage + pipeline: + - GenericStage: + - "collection" + - Value: + reference_value: "/books" + - GenericStage: + - "where" + - Expr.equal: + - Field: title + - Constant: The Hitchhiker's Guide to the Galaxy + - GenericStage: + - "select" + - Value: + map_value: + fields: + author: + field_reference_value: author + assert_results: + - author: Douglas Adams + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: The Hitchhiker's Guide to the Galaxy + name: equal + name: where + - args: + - mapValue: + fields: + author: + fieldReferenceValue: author + name: select + - description: testUnnest + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: The Hitchhiker's Guide to the Galaxy + - Unnest: + - tags + - tags_alias + - Select: tags_alias + assert_results: + - tags_alias: comedy + - tags_alias: space + - tags_alias: adventure + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: The Hitchhiker's Guide to the Galaxy + name: equal + name: where + - args: + - fieldReferenceValue: tags + - fieldReferenceValue: tags_alias + name: unnest + - args: + - mapValue: + fields: + tags_alias: + fieldReferenceValue: tags_alias + name: select + - description: testUnnestWithOptions + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: The Hitchhiker's Guide to the Galaxy + - Unnest: + field: tags + alias: tags_alias + options: + UnnestOptions: + - index + - Select: + - tags_alias + - index + assert_results: + - tags_alias: comedy + index: 0 + - tags_alias: space + index: 1 + - tags_alias: adventure + index: 2 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: The Hitchhiker's Guide to the Galaxy + name: equal + name: where + - args: + - fieldReferenceValue: tags + - fieldReferenceValue: tags_alias + name: unnest + options: + index_field: + fieldReferenceValue: index + - args: + - mapValue: + fields: + tags_alias: + fieldReferenceValue: tags_alias + index: + fieldReferenceValue: index + name: select \ No newline at end of file diff --git a/tests/system/pipeline_e2e/logical.yaml b/tests/system/pipeline_e2e/logical.yaml new file mode 100644 index 000000000..203be290d --- /dev/null +++ b/tests/system/pipeline_e2e/logical.yaml @@ -0,0 +1,673 @@ +tests: + - description: whereByMultipleConditions + pipeline: + - Collection: books + - Where: + - And: + - Expr.greater_than: + - Field: rating + - Constant: 4.5 + - Expr.equal: + - Field: genre + - Constant: Science Fiction + assert_results: + - title: Dune + author: Frank Herbert + genre: Science Fiction + published: 1965 + rating: 4.6 + tags: + - politics + - desert + - ecology + awards: + hugo: true + nebula: true + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - functionValue: + args: + - fieldReferenceValue: rating + - doubleValue: 4.5 + name: greater_than + - functionValue: + args: + - fieldReferenceValue: genre + - stringValue: Science Fiction + name: equal + name: and + name: where + - description: whereByOrCondition + pipeline: + - Collection: books + - Where: + - Or: + - Expr.equal: + - Field: genre + - Constant: Romance + - Expr.equal: + - Field: genre + - Constant: Dystopian + - Select: + - title + - Sort: + - Ordering: + - Field: title + - ASCENDING + assert_results: + - title: "1984" + - title: Pride and Prejudice + - title: The Handmaid's Tale + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - functionValue: + args: + - fieldReferenceValue: genre + - stringValue: Romance + name: equal + - functionValue: + args: + - fieldReferenceValue: genre + - stringValue: Dystopian + name: equal + name: or + name: where + - args: + - mapValue: + fields: + title: + fieldReferenceValue: title + name: select + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: title + name: sort + - description: testComparisonOperators + pipeline: + - Collection: books + - Where: + - And: + - Expr.greater_than: + - Field: rating + - Constant: 4.2 + - Expr.less_than_or_equal: + - Field: rating + - Constant: 4.5 + - Expr.not_equal: + - Field: genre + - Constant: Science Fiction + - Select: + - rating + - title + - Sort: + - Ordering: + - title + - ASCENDING + assert_results: + - rating: 4.3 + title: Crime and Punishment + - rating: 4.3 + title: One Hundred Years of Solitude + - rating: 4.5 + title: Pride and Prejudice + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - functionValue: + args: + - fieldReferenceValue: rating + - doubleValue: 4.2 + name: greater_than + - functionValue: + args: + - fieldReferenceValue: rating + - doubleValue: 4.5 + name: less_than_or_equal + - functionValue: + args: + - fieldReferenceValue: genre + - stringValue: Science Fiction + name: not_equal + name: and + name: where + - args: + - mapValue: + fields: + rating: + fieldReferenceValue: rating + title: + fieldReferenceValue: title + name: select + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: title + name: sort + - description: testLogicalOperators + pipeline: + - Collection: books + - Where: + - Or: + - And: + - Expr.greater_than: + - Field: rating + - Constant: 4.5 + - Expr.equal: + - Field: genre + - Constant: Science Fiction + - Expr.less_than: + - Field: published + - Constant: 1900 + - Select: + - title + - Sort: + - Ordering: + - Field: title + - ASCENDING + assert_results: + - title: Crime and Punishment + - title: Dune + - title: Pride and Prejudice + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - functionValue: + args: + - functionValue: + args: + - fieldReferenceValue: rating + - doubleValue: 4.5 + name: greater_than + - functionValue: + args: + - fieldReferenceValue: genre + - stringValue: Science Fiction + name: equal + name: and + - functionValue: + args: + - fieldReferenceValue: published + - integerValue: '1900' + name: less_than + name: or + name: where + - args: + - mapValue: + fields: + title: + fieldReferenceValue: title + name: select + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: title + name: sort + - description: testChecks + pipeline: + - Collection: books + - Where: + - Not: + - Expr.is_nan: + - Field: rating + - Select: + - AliasedExpr: + - Not: + - Expr.is_nan: + - Field: rating + - "ratingIsNotNaN" + - Limit: 1 + assert_results: + - ratingIsNotNaN: true + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - functionValue: + args: + - fieldReferenceValue: rating + name: is_nan + name: not + name: where + - args: + - mapValue: + fields: + ratingIsNotNaN: + functionValue: + args: + - functionValue: + args: + - fieldReferenceValue: rating + name: is_nan + name: not + name: select + - args: + - integerValue: '1' + name: limit + - description: testIsNotNull + pipeline: + - Collection: books + - Where: + - Expr.is_not_null: + - Field: rating + assert_count: 10 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: rating + name: is_not_null + name: where + - description: testIsNotNaN + pipeline: + - Collection: books + - Where: + - Expr.is_not_nan: + - Field: rating + assert_count: 10 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: rating + name: is_not_nan + name: where + - description: testIsAbsent + pipeline: + - Collection: books + - Where: + - Expr.is_absent: + - Field: awards.pulitzer + assert_count: 9 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: awards.pulitzer + name: is_absent + name: where + - description: testIfAbsent + pipeline: + - Collection: books + - Select: + - AliasedExpr: + - Expr.if_absent: + - Field: awards.pulitzer + - Constant: false + - "pulitzer_award" + - title + - Where: + - Expr.equal: + - Field: pulitzer_award + - Constant: true + assert_results: + - pulitzer_award: true + title: To Kill a Mockingbird + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + pulitzer_award: + functionValue: + name: if_absent + args: + - fieldReferenceValue: awards.pulitzer + - booleanValue: false + title: + fieldReferenceValue: title + name: select + - args: + - functionValue: + args: + - fieldReferenceValue: pulitzer_award + - booleanValue: true + name: equal + name: where + - description: testIsError + pipeline: + - Collection: books + - Select: + - AliasedExpr: + - Expr.is_error: + - Expr.divide: + - Field: rating + - Constant: "string" + - "is_error_result" + - Limit: 1 + assert_results: + - is_error_result: true + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + is_error_result: + functionValue: + name: is_error + args: + - functionValue: + name: divide + args: + - fieldReferenceValue: rating + - stringValue: "string" + name: select + - args: + - integerValue: '1' + name: limit + - description: testIfError + pipeline: + - Collection: books + - Select: + - AliasedExpr: + - Expr.if_error: + - Expr.divide: + - Field: rating + - Field: genre + - Constant: "An error occurred" + - "if_error_result" + - Limit: 1 + assert_results: + - if_error_result: "An error occurred" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + if_error_result: + functionValue: + name: if_error + args: + - functionValue: + name: divide + args: + - fieldReferenceValue: rating + - fieldReferenceValue: genre + - stringValue: "An error occurred" + name: select + - args: + - integerValue: '1' + name: limit + - description: testLogicalMinMax + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: author + - Constant: Douglas Adams + - Select: + - AliasedExpr: + - Expr.logical_maximum: + - Field: rating + - Constant: 4.5 + - "max_rating" + - AliasedExpr: + - Expr.logical_minimum: + - Field: published + - Constant: 1900 + - "min_published" + assert_results: + - max_rating: 4.5 + min_published: 1900 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: author + - stringValue: Douglas Adams + name: equal + name: where + - args: + - mapValue: + fields: + min_published: + functionValue: + args: + - fieldReferenceValue: published + - integerValue: '1900' + name: minimum + max_rating: + functionValue: + args: + - fieldReferenceValue: rating + - doubleValue: 4.5 + name: maximum + name: select + - description: testGreaterThanOrEqual + pipeline: + - Collection: books + - Where: + - Expr.greater_than_or_equal: + - Field: rating + - Constant: 4.6 + - Select: + - title + - rating + - Sort: + - Ordering: + - Field: rating + - ASCENDING + assert_results: + - title: Dune + rating: 4.6 + - title: The Lord of the Rings + rating: 4.7 + - description: testInAndNotIn + pipeline: + - Collection: books + - Where: + - And: + - Expr.equal_any: + - Field: genre + - - Constant: Romance + - Constant: Dystopian + - Expr.not_equal_any: + - Field: author + - - Constant: "George Orwell" + assert_results: + - title: "Pride and Prejudice" + author: "Jane Austen" + genre: "Romance" + published: 1813 + rating: 4.5 + tags: + - classic + - social commentary + - love + awards: + none: true + - title: "The Handmaid's Tale" + author: "Margaret Atwood" + genre: "Dystopian" + published: 1985 + rating: 4.1 + tags: + - feminism + - totalitarianism + - resistance + awards: + "arthur c. clarke": true + "booker prize": false + - description: testExists + pipeline: + - Collection: books + - Where: + - And: + - Expr.exists: + - Field: awards.pulitzer + - Expr.equal: + - Field: awards.pulitzer + - Constant: true + - Select: + - title + assert_results: + - title: To Kill a Mockingbird + - description: testXor + pipeline: + - Collection: books + - Where: + - Xor: + - - Expr.equal: + - Field: genre + - Constant: Romance + - Expr.greater_than: + - Field: published + - Constant: 1980 + - Select: + - title + - genre + - published + - Sort: + - Ordering: + - Field: title + - ASCENDING + assert_results: + - title: "Pride and Prejudice" + genre: "Romance" + published: 1813 + - title: "The Handmaid's Tale" + genre: "Dystopian" + published: 1985 + - description: testConditional + pipeline: + - Collection: books + - Select: + - title + - AliasedExpr: + - Conditional: + - Expr.greater_than: + - Field: published + - Constant: 1950 + - Constant: "Modern" + - Constant: "Classic" + - "era" + - Sort: + - Ordering: + - Field: title + - ASCENDING + - Limit: 4 + assert_results: + - title: "1984" + era: "Classic" + - title: "Crime and Punishment" + era: "Classic" + - title: "Dune" + era: "Modern" + - title: "One Hundred Years of Solitude" + era: "Modern" + - description: testFieldToFieldComparison + pipeline: + - Collection: books + - Where: + - Expr.greater_than: + - Field: published + - Field: rating + - Select: + - title + assert_count: 10 # All books were published after year 4.7 + - description: testExistsNegative + pipeline: + - Collection: books + - Where: + - Expr.exists: + - Field: non_existent_field + assert_count: 0 + - description: testConditionalWithFields + pipeline: + - Collection: books + - Where: + - Expr.equal_any: + - Field: title + - - Constant: "Dune" + - Constant: "1984" + - Select: + - title + - AliasedExpr: + - Conditional: + - Expr.greater_than: + - Field: published + - Constant: 1950 + - Field: author + - Field: genre + - "conditional_field" + - Sort: + - Ordering: + - Field: title + - ASCENDING + assert_results: + - title: "1984" + conditional_field: "Dystopian" + - title: "Dune" + conditional_field: "Frank Herbert" diff --git a/tests/system/pipeline_e2e/map.yaml b/tests/system/pipeline_e2e/map.yaml new file mode 100644 index 000000000..638fe0798 --- /dev/null +++ b/tests/system/pipeline_e2e/map.yaml @@ -0,0 +1,269 @@ +tests: + - description: testMapGet + pipeline: + - Collection: books + - Sort: + - Ordering: + - Field: published + - DESCENDING + - Select: + - AliasedExpr: + - Expr.map_get: + - Field: awards + - hugo + - "hugoAward" + - Field: title + - Where: + - Expr.equal: + - Field: hugoAward + - Constant: true + assert_results: + - hugoAward: true + title: The Hitchhiker's Guide to the Galaxy + - hugoAward: true + title: Dune + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + direction: + stringValue: descending + expression: + fieldReferenceValue: published + name: sort + - args: + - mapValue: + fields: + hugoAward: + functionValue: + args: + - fieldReferenceValue: awards + - stringValue: hugo + name: map_get + title: + fieldReferenceValue: title + name: select + - args: + - functionValue: + args: + - fieldReferenceValue: hugoAward + - booleanValue: true + name: equal + name: where + - description: testMapGetWithField + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: "Dune" + - AddFields: + - AliasedExpr: + - Constant: "hugo" + - "award_name" + - Select: + - AliasedExpr: + - Expr.map_get: + - Field: awards + - Field: award_name + - "hugoAward" + - Field: title + assert_results: + - hugoAward: true + title: Dune + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: "Dune" + name: equal + name: where + - args: + - mapValue: + fields: + award_name: + stringValue: "hugo" + name: add_fields + - args: + - mapValue: + fields: + hugoAward: + functionValue: + name: map_get + args: + - fieldReferenceValue: awards + - fieldReferenceValue: award_name + title: + fieldReferenceValue: title + name: select + - description: testMapRemove + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: "Dune" + - Select: + - AliasedExpr: + - Expr.map_remove: + - Field: awards + - "nebula" + - "awards_removed" + assert_results: + - awards_removed: + hugo: true + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: "Dune" + name: equal + name: where + - args: + - mapValue: + fields: + awards_removed: + functionValue: + name: map_remove + args: + - fieldReferenceValue: awards + - stringValue: "nebula" + name: select + - description: testMapMerge + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: "Dune" + - Select: + - AliasedExpr: + - Expr.map_merge: + - Field: awards + - Map: + elements: {"new_award": true, "hugo": false} + - Map: + elements: {"another_award": "yes"} + - "awards_merged" + assert_results: + - awards_merged: + hugo: false + nebula: true + new_award: true + another_award: "yes" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: "Dune" + name: equal + name: where + - args: + - mapValue: + fields: + awards_merged: + functionValue: + name: map_merge + args: + - fieldReferenceValue: awards + - functionValue: + name: map + args: + - stringValue: "new_award" + - booleanValue: true + - stringValue: "hugo" + - booleanValue: false + - functionValue: + name: map + args: + - stringValue: "another_award" + - stringValue: "yes" + name: select + - description: testNestedFields + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: awards.hugo + - Constant: true + - Sort: + - Ordering: + - Field: title + - DESCENDING + - Select: + - title + - Field: awards.hugo + assert_results: + - title: The Hitchhiker's Guide to the Galaxy + awards.hugo: true + - title: Dune + awards.hugo: true + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: awards.hugo + - booleanValue: true + name: equal + name: where + - args: + - mapValue: + fields: + direction: + stringValue: descending + expression: + fieldReferenceValue: title + name: sort + - args: + - mapValue: + fields: + awards.hugo: + fieldReferenceValue: awards.hugo + title: + fieldReferenceValue: title + name: select + - description: testMapMergeLiterals + pipeline: + - Collection: books + - Limit: 1 + - Select: + - AliasedExpr: + - Expr.map_merge: + - Map: + elements: {"a": "orig", "b": "orig"} + - Map: + elements: {"b": "new", "c": "new"} + - "merged" + assert_results: + - merged: + a: "orig" + b: "new" + c: "new" diff --git a/tests/system/pipeline_e2e/math.yaml b/tests/system/pipeline_e2e/math.yaml new file mode 100644 index 000000000..a5a47d4c0 --- /dev/null +++ b/tests/system/pipeline_e2e/math.yaml @@ -0,0 +1,309 @@ +tests: + - description: testFieldToFieldArithmetic + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: "Dune" + - Select: + - AliasedExpr: + - Expr.add: + - Field: published + - Field: rating + - "pub_plus_rating" + assert_results: + - pub_plus_rating: 1969.6 + - description: testMathExpressions + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: To Kill a Mockingbird + - Select: + - AliasedExpr: + - Expr.abs: + - Field: rating + - "abs_rating" + - AliasedExpr: + - Expr.ceil: + - Field: rating + - "ceil_rating" + - AliasedExpr: + - Expr.exp: + - Field: rating + - "exp_rating" + - AliasedExpr: + - Expr.floor: + - Field: rating + - "floor_rating" + - AliasedExpr: + - Expr.ln: + - Field: rating + - "ln_rating" + - AliasedExpr: + - Expr.log10: + - Field: rating + - "log_rating_base10" + - AliasedExpr: + - Expr.log: + - Field: rating + - Constant: 2 + - "log_rating_base2" + - AliasedExpr: + - Expr.pow: + - Field: rating + - Constant: 2 + - "pow_rating" + - AliasedExpr: + - Expr.sqrt: + - Field: rating + - "sqrt_rating" + assert_results_approximate: + - abs_rating: 4.2 + ceil_rating: 5.0 + exp_rating: 66.686331 + floor_rating: 4.0 + ln_rating: 1.4350845 + log_rating_base10: 0.623249 + log_rating_base2: 2.0704 + pow_rating: 17.64 + sqrt_rating: 2.049390 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: To Kill a Mockingbird + name: equal + name: where + - args: + - mapValue: + fields: + abs_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: abs + ceil_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: ceil + exp_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: exp + floor_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: floor + ln_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: ln + log_rating_base10: + functionValue: + args: + - fieldReferenceValue: rating + name: log10 + log_rating_base2: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '2' + name: log + pow_rating: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '2' + name: pow + sqrt_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: sqrt + name: select + - description: testRoundExpressions + pipeline: + - Collection: books + - Where: + - Expr.equal_any: + - Field: title + - - Constant: "To Kill a Mockingbird" # rating 4.2 + - Constant: "Pride and Prejudice" # rating 4.5 + - Constant: "The Lord of the Rings" # rating 4.7 + - Select: + - title + - AliasedExpr: + - Expr.round: + - Field: rating + - "round_rating" + - Sort: + - Ordering: + - Field: title + - ASCENDING + assert_results: + - title: "Pride and Prejudice" + round_rating: 5.0 + - title: "The Lord of the Rings" + round_rating: 5.0 + - title: "To Kill a Mockingbird" + round_rating: 4.0 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - functionValue: + args: + - stringValue: "To Kill a Mockingbird" + - stringValue: "Pride and Prejudice" + - stringValue: "The Lord of the Rings" + name: array + name: equal_any + name: where + - args: + - mapValue: + fields: + title: + fieldReferenceValue: title + round_rating: + functionValue: + args: + - fieldReferenceValue: rating + name: round + name: select + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: title + name: sort + - description: testArithmeticOperations + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: To Kill a Mockingbird + - Select: + - AliasedExpr: + - Expr.add: + - Field: rating + - Constant: 1 + - "ratingPlusOne" + - AliasedExpr: + - Expr.subtract: + - Field: published + - Constant: 1900 + - "yearsSince1900" + - AliasedExpr: + - Expr.multiply: + - Field: rating + - Constant: 10 + - "ratingTimesTen" + - AliasedExpr: + - Expr.divide: + - Field: rating + - Constant: 2 + - "ratingDividedByTwo" + - AliasedExpr: + - Expr.multiply: + - Field: rating + - Constant: 20 + - "ratingTimes20" + - AliasedExpr: + - Expr.add: + - Field: rating + - Constant: 3 + - "ratingPlus3" + - AliasedExpr: + - Expr.mod: + - Field: rating + - Constant: 2 + - "ratingMod2" + assert_results: + - ratingPlusOne: 5.2 + yearsSince1900: 60 + ratingTimesTen: 42.0 + ratingDividedByTwo: 2.1 + ratingTimes20: 84 + ratingPlus3: 7.2 + ratingMod2: 0.20000000000000018 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: To Kill a Mockingbird + name: equal + name: where + - args: + - mapValue: + fields: + ratingDividedByTwo: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '2' + name: divide + ratingPlusOne: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '1' + name: add + ratingTimesTen: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '10' + name: multiply + yearsSince1900: + functionValue: + args: + - fieldReferenceValue: published + - integerValue: '1900' + name: subtract + ratingTimes20: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '20' + name: multiply + ratingPlus3: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '3' + name: add + ratingMod2: + functionValue: + args: + - fieldReferenceValue: rating + - integerValue: '2' + name: mod + name: select \ No newline at end of file diff --git a/tests/system/pipeline_e2e/string.yaml b/tests/system/pipeline_e2e/string.yaml new file mode 100644 index 000000000..b1e3a0b64 --- /dev/null +++ b/tests/system/pipeline_e2e/string.yaml @@ -0,0 +1,654 @@ +tests: + - description: testStringConcat + pipeline: + - Collection: books + - Sort: + - Ordering: + - Field: author + - ASCENDING + - Select: + - AliasedExpr: + - Expr.string_concat: + - Field: author + - Constant: " - " + - Field: title + - "bookInfo" + - Limit: 1 + assert_results: + - bookInfo: Douglas Adams - The Hitchhiker's Guide to the Galaxy + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: author + name: sort + - args: + - mapValue: + fields: + bookInfo: + functionValue: + args: + - fieldReferenceValue: author + - stringValue: ' - ' + - fieldReferenceValue: title + name: string_concat + name: select + - args: + - integerValue: '1' + name: limit + - description: testStartsWith + pipeline: + - Collection: books + - Where: + - Expr.starts_with: + - Field: title + - Constant: The + - Select: + - title + - Sort: + - Ordering: + - Field: title + - ASCENDING + assert_results: + - title: The Great Gatsby + - title: The Handmaid's Tale + - title: The Hitchhiker's Guide to the Galaxy + - title: The Lord of the Rings + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: The + name: starts_with + name: where + - args: + - mapValue: + fields: + title: + fieldReferenceValue: title + name: select + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: title + name: sort + - description: testEndsWith + pipeline: + - Collection: books + - Where: + - Expr.ends_with: + - Field: title + - Constant: y + - Select: + - title + - Sort: + - Ordering: + - Field: title + - DESCENDING + assert_results: + - title: The Hitchhiker's Guide to the Galaxy + - title: The Great Gatsby + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: y + name: ends_with + name: where + - args: + - mapValue: + fields: + title: + fieldReferenceValue: title + name: select + - args: + - mapValue: + fields: + direction: + stringValue: descending + expression: + fieldReferenceValue: title + name: sort + - description: testConcat + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: "The Hitchhiker's Guide to the Galaxy" + - Select: + - AliasedExpr: + - Expr.concat: + - Field: author + - Constant: ": " + - Field: title + - "author_title" + - AliasedExpr: + - Expr.concat: + - Field: tags + - - Constant: "new_tag" + - "concatenatedTags" + assert_results: + - author_title: "Douglas Adams: The Hitchhiker's Guide to the Galaxy" + concatenatedTags: + - comedy + - space + - adventure + - new_tag + - description: testLength + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: title + - Constant: "The Hitchhiker's Guide to the Galaxy" + - Select: + - AliasedExpr: + - Expr.length: + - Field: title + - "titleLength" + - AliasedExpr: + - Expr.length: + - Field: tags + - "tagsLength" + - AliasedExpr: + - Expr.length: + - Field: awards + - "awardsLength" + assert_results: + - titleLength: 36 + tagsLength: 3 + awardsLength: 2 + - description: testCharLength + pipeline: + - Collection: books + - Select: + - AliasedExpr: + - Expr.char_length: + - Field: title + - "titleLength" + - title + - Where: + - Expr.greater_than: + - Field: titleLength + - Constant: 20 + - Sort: + - Ordering: + - Field: title + - ASCENDING + assert_results: + - titleLength: 29 + title: One Hundred Years of Solitude + - titleLength: 36 + title: The Hitchhiker's Guide to the Galaxy + - titleLength: 21 + title: The Lord of the Rings + - titleLength: 21 + title: To Kill a Mockingbird + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - mapValue: + fields: + title: + fieldReferenceValue: title + titleLength: + functionValue: + args: + - fieldReferenceValue: title + name: char_length + name: select + - args: + - functionValue: + args: + - fieldReferenceValue: titleLength + - integerValue: '20' + name: greater_than + name: where + - args: + - mapValue: + fields: + direction: + stringValue: ascending + expression: + fieldReferenceValue: title + name: sort + - description: CharLength + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: author + - Constant: "Douglas Adams" + - Select: + - AliasedExpr: + - Expr.char_length: + - Field: title + - "title_length" + assert_results: + - title_length: 36 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: author + - stringValue: Douglas Adams + name: equal + name: where + - args: + - mapValue: + fields: + title_length: + functionValue: + args: + - fieldReferenceValue: title + name: char_length + name: select + - description: ByteLength + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: author + - Constant: Douglas Adams + - Select: + - AliasedExpr: + - Expr.byte_length: + - Expr.string_concat: + - Field: title + - Constant: _银河系漫游指南 + - "title_byte_length" + assert_results: + - title_byte_length: 58 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: author + - stringValue: Douglas Adams + name: equal + name: where + - args: + - mapValue: + fields: + title_byte_length: + functionValue: + args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: "_\u94F6\u6CB3\u7CFB\u6F2B\u6E38\u6307\u5357" + name: string_concat + name: byte_length + name: select + - description: testLike + pipeline: + - Collection: books + - Where: + - Expr.like: + - Field: title + - Constant: "%Guide%" + - Select: + - title + assert_results: + - title: The Hitchhiker's Guide to the Galaxy + - description: testRegexContains + # Find titles that contain either "the" or "of" (case-insensitive) + pipeline: + - Collection: books + - Where: + - Expr.regex_contains: + - Field: title + - Constant: "(?i)(the|of)" + assert_count: 5 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: "(?i)(the|of)" + name: regex_contains + name: where + - description: testRegexMatches + # Find titles that contain either "the" or "of" (case-insensitive) + pipeline: + - Collection: books + - Where: + - Expr.regex_match: + - Field: title + - Constant: ".*(?i)(the|of).*" + assert_count: 5 + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: title + - stringValue: ".*(?i)(the|of).*" + name: regex_match + name: where + - description: testStringContains + pipeline: + - Collection: books + - Where: + - Expr.string_contains: + - Field: title + - Constant: "Hitchhiker's" + - Select: + - title + assert_results: + - title: "The Hitchhiker's Guide to the Galaxy" + - description: ToLower + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: author + - Constant: "Douglas Adams" + - Select: + - AliasedExpr: + - Expr.to_lower: + - Field: title + - "lower_title" + assert_results: + - lower_title: "the hitchhiker's guide to the galaxy" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: author + - stringValue: Douglas Adams + name: equal + name: where + - args: + - mapValue: + fields: + lower_title: + functionValue: + args: + - fieldReferenceValue: title + name: to_lower + name: select + - description: ToUpper + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: author + - Constant: "Douglas Adams" + - Select: + - AliasedExpr: + - Expr.to_upper: + - Field: title + - "upper_title" + assert_results: + - upper_title: "THE HITCHHIKER'S GUIDE TO THE GALAXY" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: author + - stringValue: Douglas Adams + name: equal + name: where + - args: + - mapValue: + fields: + upper_title: + functionValue: + args: + - fieldReferenceValue: title + name: to_upper + name: select + - description: Trim + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: author + - Constant: "Douglas Adams" + - Select: + - AliasedExpr: + - Expr.trim: + - Expr.string_concat: + - Constant: " " + - Field: title + - Constant: " " + - "trimmed_title" + assert_results: + - trimmed_title: "The Hitchhiker's Guide to the Galaxy" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: author + - stringValue: Douglas Adams + name: equal + name: where + - args: + - mapValue: + fields: + trimmed_title: + functionValue: + args: + - functionValue: + args: + - stringValue: " " + - fieldReferenceValue: title + - stringValue: " " + name: string_concat + name: trim + name: select + - description: StringReverse + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: author + - Constant: "Jane Austen" + - Select: + - AliasedExpr: + - Expr.string_reverse: + - Field: title + - "reversed_title" + assert_results: + - reversed_title: "ecidujerP dna edirP" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: author + - stringValue: "Jane Austen" + name: equal + name: where + - args: + - mapValue: + fields: + reversed_title: + functionValue: + args: + - fieldReferenceValue: title + name: string_reverse + name: select + - description: Substring + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: author + - Constant: "Douglas Adams" + - Select: + - AliasedExpr: + - Expr.substring: + - Field: title + - Constant: 4 + - Constant: 11 + - "substring_title" + assert_results: + - substring_title: "Hitchhiker'" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: author + - stringValue: "Douglas Adams" + name: equal + name: where + - args: + - mapValue: + fields: + substring_title: + functionValue: + args: + - fieldReferenceValue: title + - integerValue: '4' + - integerValue: '11' + name: substring + name: select + - description: Substring without length + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: author + - Constant: "Fyodor Dostoevsky" + - Select: + - AliasedExpr: + - Expr.substring: + - Field: title + - Constant: 10 + - "substring_title" + assert_results: + - substring_title: "Punishment" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: author + - stringValue: "Fyodor Dostoevsky" + name: equal + name: where + - args: + - mapValue: + fields: + substring_title: + functionValue: + args: + - fieldReferenceValue: title + - integerValue: '10' + name: substring + name: select + - description: Join + pipeline: + - Collection: books + - Where: + - Expr.equal: + - Field: author + - Constant: "Douglas Adams" + - Select: + - AliasedExpr: + - Expr.join: + - Field: tags + - Constant: ", " + - "joined_tags" + assert_results: + - joined_tags: "comedy, space, adventure" + assert_proto: + pipeline: + stages: + - args: + - referenceValue: /books + name: collection + - args: + - functionValue: + args: + - fieldReferenceValue: author + - stringValue: "Douglas Adams" + name: equal + name: where + - args: + - mapValue: + fields: + joined_tags: + functionValue: + args: + - fieldReferenceValue: tags + - stringValue: ", " + name: join + name: select diff --git a/tests/system/pipeline_e2e/vector.yaml b/tests/system/pipeline_e2e/vector.yaml new file mode 100644 index 000000000..15fc9bcaa --- /dev/null +++ b/tests/system/pipeline_e2e/vector.yaml @@ -0,0 +1,160 @@ +tests: + - description: testVectorLength + pipeline: + - Collection: vectors + - Select: + - AliasedExpr: + - Expr.vector_length: + - Field: embedding + - "embedding_length" + - Sort: + - Ordering: + - Field: embedding_length + - ASCENDING + assert_results: + - embedding_length: 3 + - embedding_length: 3 + - embedding_length: 3 + - embedding_length: 4 + - description: testFindNearestEuclidean + pipeline: + - Collection: vectors + - FindNearest: + field: embedding + vector: [1.0, 2.0, 3.0] + distance_measure: EUCLIDEAN + options: + FindNearestOptions: + limit: 2 + distance_field: + Field: distance + - Select: + - distance + assert_results: + - distance: 0.0 + - distance: 1.0 + assert_proto: + pipeline: + stages: + - name: collection + args: + - referenceValue: /vectors + - name: find_nearest + args: + - fieldReferenceValue: embedding + - mapValue: + fields: + __type__: + stringValue: __vector__ + value: + arrayValue: + values: + - doubleValue: 1.0 + - doubleValue: 2.0 + - doubleValue: 3.0 + - stringValue: euclidean + options: + limit: + integerValue: '2' + distance_field: + fieldReferenceValue: distance + - name: select + args: + - mapValue: + fields: + distance: + fieldReferenceValue: distance + - description: testFindNearestDotProduct + pipeline: + - Collection: vectors + - FindNearest: + field: embedding + vector: [1.0, 2.0, 3.0] + distance_measure: DOT_PRODUCT + options: + FindNearestOptions: + limit: 3 + distance_field: + Field: distance + - Select: + - distance + assert_results: + - distance: 38.0 + - distance: 17.0 + - distance: 14.0 + assert_proto: + pipeline: + stages: + - name: collection + args: + - referenceValue: /vectors + - name: find_nearest + args: + - fieldReferenceValue: embedding + - mapValue: + fields: + __type__: + stringValue: __vector__ + value: + arrayValue: + values: + - doubleValue: 1.0 + - doubleValue: 2.0 + - doubleValue: 3.0 + - stringValue: dot_product + options: + limit: + integerValue: '3' + distance_field: + fieldReferenceValue: distance + - name: select + args: + - mapValue: + fields: + distance: + fieldReferenceValue: distance + - description: testDotProductWithConstant + pipeline: + - Collection: vectors + - Where: + - Expr.equal: + - Field: embedding + - Vector: [1.0, 2.0, 3.0] + - Select: + - AliasedExpr: + - Expr.dot_product: + - Field: embedding + - Vector: [1.0, 1.0, 1.0] + - "dot_product_result" + assert_results: + - dot_product_result: 6.0 + - description: testEuclideanDistanceWithConstant + pipeline: + - Collection: vectors + - Where: + - Expr.equal: + - Field: embedding + - Vector: [1.0, 2.0, 3.0] + - Select: + - AliasedExpr: + - Expr.euclidean_distance: + - Field: embedding + - Vector: [1.0, 2.0, 3.0] + - "euclidean_distance_result" + assert_results: + - euclidean_distance_result: 0.0 + - description: testCosineDistanceWithConstant + pipeline: + - Collection: vectors + - Where: + - Expr.equal: + - Field: embedding + - Vector: [1.0, 2.0, 3.0] + - Select: + - AliasedExpr: + - Expr.cosine_distance: + - Field: embedding + - Vector: [1.0, 2.0, 3.0] + - "cosine_distance_result" + assert_results: + - cosine_distance_result: 0.0 diff --git a/tests/system/test__helpers.py b/tests/system/test__helpers.py index 5a93a869e..74b12b7c3 100644 --- a/tests/system/test__helpers.py +++ b/tests/system/test__helpers.py @@ -18,6 +18,7 @@ FIRESTORE_ENTERPRISE_DB = os.environ.get("ENTERPRISE_DATABASE", "enterprise-db") # run all tests against default database, and a named database -# TODO: add enterprise mode when GA (RunQuery not currently supported) TEST_DATABASES = [None, FIRESTORE_OTHER_DB] TEST_DATABASES_W_ENTERPRISE = TEST_DATABASES + [FIRESTORE_ENTERPRISE_DB] +# TODO remove when kokoro fully supports enterprise mode/pipelines +IS_KOKORO_TEST = os.getenv("KOKORO_JOB_NAME") is not None diff --git a/tests/system/test_pipeline_acceptance.py b/tests/system/test_pipeline_acceptance.py index 682fe5e23..c7eaa6aff 100644 --- a/tests/system/test_pipeline_acceptance.py +++ b/tests/system/test_pipeline_acceptance.py @@ -33,26 +33,54 @@ from google.cloud.firestore import Client, AsyncClient -from test__helpers import FIRESTORE_ENTERPRISE_DB +from test__helpers import FIRESTORE_ENTERPRISE_DB, IS_KOKORO_TEST FIRESTORE_PROJECT = os.environ.get("GCLOUD_PROJECT") +# TODO: enable kokoro tests when internal test project is whitelisted +pytestmark = pytest.mark.skipif( + condition=IS_KOKORO_TEST, + reason="Pipeline tests are currently not supported by kokoro", +) + test_dir_name = os.path.dirname(__file__) +id_format = ( + lambda x: f"{x.get('file_name', '')}: {x.get('description', '')}" +) # noqa: E731 + -def yaml_loader(field="tests", file_name="pipeline_e2e.yaml"): +def yaml_loader(field="tests", dir_name="pipeline_e2e", attach_file_name=True): """ Helper to load test cases or data from yaml file """ - with open(f"{test_dir_name}/{file_name}") as f: - test_cases = yaml.safe_load(f) - return test_cases[field] + combined_yaml = None + for file_name in os.listdir(f"{test_dir_name}/{dir_name}"): + with open(f"{test_dir_name}/{dir_name}/{file_name}") as f: + new_yaml = yaml.safe_load(f) + assert new_yaml is not None, f"found empty yaml in {file_name}" + extracted = new_yaml.get(field, None) + # attach file_name field + if attach_file_name: + if isinstance(extracted, list): + for item in extracted: + item["file_name"] = file_name + elif isinstance(extracted, dict): + extracted["file_name"] = file_name + # aggregate files + if not combined_yaml: + combined_yaml = extracted + elif isinstance(combined_yaml, dict) and extracted: + combined_yaml.update(extracted) + elif isinstance(combined_yaml, list) and extracted: + combined_yaml.extend(extracted) + return combined_yaml @pytest.mark.parametrize( "test_dict", [t for t in yaml_loader() if "assert_proto" in t], - ids=lambda x: f"{x.get('description', '')}", + ids=id_format, ) def test_pipeline_parse_proto(test_dict, client): """ @@ -69,7 +97,7 @@ def test_pipeline_parse_proto(test_dict, client): @pytest.mark.parametrize( "test_dict", [t for t in yaml_loader() if "assert_error" in t], - ids=lambda x: f"{x.get('description', '')}", + ids=id_format, ) def test_pipeline_expected_errors(test_dict, client): """ @@ -94,7 +122,7 @@ def test_pipeline_expected_errors(test_dict, client): or "assert_count" in t or "assert_results_approximate" in t ], - ids=lambda x: f"{x.get('description', '')}", + ids=id_format, ) def test_pipeline_results(test_dict, client): """ @@ -125,7 +153,7 @@ def test_pipeline_results(test_dict, client): @pytest.mark.parametrize( "test_dict", [t for t in yaml_loader() if "assert_error" in t], - ids=lambda x: f"{x.get('description', '')}", + ids=id_format, ) @pytest.mark.asyncio async def test_pipeline_expected_errors_async(test_dict, async_client): @@ -151,7 +179,7 @@ async def test_pipeline_expected_errors_async(test_dict, async_client): or "assert_count" in t or "assert_results_approximate" in t ], - ids=lambda x: f"{x.get('description', '')}", + ids=id_format, ) @pytest.mark.asyncio async def test_pipeline_results_async(test_dict, async_client): @@ -332,7 +360,7 @@ def client(): Build a client to use for requests """ client = Client(project=FIRESTORE_PROJECT, database=FIRESTORE_ENTERPRISE_DB) - data = yaml_loader("data") + data = yaml_loader("data", attach_file_name=False) to_delete = [] try: # setup data diff --git a/tests/system/test_system.py b/tests/system/test_system.py index c2bd93ef8..ce4f64b32 100644 --- a/tests/system/test_system.py +++ b/tests/system/test_system.py @@ -45,6 +45,7 @@ ENTERPRISE_MODE_ERROR, TEST_DATABASES, TEST_DATABASES_W_ENTERPRISE, + IS_KOKORO_TEST, ) @@ -64,6 +65,12 @@ def _get_credentials_and_project(): @pytest.fixture(scope="session") def database(request): + from test__helpers import FIRESTORE_ENTERPRISE_DB + + # enterprise mode currently does not support RunQuery calls in prod on kokoro test project + # TODO: remove skip when kokoro test project supports full enterprise mode + if request.param == FIRESTORE_ENTERPRISE_DB and IS_KOKORO_TEST: + pytest.skip("enterprise mode does not support RunQuery on kokoro") return request.param @@ -92,6 +99,11 @@ def verify_pipeline(query): """ from google.cloud.firestore_v1.base_aggregation import BaseAggregationQuery + # return early on kokoro. Test project doesn't currently support pipelines + # TODO: enable pipeline verification when kokoro test project is whitelisted + if IS_KOKORO_TEST: + pytest.skip("skipping pipeline verification on kokoro") + def _clean_results(results): if isinstance(results, dict): return {k: _clean_results(v) for k, v in results.items()} @@ -2203,7 +2215,7 @@ def on_snapshot(docs, changes, read_time): ) -@pytest.mark.parametrize("database", TEST_DATABASES_W_ENTERPRISE, indirect=True) +@pytest.mark.parametrize("database", TEST_DATABASES, indirect=True) def test_watch_query(client, cleanup, database): db = client collection_ref = db.collection("wq-users" + UNIQUE_RESOURCE_ID) @@ -2224,7 +2236,6 @@ def on_snapshot(docs, changes, read_time): query_ran_query = collection_ref.where(filter=FieldFilter("first", "==", "Ada")) query_ran = query_ran_query.stream() assert len(docs) == len([i for i in query_ran]) - verify_pipeline(query_ran_query) on_snapshot.called_count = 0 @@ -2565,7 +2576,7 @@ def test_chunked_and_recursive(client, cleanup, database): assert [doc.id for doc in next(iter)] == page_3_ids -@pytest.mark.parametrize("database", TEST_DATABASES_W_ENTERPRISE, indirect=True) +@pytest.mark.parametrize("database", TEST_DATABASES, indirect=True) def test_watch_query_order(client, cleanup, database): db = client collection_ref = db.collection("users") @@ -2602,7 +2613,6 @@ def on_snapshot(docs, changes, read_time): ), "expect the sort order to match, born" on_snapshot.called_count += 1 on_snapshot.last_doc_count = len(docs) - verify_pipeline(query_ref) except Exception as e: on_snapshot.failed = e diff --git a/tests/system/test_system_async.py b/tests/system/test_system_async.py index d053cbd7a..ed679402a 100644 --- a/tests/system/test_system_async.py +++ b/tests/system/test_system_async.py @@ -56,6 +56,7 @@ ENTERPRISE_MODE_ERROR, TEST_DATABASES, TEST_DATABASES_W_ENTERPRISE, + IS_KOKORO_TEST, ) RETRIES = retries.AsyncRetry( @@ -142,6 +143,12 @@ def _verify_explain_metrics_analyze_false(explain_metrics): @pytest.fixture(scope="session") def database(request): + from test__helpers import FIRESTORE_ENTERPRISE_DB + + # enterprise mode currently does not support RunQuery calls in prod on kokoro test project + # TODO: remove skip when kokoro test project supports full enterprise mode + if request.param == FIRESTORE_ENTERPRISE_DB and IS_KOKORO_TEST: + pytest.skip("enterprise mode does not support RunQuery on kokoro") return request.param @@ -172,6 +179,11 @@ async def verify_pipeline(query): """ from google.cloud.firestore_v1.base_aggregation import BaseAggregationQuery + # return early on kokoro. Test project doesn't currently support pipelines + # TODO: enable pipeline verification when kokoro test project is whitelisted + if IS_KOKORO_TEST: + pytest.skip("skipping pipeline verification on kokoro") + def _clean_results(results): if isinstance(results, dict): return {k: _clean_results(v) for k, v in results.items()} diff --git a/tests/unit/v1/test_aggregation.py b/tests/unit/v1/test_aggregation.py index 9a20fd386..66239f9ea 100644 --- a/tests/unit/v1/test_aggregation.py +++ b/tests/unit/v1/test_aggregation.py @@ -127,12 +127,12 @@ def test_avg_aggregation_no_alias_to_pb(): "in_alias,expected_alias", [("total", "total"), (None, "field_1")] ) def test_count_aggregation_to_pipeline_expr(in_alias, expected_alias): - from google.cloud.firestore_v1.pipeline_expressions import ExprWithAlias + from google.cloud.firestore_v1.pipeline_expressions import AliasedAggregate from google.cloud.firestore_v1.pipeline_expressions import Count count_aggregation = CountAggregation(alias=in_alias) got = count_aggregation._to_pipeline_expr(iter([1])) - assert isinstance(got, ExprWithAlias) + assert isinstance(got, AliasedAggregate) assert got.alias == expected_alias assert isinstance(got.expr, Count) assert len(got.expr.params) == 0 @@ -143,14 +143,13 @@ def test_count_aggregation_to_pipeline_expr(in_alias, expected_alias): [("total", "path", "total"), (None, "some_ref", "field_1")], ) def test_sum_aggregation_to_pipeline_expr(in_alias, expected_path, expected_alias): - from google.cloud.firestore_v1.pipeline_expressions import ExprWithAlias - from google.cloud.firestore_v1.pipeline_expressions import Sum + from google.cloud.firestore_v1.pipeline_expressions import AliasedAggregate count_aggregation = SumAggregation(expected_path, alias=in_alias) got = count_aggregation._to_pipeline_expr(iter([1])) - assert isinstance(got, ExprWithAlias) + assert isinstance(got, AliasedAggregate) assert got.alias == expected_alias - assert isinstance(got.expr, Sum) + assert got.expr.name == "sum" assert got.expr.params[0].path == expected_path @@ -159,14 +158,13 @@ def test_sum_aggregation_to_pipeline_expr(in_alias, expected_path, expected_alia [("total", "path", "total"), (None, "some_ref", "field_1")], ) def test_avg_aggregation_to_pipeline_expr(in_alias, expected_path, expected_alias): - from google.cloud.firestore_v1.pipeline_expressions import ExprWithAlias - from google.cloud.firestore_v1.pipeline_expressions import Avg + from google.cloud.firestore_v1.pipeline_expressions import AliasedAggregate count_aggregation = AvgAggregation(expected_path, alias=in_alias) got = count_aggregation._to_pipeline_expr(iter([1])) - assert isinstance(got, ExprWithAlias) + assert isinstance(got, AliasedAggregate) assert got.alias == expected_alias - assert isinstance(got.expr, Avg) + assert got.expr.name == "average" assert got.expr.params[0].path == expected_path @@ -1036,7 +1034,6 @@ def test_aggregation_from_query(): def test_aggreation_to_pipeline_sum(field, in_alias, out_alias): from google.cloud.firestore_v1.pipeline import Pipeline from google.cloud.firestore_v1._pipeline_stages import Collection, Aggregate - from google.cloud.firestore_v1.pipeline_expressions import Sum client = make_client() parent = client.collection("dee") @@ -1051,7 +1048,7 @@ def test_aggreation_to_pipeline_sum(field, in_alias, out_alias): aggregate_stage = pipeline.stages[1] assert isinstance(aggregate_stage, Aggregate) assert len(aggregate_stage.accumulators) == 1 - assert isinstance(aggregate_stage.accumulators[0].expr, Sum) + assert aggregate_stage.accumulators[0].expr.name == "sum" expected_field = field if isinstance(field, str) else field.to_api_repr() assert aggregate_stage.accumulators[0].expr.params[0].path == expected_field assert aggregate_stage.accumulators[0].alias == out_alias @@ -1068,7 +1065,6 @@ def test_aggreation_to_pipeline_sum(field, in_alias, out_alias): def test_aggreation_to_pipeline_avg(field, in_alias, out_alias): from google.cloud.firestore_v1.pipeline import Pipeline from google.cloud.firestore_v1._pipeline_stages import Collection, Aggregate - from google.cloud.firestore_v1.pipeline_expressions import Avg client = make_client() parent = client.collection("dee") @@ -1083,7 +1079,7 @@ def test_aggreation_to_pipeline_avg(field, in_alias, out_alias): aggregate_stage = pipeline.stages[1] assert isinstance(aggregate_stage, Aggregate) assert len(aggregate_stage.accumulators) == 1 - assert isinstance(aggregate_stage.accumulators[0].expr, Avg) + assert aggregate_stage.accumulators[0].expr.name == "average" expected_field = field if isinstance(field, str) else field.to_api_repr() assert aggregate_stage.accumulators[0].expr.params[0].path == expected_field assert aggregate_stage.accumulators[0].alias == out_alias @@ -1142,7 +1138,6 @@ def test_aggreation_to_pipeline_count_increment(): def test_aggreation_to_pipeline_complex(): from google.cloud.firestore_v1.pipeline import Pipeline from google.cloud.firestore_v1._pipeline_stages import Collection, Aggregate, Select - from google.cloud.firestore_v1.pipeline_expressions import Sum, Avg, Count client = make_client() query = client.collection("my_col").select(["field_a", "field_b.c"]) @@ -1159,11 +1154,11 @@ def test_aggreation_to_pipeline_complex(): assert isinstance(pipeline.stages[2], Aggregate) aggregate_stage = pipeline.stages[2] assert len(aggregate_stage.accumulators) == 4 - assert isinstance(aggregate_stage.accumulators[0].expr, Sum) + assert aggregate_stage.accumulators[0].expr.name == "sum" assert aggregate_stage.accumulators[0].alias == "alias" - assert isinstance(aggregate_stage.accumulators[1].expr, Count) + assert aggregate_stage.accumulators[1].expr.name == "count" assert aggregate_stage.accumulators[1].alias == "field_1" - assert isinstance(aggregate_stage.accumulators[2].expr, Avg) + assert aggregate_stage.accumulators[2].expr.name == "average" assert aggregate_stage.accumulators[2].alias == "field_2" - assert isinstance(aggregate_stage.accumulators[3].expr, Sum) + assert aggregate_stage.accumulators[3].expr.name == "sum" assert aggregate_stage.accumulators[3].alias == "field_3" diff --git a/tests/unit/v1/test_async_aggregation.py b/tests/unit/v1/test_async_aggregation.py index 701feab5b..f51db482d 100644 --- a/tests/unit/v1/test_async_aggregation.py +++ b/tests/unit/v1/test_async_aggregation.py @@ -710,7 +710,6 @@ async def test_aggregation_query_stream_w_explain_options_analyze_false(): def test_async_aggreation_to_pipeline_sum(field, in_alias, out_alias): from google.cloud.firestore_v1.async_pipeline import AsyncPipeline from google.cloud.firestore_v1._pipeline_stages import Collection, Aggregate - from google.cloud.firestore_v1.pipeline_expressions import Sum client = make_async_client() parent = client.collection("dee") @@ -725,7 +724,7 @@ def test_async_aggreation_to_pipeline_sum(field, in_alias, out_alias): aggregate_stage = pipeline.stages[1] assert isinstance(aggregate_stage, Aggregate) assert len(aggregate_stage.accumulators) == 1 - assert isinstance(aggregate_stage.accumulators[0].expr, Sum) + assert aggregate_stage.accumulators[0].expr.name == "sum" expected_field = field if isinstance(field, str) else field.to_api_repr() assert aggregate_stage.accumulators[0].expr.params[0].path == expected_field assert aggregate_stage.accumulators[0].alias == out_alias @@ -742,7 +741,6 @@ def test_async_aggreation_to_pipeline_sum(field, in_alias, out_alias): def test_async_aggreation_to_pipeline_avg(field, in_alias, out_alias): from google.cloud.firestore_v1.async_pipeline import AsyncPipeline from google.cloud.firestore_v1._pipeline_stages import Collection, Aggregate - from google.cloud.firestore_v1.pipeline_expressions import Avg client = make_async_client() parent = client.collection("dee") @@ -757,7 +755,7 @@ def test_async_aggreation_to_pipeline_avg(field, in_alias, out_alias): aggregate_stage = pipeline.stages[1] assert isinstance(aggregate_stage, Aggregate) assert len(aggregate_stage.accumulators) == 1 - assert isinstance(aggregate_stage.accumulators[0].expr, Avg) + assert aggregate_stage.accumulators[0].expr.name == "average" expected_field = field if isinstance(field, str) else field.to_api_repr() assert aggregate_stage.accumulators[0].expr.params[0].path == expected_field assert aggregate_stage.accumulators[0].alias == out_alias @@ -816,7 +814,6 @@ def test_aggreation_to_pipeline_count_increment(): def test_async_aggreation_to_pipeline_complex(): from google.cloud.firestore_v1.async_pipeline import AsyncPipeline from google.cloud.firestore_v1._pipeline_stages import Collection, Aggregate, Select - from google.cloud.firestore_v1.pipeline_expressions import Sum, Avg, Count client = make_async_client() query = client.collection("my_col").select(["field_a", "field_b.c"]) @@ -833,11 +830,11 @@ def test_async_aggreation_to_pipeline_complex(): assert isinstance(pipeline.stages[2], Aggregate) aggregate_stage = pipeline.stages[2] assert len(aggregate_stage.accumulators) == 4 - assert isinstance(aggregate_stage.accumulators[0].expr, Sum) + assert aggregate_stage.accumulators[0].expr.name == "sum" assert aggregate_stage.accumulators[0].alias == "alias" - assert isinstance(aggregate_stage.accumulators[1].expr, Count) + assert aggregate_stage.accumulators[1].expr.name == "count" assert aggregate_stage.accumulators[1].alias == "field_1" - assert isinstance(aggregate_stage.accumulators[2].expr, Avg) + assert aggregate_stage.accumulators[2].expr.name == "average" assert aggregate_stage.accumulators[2].alias == "field_2" - assert isinstance(aggregate_stage.accumulators[3].expr, Sum) + assert aggregate_stage.accumulators[3].expr.name == "sum" assert aggregate_stage.accumulators[3].alias == "field_3" diff --git a/tests/unit/v1/test_async_pipeline.py b/tests/unit/v1/test_async_pipeline.py index b3ed83337..a11a2951b 100644 --- a/tests/unit/v1/test_async_pipeline.py +++ b/tests/unit/v1/test_async_pipeline.py @@ -386,10 +386,10 @@ async def test_async_pipeline_stream_stream_equivalence_mocked(): ("select", ("name",), stages.Select), ("select", (Field.of("n"),), stages.Select), ("where", (Field.of("n").exists(),), stages.Where), - ("find_nearest", ("name", [0.1], 0), stages.FindNearest), + ("find_nearest", ("name", [0.1], "cosine"), stages.FindNearest), ( "find_nearest", - ("name", [0.1], 0, stages.FindNearestOptions(10)), + ("name", [0.1], "cosine", stages.FindNearestOptions(10)), stages.FindNearest, ), ("sort", (Field.of("n").descending(),), stages.Sort), diff --git a/tests/unit/v1/test_base_query.py b/tests/unit/v1/test_base_query.py index 9bb3e61f8..7efa0dacf 100644 --- a/tests/unit/v1/test_base_query.py +++ b/tests/unit/v1/test_base_query.py @@ -2040,9 +2040,7 @@ def test__query_pipeline_composite_filter(): client = make_client() in_filter = FieldFilter("field_a", "==", "value_a") query = client.collection("my_col").where(filter=in_filter) - with mock.patch.object( - expr.FilterCondition, "_from_query_filter_pb" - ) as convert_mock: + with mock.patch.object(expr.BooleanExpr, "_from_query_filter_pb") as convert_mock: pipeline = query.pipeline() convert_mock.assert_called_once_with(in_filter._to_pb(), client) assert len(pipeline.stages) == 2 @@ -2080,15 +2078,13 @@ def test__query_pipeline_order_exists_multiple(): assert isinstance(where_stage.condition, expr.And) assert len(where_stage.condition.params) == 2 operands = [p for p in where_stage.condition.params] - assert isinstance(operands[0], expr.Exists) + assert operands[0].name == "exists" assert operands[0].params[0].path == "field_a" - assert isinstance(operands[1], expr.Exists) + assert operands[1].name == "exists" assert operands[1].params[0].path == "field_b" def test__query_pipeline_order_exists_single(): - from google.cloud.firestore_v1 import pipeline_expressions as expr - client = make_client() query_single = client.collection("my_col").order_by("field_c") pipeline_single = query_single.pipeline() @@ -2098,7 +2094,7 @@ def test__query_pipeline_order_exists_single(): assert len(pipeline_single.stages) == 3 where_stage_single = pipeline_single.stages[1] assert isinstance(where_stage_single, stages.Where) - assert isinstance(where_stage_single.condition, expr.Exists) + assert where_stage_single.condition.name == "exists" assert where_stage_single.condition.params[0].path == "field_c" diff --git a/tests/unit/v1/test_pipeline.py b/tests/unit/v1/test_pipeline.py index f90279e00..161eef1cc 100644 --- a/tests/unit/v1/test_pipeline.py +++ b/tests/unit/v1/test_pipeline.py @@ -363,10 +363,10 @@ def test_pipeline_execute_stream_equivalence_mocked(): ("select", ("name",), stages.Select), ("select", (Field.of("n"),), stages.Select), ("where", (Field.of("n").exists(),), stages.Where), - ("find_nearest", ("name", [0.1], 0), stages.FindNearest), + ("find_nearest", ("name", [0.1], "cosine"), stages.FindNearest), ( "find_nearest", - ("name", [0.1], 0, stages.FindNearestOptions(10)), + ("name", [0.1], "cosine", stages.FindNearestOptions(10)), stages.FindNearest, ), ("sort", (Field.of("n").descending(),), stages.Sort), diff --git a/tests/unit/v1/test_pipeline_expressions.py b/tests/unit/v1/test_pipeline_expressions.py index aec721e7d..522b51c84 100644 --- a/tests/unit/v1/test_pipeline_expressions.py +++ b/tests/unit/v1/test_pipeline_expressions.py @@ -645,6 +645,14 @@ def test_w_exprs(self): ) +class TestFunction: + def test_equals(self): + assert expr.Function.sqrt("1") == expr.Function.sqrt("1") + assert expr.Function.sqrt("1") != expr.Function.sqrt("2") + assert expr.Function.sqrt("1") != expr.Function.sum("1") + assert expr.Function.sqrt("1") != object() + + class TestExpressionMethods: """ contains test methods for each Expr method @@ -692,6 +700,22 @@ def __repr__(self): arg = MockExpr(name) return arg + def test_expression_wrong_first_type(self): + """The first argument should always be an expression or field name""" + expected_message = "must be called on an Expression or a string representing a field. got ." + with pytest.raises(TypeError) as e1: + Expr.logical_minimum(5, 1) + assert str(e1.value) == f"'logical_minimum' {expected_message}" + with pytest.raises(TypeError) as e2: + Expr.sqrt(9) + assert str(e2.value) == f"'sqrt' {expected_message}" + + def test_expression_w_string(self): + """should be able to use string for first argument. Should be interpreted as Field""" + instance = Expr.logical_minimum("first", "second") + assert isinstance(instance.params[0], Field) + assert instance.params[0].path == "first" + def test_and(self): arg1 = self._make_arg() arg2 = self._make_arg() diff --git a/tests/unit/v1/test_pipeline_stages.py b/tests/unit/v1/test_pipeline_stages.py index fadea7e12..1d2ff8760 100644 --- a/tests/unit/v1/test_pipeline_stages.py +++ b/tests/unit/v1/test_pipeline_stages.py @@ -274,12 +274,11 @@ def test_to_pb(self): instance = self._make_one("/projects/p/databases/d/documents/c/doc1", "/c/doc2") result = instance._to_pb() assert result.name == "documents" - assert len(result.args) == 1 + assert len(result.args) == 2 assert ( - result.args[0].array_value.values[0].string_value - == "/projects/p/databases/d/documents/c/doc1" + result.args[0].reference_value == "/projects/p/databases/d/documents/c/doc1" ) - assert result.args[0].array_value.values[1].string_value == "/c/doc2" + assert result.args[1].reference_value == "/c/doc2" assert len(result.options) == 0 @@ -461,10 +460,22 @@ def _make_one(self, *args, **kwargs): ), ], ) - def test_ctor(self, input_args, expected_params): + def test_ctor_with_params(self, input_args, expected_params): instance = self._make_one(*input_args) assert instance.params == expected_params + def test_ctor_with_options(self): + options = {"index_field": Field.of("index")} + field = Field.of("field") + alias = Field.of("alias") + standard_unnest = stages.Unnest( + field, alias, options=stages.UnnestOptions(**options) + ) + generic_unnest = stages.GenericStage("unnest", field, alias, options=options) + assert standard_unnest._pb_args() == generic_unnest._pb_args() + assert standard_unnest._pb_options() == generic_unnest._pb_options() + assert standard_unnest._to_pb() == generic_unnest._to_pb() + @pytest.mark.parametrize( "input_args,expected", [ @@ -709,7 +720,8 @@ def _make_one_options(self, *args, **kwargs): def test_ctor_options(self): index_field_val = "my_index" instance = self._make_one_options(index_field=index_field_val) - assert instance.index_field == index_field_val + assert isinstance(instance.index_field, Field) + assert instance.index_field.path == index_field_val def test_repr(self): instance = self._make_one_options(index_field="my_idx") @@ -781,7 +793,7 @@ def test_to_pb_full(self): assert result.args[1].field_reference_value == alias_str assert len(result.options) == 1 - assert result.options["index_field"].string_value == "item_index" + assert result.options["index_field"].field_reference_value == "item_index" class TestWhere: