Fix IsNull operator (#330)

**Related Issue(s):**

- #300 

**Description:**

- Upgrade `pygeofilter` to 0.3.1
- Resolves the issue with non-functional `IsNull` operator described in
#300.
- Includes a bugfix for datetime querying
- Typo fix :)

**PR Checklist:**

- [x] Code is formatted and linted (run `pre-commit run --all-files`)
- [x] Tests pass (run `make test`)
- [x] Documentation has been updated to reflect changes, if applicable
- [x] Changes are added to the changelog

---------

Co-authored-by: Jonathan Healy <[email protected]>

Author: jamesfisher-geo

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Changed
 
 - Added note on the use of the default `*` use in route authentication dependecies. [#325](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/325)
+- Bugfixes for the `IsNull` operator and datetime filtering [#330](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/330)
 
 ## [v3.2.2] - 2024-12-15
 

README.md

@@ -317,7 +317,7 @@ Authentication is an optional feature that can be enabled through `Route Depende
 
 ## Aggregation
 
-Aggregation of points and geometries, as well as frequency distribution aggregation of any other property including dates is supported in stac-fatsapi-elasticsearch-opensearch. Aggregations can be defined at the root Catalog level (`/aggregations`) and at the Collection level (`/<collection_id>/aggregations`). Details for supported aggregations can be found at [./docs/src/aggregation.md](./docs/src/aggregation.md)
+Aggregation of points and geometries, as well as frequency distribution aggregation of any other property including dates is supported in stac-fatsapi-elasticsearch-opensearch. Aggregations can be defined at the root Catalog level (`/aggregations`) and at the Collection level (`/<collection_id>/aggregations`). Details for supported aggregations can be found in [the aggregation docs](./docs/src/aggregation.md)
 
 ## Rate Limiting
 

docs/src/aggregation.md

@@ -1,6 +1,6 @@
 ## Aggregation
 
-Stac-fatsapi-elasticsearch-opensearch supports the STAC API [Aggregation Extension](https://github.com/stac-api-extensions/aggregation). This enables aggregation of points and geometries, as well as frequency distribution aggregation of any other property including dates. Aggregations can be defined at the root Catalog level (`/aggregations`) and at the Collection level (`/<collection_id>/aggregations`). The [Filter Extension](https://github.com/stac-api-extensions/filter) is also fully supported, enabling aggregated returns of search queries. Any query made with `/search` may also be executed with `/aggregate`, provided that the relevant aggregation fields are available,
+Stac-fastapi-elasticsearch-opensearch supports the STAC API [Aggregation Extension](https://github.com/stac-api-extensions/aggregation). This enables aggregation of points and geometries, as well as frequency distribution aggregation of any other property including dates. Aggregations can be defined at the root Catalog level (`/aggregations`) and at the Collection level (`/<collection_id>/aggregations`). The [Filter Extension](https://github.com/stac-api-extensions/filter) is also fully supported, enabling aggregated returns of search queries. Any query made with `/search` may also be executed with `/aggregate`, provided that the relevant aggregation fields are available,
 
 A field named `aggregations` should be added to the Collection object for the collection for which the aggregations are available, for example:
 

stac_fastapi/core/setup.py

@@ -16,7 +16,7 @@
     "orjson",
     "overrides",
     "geojson-pydantic",
-    "pygeofilter==0.2.1",
+    "pygeofilter==0.3.1",
     "typing_extensions==4.8.0",
     "jsonschema",
     "slowapi==0.1.9",

stac_fastapi/core/stac_fastapi/core/extensions/filter.py

@@ -140,26 +140,38 @@ def to_es(query: Dict[str, Any]) -> Dict[str, Any]:
         ComparisonOp.GT,
         ComparisonOp.GTE,
     ]:
+        range_op = {
+            ComparisonOp.LT: "lt",
+            ComparisonOp.LTE: "lte",
+            ComparisonOp.GT: "gt",
+            ComparisonOp.GTE: "gte",
+        }
+
         field = to_es_field(query["args"][0]["property"])
         value = query["args"][1]
         if isinstance(value, dict) and "timestamp" in value:
-            # Handle timestamp fields specifically
             value = value["timestamp"]
-        if query["op"] == ComparisonOp.IS_NULL:
-            return {"bool": {"must_not": {"exists": {"field": field}}}}
+            if query["op"] == ComparisonOp.EQ:
+                return {"range": {field: {"gte": value, "lte": value}}}
+            elif query["op"] == ComparisonOp.NEQ:
+                return {
+                    "bool": {
+                        "must_not": [{"range": {field: {"gte": value, "lte": value}}}]
+                    }
+                }
+            else:
+                return {"range": {field: {range_op[query["op"]]: value}}}
         else:
             if query["op"] == ComparisonOp.EQ:
                 return {"term": {field: value}}
             elif query["op"] == ComparisonOp.NEQ:
                 return {"bool": {"must_not": [{"term": {field: value}}]}}
             else:
-                range_op = {
-                    ComparisonOp.LT: "lt",
-                    ComparisonOp.LTE: "lte",
-                    ComparisonOp.GT: "gt",
-                    ComparisonOp.GTE: "gte",
-                }[query["op"]]
-                return {"range": {field: {range_op: value}}}
+                return {"range": {field: {range_op[query["op"]]: value}}}
+
+    elif query["op"] == ComparisonOp.IS_NULL:
+        field = to_es_field(query["args"][0]["property"])
+        return {"bool": {"must_not": {"exists": {"field": field}}}}
 
     elif query["op"] == AdvancedComparisonOp.BETWEEN:
         field = to_es_field(query["args"][0]["property"])

stac_fastapi/tests/extensions/test_filter.py

@@ -1,4 +1,5 @@
 import json
+import logging
 import os
 from os import listdir
 from os.path import isfile, join
@@ -48,6 +49,10 @@ async def test_search_filters_post(app_client, ctx):
 
     for _filter in filters:
         resp = await app_client.post("/search", json={"filter": _filter})
+        if resp.status_code != 200:
+            logging.error(f"Failed with status {resp.status_code}")
+            logging.error(f"Response body: {resp.json()}")
+            logging.error({"filter": _filter})
         assert resp.status_code == 200
 
 
@@ -431,3 +436,48 @@ async def test_search_filter_extension_between(app_client, ctx):
 
     assert resp.status_code == 200
     assert len(resp.json()["features"]) == 1
+
+
+@pytest.mark.asyncio
+async def test_search_filter_extension_isnull_post(app_client, ctx):
+    # Test for a property that is not null
+    params = {
+        "filter-lang": "cql2-json",
+        "filter": {
+            "op": "isNull",
+            "args": [{"property": "properties.view:sun_elevation"}],
+        },
+    }
+    resp = await app_client.post("/search", json=params)
+
+    assert resp.status_code == 200
+    assert len(resp.json()["features"]) == 0
+
+    # Test for the property that is null
+    params = {
+        "filter-lang": "cql2-json",
+        "filter": {
+            "op": "isNull",
+            "args": [{"property": "properties.thispropertyisnull"}],
+        },
+    }
+    resp = await app_client.post("/search", json=params)
+
+    assert resp.status_code == 200
+    assert len(resp.json()["features"]) == 1
+
+
+@pytest.mark.asyncio
+async def test_search_filter_extension_isnull_get(app_client, ctx):
+    # Test for a property that is not null
+
+    resp = await app_client.get("/search?filter=properties.view:sun_elevation IS NULL")
+
+    assert resp.status_code == 200
+    assert len(resp.json()["features"]) == 0
+
+    # Test for the property that is null
+    resp = await app_client.get("/search?filter=properties.thispropertyisnull IS NULL")
+
+    assert resp.status_code == 200
+    assert len(resp.json()["features"]) == 1